View Javadoc
1   /*
2    * Copyright (c) 2002-2026 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit.html;
16  
17  import static org.htmlunit.BrowserVersionFeatures.EVENT_FOCUS_ON_LOAD;
18  import static org.htmlunit.BrowserVersionFeatures.HTTP_HEADER_CH_UA;
19  import static org.htmlunit.html.DomElement.ATTRIBUTE_NOT_DEFINED;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.io.ObjectInputStream;
24  import java.io.ObjectOutputStream;
25  import java.io.Serializable;
26  import java.net.MalformedURLException;
27  import java.net.URL;
28  import java.nio.charset.Charset;
29  import java.nio.charset.StandardCharsets;
30  import java.util.ArrayList;
31  import java.util.Arrays;
32  import java.util.Collection;
33  import java.util.Collections;
34  import java.util.Comparator;
35  import java.util.HashMap;
36  import java.util.HashSet;
37  import java.util.Iterator;
38  import java.util.LinkedHashSet;
39  import java.util.List;
40  import java.util.Locale;
41  import java.util.Map;
42  import java.util.Set;
43  import java.util.WeakHashMap;
44  import java.util.concurrent.ConcurrentHashMap;
45  
46  import org.apache.commons.lang3.StringUtils;
47  import org.apache.commons.logging.Log;
48  import org.apache.commons.logging.LogFactory;
49  import org.htmlunit.Cache;
50  import org.htmlunit.ElementNotFoundException;
51  import org.htmlunit.FailingHttpStatusCodeException;
52  import org.htmlunit.History;
53  import org.htmlunit.HttpHeader;
54  import org.htmlunit.OnbeforeunloadHandler;
55  import org.htmlunit.Page;
56  import org.htmlunit.ScriptResult;
57  import org.htmlunit.SgmlPage;
58  import org.htmlunit.TopLevelWindow;
59  import org.htmlunit.WebAssert;
60  import org.htmlunit.WebClient;
61  import org.htmlunit.WebClientOptions;
62  import org.htmlunit.WebRequest;
63  import org.htmlunit.WebResponse;
64  import org.htmlunit.WebWindow;
65  import org.htmlunit.corejs.javascript.Function;
66  import org.htmlunit.corejs.javascript.Script;
67  import org.htmlunit.corejs.javascript.Scriptable;
68  import org.htmlunit.corejs.javascript.ScriptableObject;
69  import org.htmlunit.corejs.javascript.VarScope;
70  import org.htmlunit.css.ComputedCssStyleDeclaration;
71  import org.htmlunit.css.CssStyleSheet;
72  import org.htmlunit.html.impl.SimpleRange;
73  import org.htmlunit.html.parser.HTMLParserDOMBuilder;
74  import org.htmlunit.http.HttpStatus;
75  import org.htmlunit.javascript.AbstractJavaScriptEngine;
76  import org.htmlunit.javascript.HtmlUnitScriptable;
77  import org.htmlunit.javascript.JavaScriptEngine;
78  import org.htmlunit.javascript.PostponedAction;
79  import org.htmlunit.javascript.host.Window;
80  import org.htmlunit.javascript.host.event.BeforeUnloadEvent;
81  import org.htmlunit.javascript.host.event.Event;
82  import org.htmlunit.javascript.host.event.EventTarget;
83  import org.htmlunit.javascript.host.html.HTMLDocument;
84  import org.htmlunit.protocol.javascript.JavaScriptURLConnection;
85  import org.htmlunit.util.MimeType;
86  import org.htmlunit.util.SerializableLock;
87  import org.htmlunit.util.UrlUtils;
88  import org.w3c.dom.Attr;
89  import org.w3c.dom.Comment;
90  import org.w3c.dom.DOMConfiguration;
91  import org.w3c.dom.DOMException;
92  import org.w3c.dom.DOMImplementation;
93  import org.w3c.dom.Document;
94  import org.w3c.dom.DocumentType;
95  import org.w3c.dom.Element;
96  import org.w3c.dom.EntityReference;
97  import org.w3c.dom.ProcessingInstruction;
98  
99  /**
100  * A representation of an HTML page returned from a server.
101  * <p>
102  * This class provides different methods to access the page's content like
103  * {@link #getForms()}, {@link #getAnchors()}, {@link #getElementById(String)}, ... as well as the
104  * very powerful inherited methods {@link #getByXPath(String)} and {@link #getFirstByXPath(String)}
105  * for fine grained user specific access to child nodes.
106  * </p>
107  * <p>
108  * Child elements allowing user interaction provide methods for this purpose like {@link HtmlAnchor#click()},
109  * {@link HtmlInput#type(String)}, {@link HtmlOption#setSelected(boolean)}, ...
110  * </p>
111  * <p>
112  * HtmlPage instances should not be instantiated directly. They will be returned by {@link WebClient#getPage(String)}
113  * when the content type of the server's response is <code>text/html</code> (or one of its variations).<br>
114  * <br>
115  * <b>Example:</b><br>
116  * <br>
117  * <code>
118  * final HtmlPage page = webClient.{@link WebClient#getPage(String) getPage}("http://mywebsite/some/page.html");
119  * </code>
120  * </p>
121  *
122  * @author Mike Bowler
123  * @author Alex Nikiforoff
124  * @author Noboru Sinohara
125  * @author David K. Taylor
126  * @author Andreas Hangler
127  * @author Christian Sell
128  * @author Chris Erskine
129  * @author Marc Guillemot
130  * @author Ahmed Ashour
131  * @author Daniel Gredler
132  * @author Dmitri Zoubkov
133  * @author Sudhan Moghe
134  * @author Ethan Glasser-Camp
135  * @author Tom Anderson
136  * @author Ronald Brill
137  * @author Frank Danek
138  * @author Joerg Werner
139  * @author Atsushi Nakagawa
140  * @author Rural Hunter
141  * @author Ronny Shapiro
142  * @author Lai Quang Duong
143  * @author Sven Strickroth
144  */
145 @SuppressWarnings("PMD.TooManyFields")
146 public class HtmlPage extends SgmlPage {
147 
148     private static final Log LOG = LogFactory.getLog(HtmlPage.class);
149 
150     private static final Comparator<DomElement> DOCUMENT_POSITION_COMPERATOR = new DocumentPositionComparator();
151 
152     private HTMLParserDOMBuilder domBuilder_;
153     private transient Charset originalCharset_;
154     private final Object lock_ = new SerializableLock(); // used for synchronization
155 
156     private Map<String, MappedElementIndexEntry> idMap_ = new ConcurrentHashMap<>();
157     private Map<String, MappedElementIndexEntry> nameMap_ = new ConcurrentHashMap<>();
158     // The id/name lookup index is built lazily on first use. Until then,
159     // notifyNodeAdded / fireAttributeChange skip the per-element index updates.
160     // Reads must call ensureMappedElementsBuilt() before consulting idMap_/nameMap_.
161     private boolean mappedElementsBuilt_;
162 
163     private List<BaseFrameElement> frameElements_ = new ArrayList<>();
164     private int parserCount_;
165     private int snippetParserCount_;
166     private int inlineSnippetParserCount_;
167     private Collection<HtmlAttributeChangeListener> attributeListeners_;
168     private List<PostponedAction> afterLoadActions_ = Collections.synchronizedList(new ArrayList<>());
169     private boolean cleaning_;
170     private HtmlBase base_;
171     private URL baseUrl_;
172     private List<AutoCloseable> autoCloseableList_;
173     private ElementFromPointHandler elementFromPointHandler_;
174     private DomElement elementWithFocus_;
175     private List<SimpleRange> selectionRanges_ = new ArrayList<>(3);
176 
177     private transient ComputedStylesCache computedStylesCache_;
178 
179     private static final HashSet<String> TABBABLE_TAGS =
180             new HashSet<>(Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
181                     HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlObject.TAG_NAME,
182                     HtmlSelect.TAG_NAME, HtmlTextArea.TAG_NAME));
183     private static final HashSet<String> ACCEPTABLE_TAG_NAMES =
184             new HashSet<>(Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
185                     HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlLabel.TAG_NAME,
186                     HtmlLegend.TAG_NAME, HtmlTextArea.TAG_NAME));
187 
188     /** Definition of special cases for the smart DomHtmlAttributeChangeListenerImpl. */
189     private static final Set<String> ATTRIBUTES_AFFECTING_PARENT = new HashSet<>(Arrays.asList(
190             "style",
191             "class",
192             "height",
193             "width"));
194 
195     static class DocumentPositionComparator implements Comparator<DomElement>, Serializable {
196         @Override
197         public int compare(final DomElement elt1, final DomElement elt2) {
198             final short relation = elt1.compareDocumentPosition(elt2);
199             if (relation == 0) {
200                 return 0; // same node
201             }
202             if ((relation & DOCUMENT_POSITION_CONTAINS) != 0 || (relation & DOCUMENT_POSITION_PRECEDING) != 0) {
203                 return 1;
204             }
205 
206             return -1;
207         }
208     }
209 
210     /**
211      * Creates an instance of HtmlPage.
212      * An HtmlPage instance is normally retrieved with {@link WebClient#getPage(String)}.
213      *
214      * @param webResponse the web response that was used to create this page
215      * @param webWindow the window that this page is being loaded into
216      */
217     public HtmlPage(final WebResponse webResponse, final WebWindow webWindow) {
218         super(webResponse, webWindow);
219     }
220 
221     /**
222      * {@inheritDoc}
223      */
224     @Override
225     public HtmlPage getPage() {
226         return this;
227     }
228 
229     /**
230      * {@inheritDoc}
231      */
232     @Override
233     public boolean hasCaseSensitiveTagNames() {
234         return false;
235     }
236 
237     /**
238      * Initialize this page.
239      * @throws IOException if an IO problem occurs
240      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
241      *         {@link org.htmlunit.WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set
242      *         to true.
243      */
244     @Override
245     public void initialize() throws IOException, FailingHttpStatusCodeException {
246         final WebWindow enclosingWindow = getEnclosingWindow();
247         final boolean isAboutBlank = getUrl() == UrlUtils.URL_ABOUT_BLANK;
248         if (isAboutBlank) {
249             // a frame contains first a faked "about:blank" before its real content specified by src gets loaded
250             if (enclosingWindow instanceof FrameWindow window
251                     && !window.getFrameElement().isContentLoaded()) {
252                 return;
253             }
254 
255             // save the URL that should be used to resolve relative URLs in this page
256             if (enclosingWindow instanceof TopLevelWindow topWindow) {
257                 final WebWindow openerWindow = topWindow.getOpener();
258                 if (openerWindow != null && openerWindow.getEnclosedPage() != null) {
259                     baseUrl_ = openerWindow.getEnclosedPage().getWebResponse().getWebRequest().getUrl();
260                 }
261             }
262         }
263 
264         if (!isAboutBlank) {
265             setReadyState(READY_STATE_INTERACTIVE);
266             getDocumentElement().setReadyState(READY_STATE_INTERACTIVE);
267             executeEventHandlersIfNeeded(Event.TYPE_READY_STATE_CHANGE);
268         }
269 
270         executeDeferredScriptsIfNeeded();
271 
272         executeEventHandlersIfNeeded(Event.TYPE_DOM_DOCUMENT_LOADED);
273 
274         // postponed actions are more or less the async scripts,
275         // they are running in real browsers whenever the download is done
276         processPostponedActionsIfNeeded();
277 
278         loadFrames();
279 
280         // don't set the ready state if we really load the blank page into the window
281         // see Node.initInlineFrameIfNeeded()
282         if (!isAboutBlank) {
283             setReadyState(READY_STATE_COMPLETE);
284             getDocumentElement().setReadyState(READY_STATE_COMPLETE);
285             executeEventHandlersIfNeeded(Event.TYPE_READY_STATE_CHANGE);
286         }
287 
288         // frame initialization has a different order
289         boolean isFrameWindow = enclosingWindow instanceof FrameWindow;
290         boolean isFirstPageInFrameWindow = false;
291         if (isFrameWindow) {
292             isFrameWindow = ((FrameWindow) enclosingWindow).getFrameElement() instanceof HtmlFrame;
293 
294             final History hist = enclosingWindow.getHistory();
295             if (hist.getLength() > 0 && UrlUtils.URL_ABOUT_BLANK == hist.getUrl(0)) {
296                 isFirstPageInFrameWindow = hist.getLength() <= 2;
297             }
298             else {
299                 isFirstPageInFrameWindow = enclosingWindow.getHistory().getLength() < 2;
300             }
301         }
302 
303         if (isFrameWindow && !isFirstPageInFrameWindow) {
304             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
305         }
306 
307         for (final BaseFrameElement frameElement : new ArrayList<>(frameElements_)) {
308             if (frameElement instanceof HtmlFrame) {
309                 final Page page = frameElement.getEnclosedWindow().getEnclosedPage();
310                 if (page != null && page.isHtmlPage()) {
311                     ((HtmlPage) page).executeEventHandlersIfNeeded(Event.TYPE_LOAD);
312                 }
313             }
314         }
315 
316         if (!isFrameWindow) {
317             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
318 
319             if (!isAboutBlank && enclosingWindow.getWebClient().isJavaScriptEnabled()
320                     && hasFeature(EVENT_FOCUS_ON_LOAD)) {
321                 final HtmlElement body = getBody();
322                 if (body != null) {
323                     final Event event = new Event((Window) enclosingWindow.getScriptableObject(), Event.TYPE_FOCUS);
324                     body.fireEvent(event);
325                 }
326             }
327         }
328 
329         try {
330             while (!afterLoadActions_.isEmpty()) {
331                 final PostponedAction action = afterLoadActions_.remove(0);
332                 action.execute();
333             }
334         }
335         catch (final IOException e) {
336             throw e;
337         }
338         catch (final Exception e) {
339             throw new RuntimeException(e);
340         }
341         executeRefreshIfNeeded();
342     }
343 
344     /**
345      * Adds an action that should be executed once the page has been loaded.
346      * @param action the action
347      */
348     void addAfterLoadAction(final PostponedAction action) {
349         afterLoadActions_.add(action);
350     }
351 
352     /**
353      * Clean up this page.
354      */
355     @Override
356     public void cleanUp() {
357         //To avoid endless recursion caused by window.close() in onUnload
358         if (cleaning_) {
359             return;
360         }
361 
362         cleaning_ = true;
363         try {
364             super.cleanUp();
365             executeEventHandlersIfNeeded(Event.TYPE_UNLOAD);
366             deregisterFramesIfNeeded();
367         }
368         finally {
369             cleaning_ = false;
370 
371             if (autoCloseableList_ != null) {
372                 for (final AutoCloseable closeable : new ArrayList<>(autoCloseableList_)) {
373                     try {
374                         closeable.close();
375                     }
376                     catch (final Exception e) {
377                         LOG.error("Closing the autoclosable " + closeable + " failed", e);
378                     }
379                 }
380             }
381         }
382     }
383 
384     /**
385      * {@inheritDoc}
386      */
387     @Override
388     public HtmlElement getDocumentElement() {
389         return (HtmlElement) super.getDocumentElement();
390     }
391 
392     /**
393      * Returns the document's {@code body} element.
394      *
395      * @return the document's {@code body} element, or {@code null} if it does
396      *         not exist
397      */
398     public HtmlBody getBody() {
399         final DomElement doc = getDocumentElement();
400         if (doc != null) {
401             for (final DomNode node : doc.getChildren()) {
402                 if (node instanceof HtmlBody body) {
403                     return body;
404                 }
405             }
406         }
407         return null;
408     }
409 
410     /**
411      * Returns the head element.
412      * @return the head element
413      */
414     public HtmlElement getHead() {
415         final DomElement doc = getDocumentElement();
416         if (doc != null) {
417             for (final DomNode node : doc.getChildren()) {
418                 if (node instanceof HtmlHead) {
419                     return (HtmlElement) node;
420                 }
421             }
422         }
423         return null;
424     }
425 
426     /**
427      * {@inheritDoc}
428      */
429     @Override
430     public Document getOwnerDocument() {
431         return null;
432     }
433 
434     /**
435      * {@inheritDoc}
436      * Not yet implemented.
437      */
438     @Override
439     public org.w3c.dom.Node importNode(final org.w3c.dom.Node importedNode, final boolean deep) {
440         throw new UnsupportedOperationException("HtmlPage.importNode is not yet implemented.");
441     }
442 
443     /**
444      * {@inheritDoc}
445      * Not yet implemented.
446      */
447     @Override
448     public String getInputEncoding() {
449         throw new UnsupportedOperationException("HtmlPage.getInputEncoding is not yet implemented.");
450     }
451 
452     /**
453      * {@inheritDoc}
454      */
455     @Override
456     public String getXmlEncoding() {
457         return null;
458     }
459 
460     /**
461      * {@inheritDoc}
462      */
463     @Override
464     public boolean getXmlStandalone() {
465         return false;
466     }
467 
468     /**
469      * {@inheritDoc}
470      * Not yet implemented.
471      */
472     @Override
473     public void setXmlStandalone(final boolean xmlStandalone) throws DOMException {
474         throw new UnsupportedOperationException("HtmlPage.setXmlStandalone is not yet implemented.");
475     }
476 
477     /**
478      * {@inheritDoc}
479      */
480     @Override
481     public String getXmlVersion() {
482         return null;
483     }
484 
485     /**
486      * {@inheritDoc}
487      * Not yet implemented.
488      */
489     @Override
490     public void setXmlVersion(final String xmlVersion) throws DOMException {
491         throw new UnsupportedOperationException("HtmlPage.setXmlVersion is not yet implemented.");
492     }
493 
494     /**
495      * {@inheritDoc}
496      * Not yet implemented.
497      */
498     @Override
499     public boolean getStrictErrorChecking() {
500         throw new UnsupportedOperationException("HtmlPage.getStrictErrorChecking is not yet implemented.");
501     }
502 
503     /**
504      * {@inheritDoc}
505      * Not yet implemented.
506      */
507     @Override
508     public void setStrictErrorChecking(final boolean strictErrorChecking) {
509         throw new UnsupportedOperationException("HtmlPage.setStrictErrorChecking is not yet implemented.");
510     }
511 
512     /**
513      * {@inheritDoc}
514      * Not yet implemented.
515      */
516     @Override
517     public String getDocumentURI() {
518         throw new UnsupportedOperationException("HtmlPage.getDocumentURI is not yet implemented.");
519     }
520 
521     /**
522      * {@inheritDoc}
523      * Not yet implemented.
524      */
525     @Override
526     public void setDocumentURI(final String documentURI) {
527         throw new UnsupportedOperationException("HtmlPage.setDocumentURI is not yet implemented.");
528     }
529 
530     /**
531      * {@inheritDoc}
532      * Not yet implemented.
533      */
534     @Override
535     public org.w3c.dom.Node adoptNode(final org.w3c.dom.Node source) throws DOMException {
536         throw new UnsupportedOperationException("HtmlPage.adoptNode is not yet implemented.");
537     }
538 
539     /**
540      * {@inheritDoc}
541      * Not yet implemented.
542      */
543     @Override
544     public DOMConfiguration getDomConfig() {
545         throw new UnsupportedOperationException("HtmlPage.getDomConfig is not yet implemented.");
546     }
547 
548     /**
549      * {@inheritDoc}
550      * Not yet implemented.
551      */
552     @Override
553     public org.w3c.dom.Node renameNode(final org.w3c.dom.Node newNode, final String namespaceURI,
554         final String qualifiedName) throws DOMException {
555         throw new UnsupportedOperationException("HtmlPage.renameNode is not yet implemented.");
556     }
557 
558     /**
559      * {@inheritDoc}
560      */
561     @Override
562     public Charset getCharset() {
563         if (originalCharset_ == null) {
564             originalCharset_ = getWebResponse().getContentCharset();
565         }
566         return originalCharset_;
567     }
568 
569     /**
570      * {@inheritDoc}
571      */
572     @Override
573     public String getContentType() {
574         return getWebResponse().getContentType();
575     }
576 
577     /**
578      * {@inheritDoc}
579      * Not yet implemented.
580      */
581     @Override
582     public DOMImplementation getImplementation() {
583         throw new UnsupportedOperationException("HtmlPage.getImplementation is not yet implemented.");
584     }
585 
586     /**
587      * {@inheritDoc}
588      * @param tagName the tag name, preferably in lowercase
589      */
590     @Override
591     public DomElement createElement(String tagName) {
592         if (tagName.indexOf(':') == -1) {
593             tagName = org.htmlunit.util.StringUtils.toRootLowerCase(tagName);
594         }
595         return getWebClient().getPageCreator().getHtmlParser().getFactory(tagName)
596                     .createElementNS(this, null, tagName, null);
597     }
598 
599     /**
600      * {@inheritDoc}
601      */
602     @Override
603     public DomElement createElementNS(final String namespaceURI, final String qualifiedName) {
604         return getWebClient().getPageCreator().getHtmlParser()
605                 .getElementFactory(this, namespaceURI, qualifiedName, false, true)
606                 .createElementNS(this, namespaceURI, qualifiedName, null);
607     }
608 
609     /**
610      * {@inheritDoc}
611      * Not yet implemented.
612      */
613     @Override
614     public Attr createAttributeNS(final String namespaceURI, final String qualifiedName) {
615         throw new UnsupportedOperationException("HtmlPage.createAttributeNS is not yet implemented.");
616     }
617 
618     /**
619      * {@inheritDoc}
620      * Not yet implemented.
621      */
622     @Override
623     public EntityReference createEntityReference(final String id) {
624         throw new UnsupportedOperationException("HtmlPage.createEntityReference is not yet implemented.");
625     }
626 
627     /**
628      * {@inheritDoc}
629      * Not yet implemented.
630      */
631     @Override
632     public ProcessingInstruction createProcessingInstruction(final String namespaceURI, final String qualifiedName) {
633         throw new UnsupportedOperationException("HtmlPage.createProcessingInstruction is not yet implemented.");
634     }
635 
636     /**
637      * {@inheritDoc}
638      */
639     @Override
640     public DomElement getElementById(final String elementId) {
641         if (elementId != null) {
642             ensureMappedElementsBuilt();
643             final MappedElementIndexEntry elements = idMap_.get(elementId);
644             if (elements != null) {
645                 return elements.first();
646             }
647         }
648         return null;
649     }
650 
651     /**
652      * Returns the {@link HtmlAnchor} with the specified name.
653      *
654      * @param name the name to search by
655      * @return the {@link HtmlAnchor} with the specified name
656      * @throws ElementNotFoundException if the anchor could not be found
657      */
658     public HtmlAnchor getAnchorByName(final String name) throws ElementNotFoundException {
659         return getDocumentElement().getOneHtmlElementByAttribute("a", DomElement.NAME_ATTRIBUTE, name);
660     }
661 
662     /**
663      * Returns the {@link HtmlAnchor} with the specified href.
664      *
665      * @param href the string to search by
666      * @return the HtmlAnchor
667      * @throws ElementNotFoundException if the anchor could not be found
668      */
669     public HtmlAnchor getAnchorByHref(final String href) throws ElementNotFoundException {
670         return getDocumentElement().getOneHtmlElementByAttribute("a", "href", href);
671     }
672 
673     /**
674      * Returns a list of all anchors contained in this page.
675      * @return the list of {@link HtmlAnchor} in this page
676      */
677     public List<HtmlAnchor> getAnchors() {
678         return getDocumentElement().getElementsByTagNameImpl("a");
679     }
680 
681     /**
682      * Returns the first anchor with the specified text.
683      * @param text the text to search for
684      * @return the first anchor that was found
685      * @throws ElementNotFoundException if no anchors are found with the specified text
686      */
687     public HtmlAnchor getAnchorByText(final String text) throws ElementNotFoundException {
688         WebAssert.notNull("text", text);
689 
690         for (final HtmlAnchor anchor : getAnchors()) {
691             if (text.equals(anchor.asNormalizedText())) {
692                 return anchor;
693             }
694         }
695         throw new ElementNotFoundException("a", "<text>", text);
696     }
697 
698     /**
699      * Returns the first form that matches the specified name.
700      * @param name the name to search for
701      * @return the first form
702      * @throws ElementNotFoundException If no forms match the specified result.
703      */
704     public HtmlForm getFormByName(final String name) throws ElementNotFoundException {
705         final List<HtmlForm> forms = getDocumentElement()
706                 .getElementsByAttribute("form", DomElement.NAME_ATTRIBUTE, name);
707         if (forms.isEmpty()) {
708             throw new ElementNotFoundException("form", DomElement.NAME_ATTRIBUTE, name);
709         }
710         return forms.get(0);
711     }
712 
713     /**
714      * Returns a list of all the forms in this page.
715      * @return all the forms in this page
716      */
717     public List<HtmlForm> getForms() {
718         return getDocumentElement().getElementsByTagNameImpl("form");
719     }
720 
721     /**
722      * Given a relative URL (ie <code>/foo</code>), returns a fully-qualified URL based on
723      * the URL that was used to load this page.
724      *
725      * @param relativeUrl the relative URL
726      * @return the fully-qualified URL for the specified relative URL
727      * @throws MalformedURLException if an error occurred when creating a URL object
728      */
729     public URL getFullyQualifiedUrl(String relativeUrl) throws MalformedURLException {
730         // to handle http: and http:/ in FF (Bug #474)
731         boolean incorrectnessNotified = false;
732         while (relativeUrl.startsWith("http:") && !relativeUrl.startsWith("http://")) {
733             if (!incorrectnessNotified) {
734                 notifyIncorrectness("Incorrect URL \"" + relativeUrl + "\" has been corrected");
735                 incorrectnessNotified = true;
736             }
737             relativeUrl = "http:/" + relativeUrl.substring(5);
738         }
739 
740         return WebClient.expandUrl(getBaseURL(), relativeUrl);
741     }
742 
743     /**
744      * Given a target attribute value, resolve the target using a base target for the page.
745      *
746      * @param elementTarget the target specified as an attribute of the element
747      * @return the resolved target to use for the element
748      */
749     public String getResolvedTarget(final String elementTarget) {
750         final String resolvedTarget;
751         if (base_ == null) {
752             resolvedTarget = elementTarget;
753         }
754         else if (elementTarget != null && !elementTarget.isEmpty()) {
755             resolvedTarget = elementTarget;
756         }
757         else {
758             resolvedTarget = base_.getTargetAttribute();
759         }
760         return resolvedTarget;
761     }
762 
763     /**
764      * Returns a list of ids (strings) that correspond to the tabbable elements
765      * in this page. Return them in the same order specified in {@link #getTabbableElements}
766      *
767      * @return the list of id's
768      */
769     public List<String> getTabbableElementIds() {
770         final List<String> list = new ArrayList<>();
771 
772         for (final HtmlElement element : getTabbableElements()) {
773             list.add(element.getId());
774         }
775 
776         return Collections.unmodifiableList(list);
777     }
778 
779     /**
780      * Returns a list of all elements that are tabbable in the order that will
781      * be used for tabbing.
782      * <p>
783      * The rules for determining tab order are as follows:
784      * </p>
785      * <ol>
786      *   <li>Those elements that support the tabindex attribute and assign a
787      *   positive value to it are navigated first. Navigation proceeds from the
788      *   element with the lowest tabindex value to the element with the highest
789      *   value. Values need not be sequential nor must they begin with any
790      *   particular value. Elements that have identical tabindex values should
791      *   be navigated in the order they appear in the character stream.</li>
792      *   <li>Those elements that do not support the tabindex attribute or
793      *   support it and assign it a value of "0" are navigated next. These
794      *   elements are navigated in the order they appear in the character
795      *   stream.</li>
796      *   <li>Elements that are disabled do not participate in the tabbing
797      *   order.</li>
798      * </ol>
799      * <p>
800      * Additionally, the value of tabindex must be within 0 and 32767. Any
801      * values outside this range will be ignored.
802      * </p>
803      * <p>
804      * The following elements support the <code>tabindex</code> attribute:
805      * A, AREA, BUTTON, INPUT, OBJECT, SELECT, and TEXTAREA.
806      * </p>
807      *
808      * @return all the tabbable elements in proper tab order
809      */
810     public List<HtmlElement> getTabbableElements() {
811         final List<HtmlElement> tabbableElements = new ArrayList<>();
812         for (final HtmlElement element : getHtmlElementDescendants()) {
813             final String tagName = element.getTagName();
814             if (TABBABLE_TAGS.contains(tagName)) {
815                 final boolean disabled = element.isDisabledElementAndDisabled();
816                 if (!disabled && !HtmlElement.TAB_INDEX_OUT_OF_BOUNDS.equals(element.getTabIndex())) {
817                     tabbableElements.add(element);
818                 }
819             }
820         }
821         tabbableElements.sort(createTabOrderComparator());
822         return Collections.unmodifiableList(tabbableElements);
823     }
824 
825     private static Comparator<HtmlElement> createTabOrderComparator() {
826         return (element1, element2) -> {
827             final Short i1 = element1.getTabIndex();
828             final Short i2 = element2.getTabIndex();
829 
830             final short index1;
831             if (i1 == null) {
832                 index1 = -1;
833             }
834             else {
835                 index1 = i1.shortValue();
836             }
837 
838             final short index2;
839             if (i2 == null) {
840                 index2 = -1;
841             }
842             else {
843                 index2 = i2.shortValue();
844             }
845 
846             final int result;
847             if (index1 > 0 && index2 > 0) {
848                 result = index1 - index2;
849             }
850             else if (index1 > 0) {
851                 result = -1;
852             }
853             else if (index2 > 0) {
854                 result = 1;
855             }
856             else if (index1 == index2) {
857                 result = 0;
858             }
859             else {
860                 result = index2 - index1;
861             }
862 
863             return result;
864         };
865     }
866 
867     /**
868      * Returns the HTML element that is assigned to the specified access key. An
869      * access key (aka mnemonic key) is used for keyboard navigation of the
870      * page.
871      * <p>
872      * Only the following HTML elements may have <code>accesskey</code>s defined: A, AREA,
873      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
874      * </p>
875      *
876      * @param accessKey the key to look for
877      * @return the HTML element that is assigned to the specified key or null
878      *      if no elements can be found that match the specified key.
879      */
880     public HtmlElement getHtmlElementByAccessKey(final char accessKey) {
881         final List<HtmlElement> elements = getHtmlElementsByAccessKey(accessKey);
882         if (elements.isEmpty()) {
883             return null;
884         }
885         return elements.get(0);
886     }
887 
888     /**
889      * Returns all the HTML elements that are assigned to the specified access key. An
890      * access key (aka mnemonic key) is used for keyboard navigation of the
891      * page.
892      * <p>
893      * The HTML specification seems to indicate that one accesskey cannot be used
894      * for multiple elements however Internet Explorer does seem to support this.
895      * It's worth noting that Firefox does not support multiple elements with one
896      * access key so you are making your HTML browser specific if you rely on this
897      * feature.
898      * </p>
899      *
900      * <p>
901      * Only the following HTML elements may have <code>accesskey</code>s defined: A, AREA,
902      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
903      * </p>
904      *
905      * @param accessKey the key to look for
906      * @return the elements that are assigned to the specified accesskey
907      */
908     public List<HtmlElement> getHtmlElementsByAccessKey(final char accessKey) {
909         final List<HtmlElement> elements = new ArrayList<>();
910 
911         final String searchString = Character.toString(accessKey).toLowerCase(Locale.ROOT);
912         for (final HtmlElement element : getHtmlElementDescendants()) {
913             if (ACCEPTABLE_TAG_NAMES.contains(element.getTagName())) {
914                 final String accessKeyAttribute = element.getAttributeDirect("accesskey");
915                 if (searchString.equalsIgnoreCase(accessKeyAttribute)) {
916                     elements.add(element);
917                 }
918             }
919         }
920 
921         return elements;
922     }
923 
924     /**
925      * <p>Executes the specified JavaScript code within the page. The usage would be similar to what can
926      * be achieved to execute JavaScript in the current page by entering "javascript:...some JS code..."
927      * in the URL field of a native browser.</p>
928      * <p><b>Note:</b> the provided code won't be executed if JavaScript has been disabled on the WebClient
929      * (see {@link org.htmlunit.WebClient#isJavaScriptEnabled()}).</p>
930      * @param sourceCode the JavaScript code to execute
931      * @return a ScriptResult which will contain both the current page (which may be different from
932      *         the previous page) and a JavaScript result object
933      */
934     public ScriptResult executeJavaScript(final String sourceCode) {
935         return executeJavaScript(sourceCode, "injected script", 1);
936     }
937 
938     /**
939      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
940      * <p>
941      * Execute the specified JavaScript if a JavaScript engine was successfully
942      * instantiated. If this JavaScript causes the current page to be reloaded
943      * (through location="" or form.submit()) then return the new page, otherwise
944      * return the current page.
945      * </p>
946      * <p><b>Please note:</b> Although this method is public, it is not intended for
947      * general execution of JavaScript. Users of HtmlUnit should interact with the pages
948      * as a user would by clicking on buttons or links and having the JavaScript event
949      * handlers execute as needed.
950      * </p>
951      *
952      * @param sourceCode the JavaScript code to execute
953      * @param sourceName the name for this chunk of code (will be displayed in error messages)
954      * @param startLine the line at which the script source starts
955      * @return a ScriptResult which will contain both the current page (which may be different from
956      *         the previous page) and a JavaScript result object.
957      */
958     public ScriptResult executeJavaScript(String sourceCode, final String sourceName, final int startLine) {
959         if (!getWebClient().isJavaScriptEnabled()) {
960             return new ScriptResult(JavaScriptEngine.UNDEFINED);
961         }
962 
963         if (org.htmlunit.util.StringUtils.startsWithIgnoreCase(sourceCode,
964                                                 JavaScriptURLConnection.JAVASCRIPT_PREFIX)) {
965             sourceCode = sourceCode.substring(JavaScriptURLConnection.JAVASCRIPT_PREFIX.length()).trim();
966             if (sourceCode.startsWith("return ")) {
967                 sourceCode = sourceCode.substring("return ".length());
968             }
969         }
970 
971         final Window window = getEnclosingWindow().getScriptableObject();
972         final VarScope scope = ScriptableObject.getTopLevelScope(window.getParentScope());
973 
974         final Object result = getWebClient().getJavaScriptEngine()
975                 .execute(this, scope, sourceCode, sourceName, startLine);
976         return new ScriptResult(result);
977     }
978 
979     /** Various possible external JavaScript file loading results. */
980     enum JavaScriptLoadResult {
981         /** The load was aborted and nothing was done. */
982         NOOP,
983         /** The load was aborted and nothing was done. */
984         NO_CONTENT,
985         /** The external JavaScript file was downloaded and compiled successfully. */
986         SUCCESS,
987         /** The external JavaScript file was not downloaded successfully. */
988         DOWNLOAD_ERROR,
989         /** The external JavaScript file was downloaded but was not compiled successfully. */
990         COMPILATION_ERROR
991     }
992 
993     /**
994      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
995      *
996      * @param srcAttribute the source attribute from the script tag
997      * @param scriptCharset the charset from the script tag
998      * @return the result of loading the specified external JavaScript file
999      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
1000      *         failure and the {@link WebClient} was configured to throw exceptions on failing
1001      *         HTTP status codes
1002      */
1003     JavaScriptLoadResult loadExternalJavaScriptFile(final String srcAttribute,
1004                             final Charset scriptCharset, final boolean crossorigin)
1005         throws FailingHttpStatusCodeException {
1006 
1007         final WebClient client = getWebClient();
1008         if (org.htmlunit.util.StringUtils.isBlank(srcAttribute) || !client.isJavaScriptEnabled()) {
1009             return JavaScriptLoadResult.NOOP;
1010         }
1011 
1012         final URL scriptURL;
1013         try {
1014             scriptURL = getFullyQualifiedUrl(srcAttribute);
1015             final String protocol = scriptURL.getProtocol();
1016             if ("javascript".equals(protocol)) {
1017                 if (LOG.isInfoEnabled()) {
1018                     LOG.info("Ignoring script src [" + srcAttribute + "]");
1019                 }
1020                 return JavaScriptLoadResult.NOOP;
1021             }
1022             if (!"http".equals(protocol) && !"https".equals(protocol)
1023                     && !"data".equals(protocol) && !"file".equals(protocol)) {
1024                 client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute,
1025                         new MalformedURLException("unknown protocol: '" + protocol + "'"));
1026                 return JavaScriptLoadResult.NOOP;
1027             }
1028         }
1029         catch (final MalformedURLException e) {
1030             client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute, e);
1031             return JavaScriptLoadResult.NOOP;
1032         }
1033 
1034         final Object script;
1035         try {
1036             script = loadJavaScriptFromUrl(scriptURL, scriptCharset, crossorigin);
1037         }
1038         catch (final IOException e) {
1039             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
1040             return JavaScriptLoadResult.DOWNLOAD_ERROR;
1041         }
1042         catch (final FailingHttpStatusCodeException e) {
1043             if (e.getStatusCode() == HttpStatus.NO_CONTENT_204) {
1044                 return JavaScriptLoadResult.NO_CONTENT;
1045             }
1046             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
1047             throw e;
1048         }
1049 
1050         if (script == null) {
1051             return JavaScriptLoadResult.COMPILATION_ERROR;
1052         }
1053 
1054         final Window window = getEnclosingWindow().getScriptableObject();
1055         final VarScope scope = ScriptableObject.getTopLevelScope(window.getParentScope());
1056 
1057         @SuppressWarnings("unchecked")
1058         final AbstractJavaScriptEngine<Object> engine = (AbstractJavaScriptEngine<Object>) client.getJavaScriptEngine();
1059         engine.execute(this, scope, script);
1060         return JavaScriptLoadResult.SUCCESS;
1061     }
1062 
1063     /**
1064      * Loads JavaScript from the specified URL. This method may return {@code null} if
1065      * there is a problem loading the code from the specified URL.
1066      *
1067      * @param url the URL of the script
1068      * @param scriptCharset the charset from the script tag
1069      * @return the content of the file, or {@code null} if we ran into a compile error
1070      * @throws IOException if there is a problem downloading the JavaScript file
1071      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
1072      *         failure and the {@link WebClient} was configured to throw exceptions on failing
1073      *         HTTP status codes
1074      */
1075     private Object loadJavaScriptFromUrl(final URL url, final Charset scriptCharset,
1076                     final boolean crossorigin) throws IOException,
1077         FailingHttpStatusCodeException {
1078 
1079         final WebRequest referringRequest = getWebResponse().getWebRequest();
1080 
1081         final WebClient client = getWebClient();
1082         final WebRequest request = new WebRequest(url);
1083         // copy all headers from the referring request
1084         request.setAdditionalHeaders(new HashMap<>(referringRequest.getAdditionalHeaders()));
1085 
1086         // at least overwrite this headers
1087         request.setAdditionalHeader(HttpHeader.ACCEPT, client.getBrowserVersion().getScriptAcceptHeader());
1088 
1089         request.setFetchDestination(WebRequest.FetchDestination.SCRIPT);
1090         request.setRequestingUrl(referringRequest.getUrl());
1091         request.setFetchModeOverride(WebRequest.FetchMode.NO_CORS);
1092 
1093         request.setRefererHeader(referringRequest.getUrl());
1094         request.setCharset(scriptCharset);
1095 
1096         // use info from script tag or fall back to utf-8
1097         // https://www.rfc-editor.org/rfc/rfc9239#section-4.2
1098         if (scriptCharset != null) {
1099             request.setDefaultResponseContentCharset(scriptCharset);
1100         }
1101         else {
1102             request.setDefaultResponseContentCharset(StandardCharsets.UTF_8);
1103         }
1104 
1105         if (crossorigin) {
1106             request.setFetchModeOverride(WebRequest.FetchMode.CORS);
1107 
1108             if (client.getBrowserVersion().hasFeature(HTTP_HEADER_CH_UA)) {
1109                 request.setAdditionalHeader(HttpHeader.ORIGIN,
1110                         UrlUtils.getUrlWithProtocolAndAuthority(url).toExternalForm());
1111             }
1112         }
1113 
1114         // our cache is a bit strange;
1115         // loadWebResponse check the cache for the web response
1116         // AND also fixes the request url for the following cache lookups
1117         final WebResponse response = client.loadWebResponse(request);
1118 
1119         // now we can look into the cache with the fixed request for
1120         // a cached script
1121         final Cache cache = client.getCache();
1122         final Object cachedScript = cache.getCachedObject(request);
1123         if (cachedScript instanceof Script) {
1124             return cachedScript;
1125         }
1126 
1127         client.printContentIfNecessary(response);
1128         client.throwFailingHttpStatusCodeExceptionIfNecessary(response);
1129 
1130         final int statusCode = response.getStatusCode();
1131         if (statusCode == HttpStatus.NO_CONTENT_204) {
1132             throw new FailingHttpStatusCodeException(response);
1133         }
1134 
1135         if (!response.isSuccess()) {
1136             throw new IOException("Unable to download JavaScript from '" + url + "' (status " + statusCode + ").");
1137         }
1138 
1139         final String contentType = response.getContentType();
1140         if (contentType != null) {
1141             if (MimeType.isObsoleteJavascriptMimeType(contentType)) {
1142                 getWebClient().getIncorrectnessListener().notify(
1143                         "Obsolete content type encountered: '" + contentType + "' "
1144                                 + "for remotely loaded JavaScript element at '" + url + "'.", this);
1145             }
1146             else if (!MimeType.isJavascriptMimeType(contentType)) {
1147                 getWebClient().getIncorrectnessListener().notify(
1148                         "Expect content type of '" + MimeType.TEXT_JAVASCRIPT + "' "
1149                                 + "for remotely loaded JavaScript element at '" + url + "', "
1150                                 + "but got '" + contentType + "'.", this);
1151             }
1152         }
1153 
1154         final Charset scriptEncoding = response.getContentCharset();
1155         final String scriptCode = response.getContentAsString(scriptEncoding);
1156         if (null != scriptCode) {
1157             final AbstractJavaScriptEngine<?> javaScriptEngine = client.getJavaScriptEngine();
1158 
1159             final Window window = getEnclosingWindow().getScriptableObject();
1160             final VarScope scope = ScriptableObject.getTopLevelScope(window.getParentScope());
1161 
1162             final Object script = javaScriptEngine.compile(this, scope, scriptCode, url.toExternalForm(), 1);
1163             if (script != null && cache.cacheIfPossible(request, response, script)) {
1164                 // no cleanup if the response is stored inside the cache
1165                 return script;
1166             }
1167 
1168             response.cleanUp();
1169             return script;
1170         }
1171 
1172         response.cleanUp();
1173         return null;
1174     }
1175 
1176     /**
1177      * Returns the title like {@code document.title} does: the text of the first title element
1178      * with ASCII whitespace stripped and collapsed. Other spaces, including the non-breaking space, are kept.
1179      *
1180      * @return the title like {@code document.title}
1181      */
1182     public String getDocumentTitle() {
1183         final HtmlTitle title = getTitleElement();
1184         if (title == null) {
1185             return "";
1186         }
1187 
1188         final String titleTextContent = title.getText();
1189         final StringBuilder sb = new StringBuilder(titleTextContent.length());
1190         boolean pendingSpace = false;
1191         for (int i = 0; i < titleTextContent.length(); i++) {
1192             final char c = titleTextContent.charAt(i);
1193             if (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r') {
1194                 pendingSpace = sb.length() > 0;
1195             }
1196             else {
1197                 if (pendingSpace) {
1198                     sb.append(' ');
1199                     pendingSpace = false;
1200                 }
1201                 sb.append(c);
1202             }
1203         }
1204         return sb.toString();
1205     }
1206 
1207     /**
1208      * Returns the normalized title of this page, or an empty string if no title element is present.
1209      * <p>
1210      * The returned text is normalized by collapsing consecutive whitespace characters
1211      * (including newlines and tabs) into a single space and trimming leading and trailing whitespace.
1212      * </p>
1213      *
1214      * @return the normalized page title, or an empty string if the title was not specified
1215      */
1216     public String getTitleText() {
1217         final HtmlTitle titleElement = getTitleElement();
1218         if (titleElement != null) {
1219             return titleElement.asNormalizedText();
1220         }
1221         return "";
1222     }
1223 
1224     /**
1225      * Sets the text for the title of this page. If there is not a title element
1226      * on this page, then one has to be generated.
1227      * @param message the new text
1228      */
1229     public void setTitleText(final String message) {
1230         HtmlTitle titleElement = getTitleElement();
1231         if (titleElement == null) {
1232             LOG.debug("No title element, creating one");
1233             final HtmlHead head = (HtmlHead) getFirstChildElement(getDocumentElement(), HtmlHead.class);
1234             if (head == null) {
1235                 // perhaps should we create head too?
1236                 throw new IllegalStateException("Headelement was not defined for this page");
1237             }
1238             final Map<String, DomAttr> emptyMap = Collections.emptyMap();
1239             titleElement = new HtmlTitle(HtmlTitle.TAG_NAME, this, emptyMap);
1240             if (head.getFirstChild() != null) {
1241                 head.getFirstChild().insertBefore(titleElement);
1242             }
1243             else {
1244                 head.appendChild(titleElement);
1245             }
1246         }
1247 
1248         titleElement.setText(message);
1249     }
1250 
1251     /**
1252      * Gets the first child of startElement that is an instance of the given class.
1253      * @param startElement the parent element
1254      * @param clazz the class to search for
1255      * @return {@code null} if no child found
1256      */
1257     private static DomElement getFirstChildElement(final DomElement startElement, final Class<?> clazz) {
1258         if (startElement == null) {
1259             return null;
1260         }
1261         for (final DomElement element : startElement.getChildElements()) {
1262             if (clazz.isInstance(element)) {
1263                 return element;
1264             }
1265         }
1266 
1267         return null;
1268     }
1269 
1270     /**
1271      * Gets the first child of startElement or it's children that is an instance of the given class.
1272      * @param startElement the parent element
1273      * @param clazz the class to search for
1274      * @return {@code null} if no child found
1275      */
1276     private DomElement getFirstChildElementRecursive(final DomElement startElement, final Class<?> clazz) {
1277         if (startElement == null) {
1278             return null;
1279         }
1280         for (final DomElement element : startElement.getChildElements()) {
1281             if (clazz.isInstance(element)) {
1282                 return element;
1283             }
1284             final DomElement childFound = getFirstChildElementRecursive(element, clazz);
1285             if (childFound != null) {
1286                 return childFound;
1287             }
1288         }
1289 
1290         return null;
1291     }
1292 
1293     /**
1294      * Gets the title element for this page. Returns null if one is not found.
1295      *
1296      * @return the title element for this page or null if this is not one
1297      */
1298     private HtmlTitle getTitleElement() {
1299         return (HtmlTitle) getFirstChildElementRecursive(getDocumentElement(), HtmlTitle.class);
1300     }
1301 
1302     /**
1303      * Looks for and executes any appropriate event handlers. Looks for body and frame tags.
1304      * @param eventType either {@link Event#TYPE_LOAD}, {@link Event#TYPE_UNLOAD}, or {@link Event#TYPE_BEFORE_UNLOAD}
1305      * @return {@code true} if user accepted <code>onbeforeunload</code> (not relevant to other events)
1306      */
1307     private boolean executeEventHandlersIfNeeded(final String eventType) {
1308         // If JavaScript isn't enabled, there's nothing for us to do.
1309         if (!getWebClient().isJavaScriptEnabled()) {
1310             return true;
1311         }
1312 
1313         // Execute the specified event on the document element.
1314         final WebWindow window = getEnclosingWindow();
1315         if (window.getScriptableObject() instanceof Window) {
1316             final Event event;
1317             if (Event.TYPE_BEFORE_UNLOAD.equals(eventType)) {
1318                 event = new BeforeUnloadEvent(this, eventType);
1319             }
1320             else {
1321                 event = new Event(this, eventType);
1322             }
1323 
1324             // This is the same as DomElement.fireEvent() and was copied
1325             // here so it could be used with HtmlPage.
1326             if (LOG.isDebugEnabled()) {
1327                 LOG.debug("Firing " + event);
1328             }
1329 
1330             final EventTarget jsNode;
1331             if (Event.TYPE_DOM_DOCUMENT_LOADED.equals(eventType)) {
1332                 jsNode = getScriptableObject();
1333             }
1334             else if (Event.TYPE_READY_STATE_CHANGE.equals(eventType)) {
1335                 jsNode = getDocumentElement().getScriptableObject();
1336             }
1337             else {
1338                 // The load/beforeunload/unload events target Document but paths Window only (tested in Chrome/FF)
1339                 jsNode = window.getScriptableObject();
1340             }
1341 
1342             ((JavaScriptEngine) getWebClient().getJavaScriptEngine()).callSecured(cx -> jsNode.fireEvent(event), this);
1343 
1344             if (!isOnbeforeunloadAccepted(this, event)) {
1345                 return false;
1346             }
1347         }
1348 
1349         // If this page was loaded in a frame, execute the version of the event specified on the frame tag.
1350         if (window instanceof FrameWindow fw) {
1351             final BaseFrameElement frame = fw.getFrameElement();
1352 
1353             // if part of a document fragment, then the load event is not triggered
1354             if (Event.TYPE_LOAD.equals(eventType) && frame.getParentNode() instanceof DomDocumentFragment) {
1355                 return true;
1356             }
1357 
1358             if (frame.hasEventHandlers("on" + eventType)) {
1359                 if (LOG.isDebugEnabled()) {
1360                     LOG.debug("Executing on" + eventType + " handler for " + frame);
1361                 }
1362                 if (window.getScriptableObject() instanceof Window) {
1363                     final Event event;
1364                     if (Event.TYPE_BEFORE_UNLOAD.equals(eventType)) {
1365                         event = new BeforeUnloadEvent(frame, eventType);
1366                     }
1367                     else {
1368                         event = new Event(frame, eventType);
1369                     }
1370                     // This fires the "load" event for the <frame> element which, like all non-window
1371                     // load events, propagates up to Document but not Window.  The "load" event for
1372                     // <frameset> on the other hand, like that of <body>, is handled above where it is
1373                     // fired against Document and directed to Window.
1374                     frame.fireEvent(event);
1375 
1376                     if (!isOnbeforeunloadAccepted((HtmlPage) frame.getPage(), event)) {
1377                         return false;
1378                     }
1379                 }
1380             }
1381         }
1382 
1383         return true;
1384     }
1385 
1386     /**
1387      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1388      *
1389      * @return true if the OnbeforeunloadHandler has accepted to change the page
1390      */
1391     public boolean isOnbeforeunloadAccepted() {
1392         return executeEventHandlersIfNeeded(Event.TYPE_BEFORE_UNLOAD);
1393     }
1394 
1395     private boolean isOnbeforeunloadAccepted(final HtmlPage page, final Event event) {
1396         if (event instanceof BeforeUnloadEvent beforeUnloadEvent) {
1397             if (beforeUnloadEvent.isBeforeUnloadMessageSet()) {
1398                 final OnbeforeunloadHandler handler = getWebClient().getOnbeforeunloadHandler();
1399                 if (handler == null) {
1400                     LOG.warn("document.onbeforeunload() returned a string in event.returnValue,"
1401                             + " but no onbeforeunload handler installed.");
1402                 }
1403                 else {
1404                     final String message = JavaScriptEngine.toString(beforeUnloadEvent.getReturnValue());
1405                     return handler.handleEvent(page, message);
1406                 }
1407             }
1408         }
1409         return true;
1410     }
1411 
1412     /**
1413      * If a refresh has been specified either through a meta tag or an HTTP
1414      * response header, then perform that refresh.
1415      * @throws IOException if an IO problem occurs
1416      */
1417     private void executeRefreshIfNeeded() throws IOException {
1418         // If this page is not in a frame then a refresh has already happened,
1419         // most likely through the JavaScript onload handler, so we don't do a
1420         // second refresh.
1421         final WebWindow window = getEnclosingWindow();
1422         if (window == null) {
1423             return;
1424         }
1425 
1426         final String refreshString = getRefreshStringOrNull();
1427         if (refreshString == null || refreshString.isEmpty()) {
1428             return;
1429         }
1430 
1431         final double time;
1432         final URL url;
1433 
1434         final int index = StringUtils.indexOfAnyBut(refreshString, "0123456789.");
1435 
1436         if (index == -1) {
1437             // Format: <meta http-equiv='refresh' content='10'>
1438             try {
1439                 time = Double.parseDouble(refreshString);
1440             }
1441             catch (final NumberFormatException e) {
1442                 if (LOG.isErrorEnabled()) {
1443                     LOG.error("Malformed refresh string (no ';' but not a number): " + refreshString, e);
1444                 }
1445                 return;
1446             }
1447             url = getUrl();
1448         }
1449         else {
1450             // Format: <meta http-equiv='refresh' content='10;url=http://www.blah.com'>
1451             try {
1452                 time = Double.parseDouble(refreshString.substring(0, index));
1453             }
1454             catch (final NumberFormatException e) {
1455                 if (LOG.isErrorEnabled()) {
1456                     LOG.error("Malformed refresh string (no valid number before ';') " + refreshString, e);
1457                 }
1458                 return;
1459             }
1460 
1461             String urlPart = refreshString.substring(index);
1462             final char separator = urlPart.charAt(0);
1463             if (";, \r\n\t".indexOf(separator) >= 0) {
1464                 urlPart = StringUtils.stripStart(urlPart, ";, \r\n\t");
1465                 urlPart = urlPart.toLowerCase(Locale.ROOT);
1466                 if (urlPart.startsWith("url")) {
1467                     urlPart = urlPart.substring(3);
1468                     urlPart = urlPart.trim();
1469 
1470                     if (urlPart.length() > 0 && urlPart.charAt(0) == '=') {
1471                         urlPart = urlPart.substring(1);
1472                         urlPart = urlPart.trim();
1473                     }
1474                 }
1475 
1476                 if (org.htmlunit.util.StringUtils.isBlank(urlPart)) {
1477                     //content='10; URL=' is treated as content='10'
1478                     url = getUrl();
1479                 }
1480                 else {
1481                     if (urlPart.charAt(0) == '"' || urlPart.charAt(0) == 0x27) {
1482                         urlPart = urlPart.substring(1);
1483                     }
1484                     if (urlPart.charAt(urlPart.length() - 1) == '"' || urlPart.charAt(urlPart.length() - 1) == 0x27) {
1485                         urlPart = urlPart.substring(0, urlPart.length() - 1);
1486                     }
1487                     try {
1488                         url = getFullyQualifiedUrl(urlPart);
1489                     }
1490                     catch (final MalformedURLException e) {
1491                         if (LOG.isErrorEnabled()) {
1492                             LOG.error("Malformed URL in refresh string: " + refreshString, e);
1493                         }
1494                         return;
1495                     }
1496                 }
1497             }
1498             else {
1499                 if (LOG.isErrorEnabled()) {
1500                     LOG.error("Malformed refresh string (separator after time missing): " + refreshString);
1501                 }
1502                 return;
1503             }
1504         }
1505 
1506         processRefresh(url, time);
1507     }
1508 
1509     // this is different from what is done in org.htmlunit.WebClient.loadWebResponseFromWebConnection(WebRequest, int)
1510     // because there we are directly replacing the response before loading the response into the window
1511     // here we are replacing the page in the window (maybe after some time)
1512     private void processRefresh(final URL url, final double time) throws IOException {
1513         final WebClient webClient = getWebClient();
1514 
1515         final int refreshLimit = webClient.getOptions().getPageRefreshLimit();
1516         if (refreshLimit == 0) {
1517             final WebResponse webResponse = getWebResponse();
1518             throw new FailingHttpStatusCodeException("Too many redirects for "
1519                     + webResponse.getWebRequest().getUrl(), webResponse);
1520         }
1521 
1522         if (refreshLimit >= 0) {
1523             final StackTraceElement[] elements = new Exception().getStackTrace();
1524             int count = 0;
1525             final int elementCountLimit = refreshLimit > 50 ? 400 : refreshLimit > 10 ? 80 : 5;
1526             final int elementCount = elements.length;
1527 
1528             if (elementCount > elementCountLimit) {
1529                 for (int i = 0; i < elementCount; i++) {
1530                     if ("processRefresh".equals(elements[i].getMethodName())
1531                             && "org.htmlunit.html.HtmlPage".equals(elements[i].getClassName())) {
1532                         count++;
1533                         if (count >= refreshLimit) {
1534                             final WebResponse webResponse = getWebResponse();
1535                             throw new FailingHttpStatusCodeException(
1536                                             "Too many redirects (>= " + count + ") for "
1537                                                 + webResponse.getWebRequest().getUrl(), webResponse);
1538                         }
1539                     }
1540                 }
1541             }
1542         }
1543 
1544         webClient.getRefreshHandler().handleRefresh(this, url, (int) time);
1545     }
1546 
1547     /**
1548      * Returns an auto-refresh string if specified. This will look in both the meta
1549      * tags and inside the HTTP response headers.
1550      * @return the auto-refresh string
1551      */
1552     private String getRefreshStringOrNull() {
1553         final List<HtmlMeta> metaTags = getMetaTags("refresh");
1554         if (!metaTags.isEmpty()) {
1555             return metaTags.get(0).getContentAttribute().trim();
1556         }
1557         return getWebResponse().getResponseHeaderValue("Refresh");
1558     }
1559 
1560     private void processPostponedActionsIfNeeded() {
1561         if (!getWebClient().isJavaScriptEnabled()) {
1562             return;
1563         }
1564         getWebClient().getJavaScriptEngine().processPostponedActions();
1565     }
1566 
1567     /**
1568      * Executes any deferred scripts, if necessary.
1569      */
1570     private void executeDeferredScriptsIfNeeded() {
1571         if (!getWebClient().isJavaScriptEnabled()) {
1572             return;
1573         }
1574         final DomElement doc = getDocumentElement();
1575         final List<HtmlScript> scripts = new ArrayList<>();
1576 
1577         // don't call getElementsByTagName() here because it creates a live collection
1578         for (final HtmlElement elem : doc.getHtmlElementDescendants()) {
1579             if ("script".equals(elem.getLocalName()) && (elem instanceof HtmlScript script)) {
1580                 if (script.isDeferred() && ATTRIBUTE_NOT_DEFINED != script.getSrcAttribute()) {
1581                     scripts.add(script);
1582                 }
1583             }
1584         }
1585         for (final HtmlScript script : scripts) {
1586             ScriptElementSupport.executeScriptIfNeeded(script, true, true);
1587         }
1588     }
1589 
1590     /**
1591      * Deregister frames that are no longer in use.
1592      */
1593     public void deregisterFramesIfNeeded() {
1594         final List<BaseFrameElement> frameElementsCopy = new ArrayList<>(frameElements_);
1595         for (final BaseFrameElement frameElement : frameElementsCopy) {
1596             final WebWindow window = frameElement.getEnclosedWindow();
1597             getWebClient().deregisterWebWindow(window);
1598             final Page page = window.getEnclosedPage();
1599             if (page != null && page.isHtmlPage()) {
1600                 // seems quite silly, but for instance if the src attribute of an iframe is not
1601                 // set, the error only occurs when leaving the page
1602                 ((HtmlPage) page).deregisterFramesIfNeeded();
1603             }
1604         }
1605     }
1606 
1607     /**
1608      * Returns a list containing all the frames (from frame and iframe tags) in this page
1609      * in document order.
1610      * @return a list of {@link FrameWindow}
1611      */
1612     public List<FrameWindow> getFrames() {
1613         final List<BaseFrameElement> frameElements = new ArrayList<>(frameElements_);
1614         frameElements.sort(DOCUMENT_POSITION_COMPERATOR);
1615 
1616         final List<FrameWindow> list = new ArrayList<>(frameElements.size());
1617         for (final BaseFrameElement frameElement : frameElements) {
1618             list.add(frameElement.getEnclosedWindow());
1619         }
1620         return list;
1621     }
1622 
1623     /**
1624      * Returns the first frame contained in this page with the specified name.
1625      * @param name the name to search for
1626      * @return the first frame found
1627      * @throws ElementNotFoundException If no frame exist in this page with the specified name.
1628      */
1629     public FrameWindow getFrameByName(final String name) throws ElementNotFoundException {
1630         for (final BaseFrameElement frameElement : frameElements_) {
1631             final FrameWindow fw = frameElement.getEnclosedWindow();
1632             if (fw.getName().equals(name)) {
1633                 return fw;
1634             }
1635         }
1636 
1637         throw new ElementNotFoundException("frame or iframe", DomElement.NAME_ATTRIBUTE, name);
1638     }
1639 
1640     /**
1641      * Simulate pressing an access key. This may change the focus, may click buttons and may invoke
1642      * JavaScript.
1643      *
1644      * @param accessKey the key that will be pressed
1645      * @return the element that has the focus after pressing this access key or null if no element
1646      *         has the focus.
1647      * @throws IOException if an IO error occurs during the processing of this access key (this
1648      *         would only happen if the access key triggered a button which in turn caused a page load)
1649      */
1650     public DomElement pressAccessKey(final char accessKey) throws IOException {
1651         final HtmlElement element = getHtmlElementByAccessKey(accessKey);
1652         if (element != null) {
1653             element.focus();
1654             if (element instanceof HtmlAnchor
1655                     || element instanceof HtmlArea
1656                     || element instanceof HtmlButton
1657                     || element instanceof HtmlInput
1658                     || element instanceof HtmlLabel
1659                     || element instanceof HtmlLegend
1660                     || element instanceof HtmlTextArea) {
1661                 final Page newPage = element.click();
1662 
1663                 if (newPage != this && getFocusedElement() == element) {
1664                     // The page was reloaded therefore no element on this page will have the focus.
1665                     getFocusedElement().blur();
1666                 }
1667             }
1668         }
1669 
1670         return getFocusedElement();
1671     }
1672 
1673     /**
1674      * Move the focus to the next element in the tab order. To determine the specified tab
1675      * order, refer to {@link HtmlPage#getTabbableElements()}
1676      *
1677      * @return the element that has focus after calling this method
1678      */
1679     public HtmlElement tabToNextElement() {
1680         final List<HtmlElement> elements = getTabbableElements();
1681         if (elements.isEmpty()) {
1682             setFocusedElement(null);
1683             return null;
1684         }
1685 
1686         final HtmlElement elementToGiveFocus;
1687         final DomElement elementWithFocus = getFocusedElement();
1688         if (elementWithFocus == null) {
1689             elementToGiveFocus = elements.get(0);
1690         }
1691         else {
1692             final int index = elements.indexOf(elementWithFocus);
1693             if (index == -1) {
1694                 // The element with focus isn't on this page
1695                 elementToGiveFocus = elements.get(0);
1696             }
1697             else if (index == elements.size() - 1) {
1698                 // if at last jump to start
1699                 elementToGiveFocus = elements.get(0);
1700             }
1701             else {
1702                 elementToGiveFocus = elements.get(index + 1);
1703             }
1704         }
1705 
1706         setFocusedElement(elementToGiveFocus);
1707         return elementToGiveFocus;
1708     }
1709 
1710     /**
1711      * Move the focus to the previous element in the tab order. To determine the specified tab
1712      * order, refer to {@link HtmlPage#getTabbableElements()}
1713      *
1714      * @return the element that has focus after calling this method
1715      */
1716     public HtmlElement tabToPreviousElement() {
1717         final List<HtmlElement> elements = getTabbableElements();
1718         if (elements.isEmpty()) {
1719             setFocusedElement(null);
1720             return null;
1721         }
1722 
1723         final HtmlElement elementToGiveFocus;
1724         final DomElement elementWithFocus = getFocusedElement();
1725         if (elementWithFocus == null) {
1726             elementToGiveFocus = elements.get(elements.size() - 1);
1727         }
1728         else {
1729             final int index = elements.indexOf(elementWithFocus);
1730             if (index == -1) {
1731                 // The element with focus isn't on this page
1732                 elementToGiveFocus = elements.get(elements.size() - 1);
1733             }
1734             else if (index == 0) {
1735                 // first; back to the last
1736                 elementToGiveFocus = elements.get(elements.size() - 1);
1737             }
1738             else {
1739                 elementToGiveFocus = elements.get(index - 1);
1740             }
1741         }
1742 
1743         setFocusedElement(elementToGiveFocus);
1744         return elementToGiveFocus;
1745     }
1746 
1747     /**
1748      * Returns the HTML element with the specified ID. If more than one element
1749      * has this ID (not allowed by the HTML spec), then this method returns the
1750      * first one.
1751      *
1752      * @param elementId the ID value to search for
1753      * @param <E> the element type
1754      * @return the HTML element with the specified ID
1755      * @throws ElementNotFoundException if no element was found matching the specified ID
1756      */
1757     @SuppressWarnings("unchecked")
1758     public <E extends HtmlElement> E getHtmlElementById(final String elementId) throws ElementNotFoundException {
1759         final DomElement element = getElementById(elementId);
1760         if (element == null) {
1761             throw new ElementNotFoundException("*", DomElement.ID_ATTRIBUTE, elementId);
1762         }
1763         return (E) element;
1764     }
1765 
1766     /**
1767      * Returns the elements with the specified ID. If there are no elements
1768      * with the specified ID, this method returns an empty list. Please note that
1769      * the lists returned by this method are immutable.
1770      *
1771      * @param elementId the ID value to search for
1772      * @return the elements with the specified name attribute
1773      */
1774     public List<DomElement> getElementsById(final String elementId) {
1775         if (elementId != null) {
1776             ensureMappedElementsBuilt();
1777             final MappedElementIndexEntry elements = idMap_.get(elementId);
1778             if (elements != null) {
1779                 return new ArrayList<>(elements.elements());
1780             }
1781         }
1782         return Collections.emptyList();
1783     }
1784 
1785     /**
1786      * Returns the element with the specified name. If more than one element
1787      * has this name, then this method returns the first one.
1788      *
1789      * @param name the name value to search for
1790      * @param <E> the element type
1791      * @return the element with the specified name
1792      * @throws ElementNotFoundException if no element was found matching the specified name
1793      */
1794     @SuppressWarnings("unchecked")
1795     public <E extends DomElement> E getElementByName(final String name) throws ElementNotFoundException {
1796         if (name != null) {
1797             ensureMappedElementsBuilt();
1798             final MappedElementIndexEntry elements = nameMap_.get(name);
1799             if (elements != null) {
1800                 return (E) elements.first();
1801             }
1802         }
1803         throw new ElementNotFoundException("*", DomElement.NAME_ATTRIBUTE, name);
1804     }
1805 
1806     /**
1807      * Returns the elements with the specified name attribute. If there are no elements
1808      * with the specified name, this method returns an empty list. Please note that
1809      * the lists returned by this method are immutable.
1810      *
1811      * @param name the name value to search for
1812      * @return the elements with the specified name attribute
1813      */
1814     public List<DomElement> getElementsByName(final String name) {
1815         if (name != null) {
1816             ensureMappedElementsBuilt();
1817             final MappedElementIndexEntry elements = nameMap_.get(name);
1818             if (elements != null) {
1819                 return new ArrayList<>(elements.elements());
1820             }
1821         }
1822         return Collections.emptyList();
1823     }
1824 
1825     /**
1826      * Returns the elements with the specified string for their name or ID. If there are
1827      * no elements with the specified name or ID, this method returns an empty list.
1828      *
1829      * @param idAndOrName the value to search for
1830      * @return the elements with the specified string for their name or ID
1831      */
1832     public List<DomElement> getElementsByIdAndOrName(final String idAndOrName) {
1833         if (idAndOrName == null) {
1834             return Collections.emptyList();
1835         }
1836         ensureMappedElementsBuilt();
1837         final MappedElementIndexEntry list1 = idMap_.get(idAndOrName);
1838         final MappedElementIndexEntry list2 = nameMap_.get(idAndOrName);
1839         final List<DomElement> list = new ArrayList<>();
1840         if (list1 != null) {
1841             list.addAll(list1.elements());
1842         }
1843         if (list2 != null) {
1844             for (final DomElement elt : list2.elements()) {
1845                 if (!list.contains(elt)) {
1846                     list.add(elt);
1847                 }
1848             }
1849         }
1850         return list;
1851     }
1852 
1853     /**
1854      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1855      *
1856      * @param node the node that has just been added to the document
1857      */
1858     void notifyNodeAdded(final DomNode node) {
1859         if (node instanceof DomElement element1) {
1860             addMappedElement(element1, true);
1861 
1862             if (node instanceof BaseFrameElement element) {
1863                 frameElements_.add(element);
1864             }
1865 
1866             if (node.getFirstChild() != null) {
1867                 for (final Iterator<HtmlElement> iterator = node.new DescendantHtmlElementsIterator();
1868                         iterator.hasNext();) {
1869                     final HtmlElement child = iterator.next();
1870                     if (child instanceof BaseFrameElement element) {
1871                         frameElements_.add(element);
1872                     }
1873                 }
1874             }
1875 
1876             if ("base".equals(node.getNodeName())) {
1877                 calculateBase();
1878             }
1879         }
1880         node.onAddedToPage();
1881     }
1882 
1883     /**
1884      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1885      *
1886      * @param node the node that has just been removed from the tree
1887      */
1888     void notifyNodeRemoved(final DomNode node) {
1889         if (node instanceof HtmlElement element) {
1890             removeMappedElement(element, true, true);
1891 
1892             if (node instanceof BaseFrameElement) {
1893                 frameElements_.remove(node);
1894             }
1895             for (final HtmlElement child : node.getHtmlElementDescendants()) {
1896                 if (child instanceof BaseFrameElement) {
1897                     frameElements_.remove(child);
1898                 }
1899             }
1900 
1901             if ("base".equals(node.getNodeName())) {
1902                 calculateBase();
1903             }
1904         }
1905     }
1906 
1907     /**
1908      * Adds an element to the ID and name maps, if necessary.
1909      * @param element the element to be added to the ID and name maps
1910      * @param recurse indicates if children must be added too
1911      */
1912     void addMappedElement(final DomElement element, final boolean recurse) {
1913         // Index is built lazily; skip while not built. ensureMappedElementsBuilt()
1914         // walks the tree once and populates everything on first read.
1915         if (!mappedElementsBuilt_) {
1916             return;
1917         }
1918         if (isAncestorOf(element)) {
1919             addElement(element, recurse);
1920         }
1921     }
1922 
1923     private void ensureMappedElementsBuilt() {
1924         if (mappedElementsBuilt_) {
1925             return;
1926         }
1927 
1928         final DomElement root = getDocumentElement();
1929         if (root != null) {
1930             addElement(root, true);
1931         }
1932 
1933         // Flip the flag only after the maps are populated, so a partial
1934         // failure mid-walk leaves us with built_=false and the next read
1935         // tries again rather than seeing a half-populated index.
1936         mappedElementsBuilt_ = true;
1937     }
1938 
1939     private void addElement(final DomElement element, final boolean recurse) {
1940         final String idValue = element.getAttribute(DomElement.ID_ATTRIBUTE);
1941         if (ATTRIBUTE_NOT_DEFINED != idValue) {
1942             MappedElementIndexEntry elements = idMap_.get(idValue);
1943             if (elements == null) {
1944                 elements = new MappedElementIndexEntry();
1945                 elements.add(element);
1946                 idMap_.put(idValue, elements);
1947             }
1948             else {
1949                 elements.add(element);
1950             }
1951         }
1952 
1953         final String nameValue = element.getAttribute(DomElement.NAME_ATTRIBUTE);
1954         if (ATTRIBUTE_NOT_DEFINED != nameValue) {
1955             MappedElementIndexEntry elements = nameMap_.get(nameValue);
1956             if (elements == null) {
1957                 elements = new MappedElementIndexEntry();
1958                 elements.add(element);
1959                 nameMap_.put(nameValue, elements);
1960             }
1961             else {
1962                 elements.add(element);
1963             }
1964         }
1965 
1966         if (recurse) {
1967             // poor man's approach - we don't use getChildElements()
1968             // to avoid a bunch of object constructions
1969             DomNode nextChild = element.getFirstChild();
1970             while (nextChild != null) {
1971                 if (nextChild instanceof DomElement domElement) {
1972                     addElement(domElement, true);
1973                 }
1974                 nextChild = nextChild.getNextSibling();
1975             }
1976         }
1977     }
1978 
1979     /**
1980      * Removes an element and optionally its children from the ID and name maps, if necessary.
1981      * @param element the element to be removed from the ID and name maps
1982      * @param recurse indicates if children must be removed too
1983      * @param descendant indicates of the element was descendant of this HtmlPage, but now its parent might be null
1984      */
1985     void removeMappedElement(final DomElement element, final boolean recurse, final boolean descendant) {
1986         // see addMappedElement: while the index is unbuilt, removals are also no-ops.
1987         if (!mappedElementsBuilt_) {
1988             return;
1989         }
1990         if (descendant || isAncestorOf(element)) {
1991             removeElement(element, recurse);
1992         }
1993     }
1994 
1995     private void removeElement(final DomElement element, final boolean recurse) {
1996         final String idValue = element.getAttribute(DomElement.ID_ATTRIBUTE);
1997         if (ATTRIBUTE_NOT_DEFINED != idValue) {
1998             final MappedElementIndexEntry elements = idMap_.remove(idValue);
1999             if (elements != null) {
2000                 elements.remove(element);
2001                 if (!elements.elements_.isEmpty()) {
2002                     idMap_.put(idValue, elements);
2003                 }
2004             }
2005         }
2006 
2007         final String nameValue = element.getAttribute(DomElement.NAME_ATTRIBUTE);
2008         if (ATTRIBUTE_NOT_DEFINED != nameValue) {
2009             final MappedElementIndexEntry elements = nameMap_.remove(nameValue);
2010             if (elements != null) {
2011                 elements.remove(element);
2012                 if (!elements.elements_.isEmpty()) {
2013                     nameMap_.put(nameValue, elements);
2014                 }
2015             }
2016         }
2017 
2018         if (recurse) {
2019             for (final DomElement child : element.getChildElements()) {
2020                 removeElement(child, true);
2021             }
2022         }
2023     }
2024 
2025     /**
2026      * Indicates if the attribute name indicates that the owning element is mapped.
2027      * @param document the owning document
2028      * @param attributeName the name of the attribute to consider
2029      * @return {@code true} if the owning element should be mapped in its owning page
2030      */
2031     static boolean isMappedElement(final Document document, final String attributeName) {
2032         return document instanceof HtmlPage
2033             && (DomElement.NAME_ATTRIBUTE.equals(attributeName) || DomElement.ID_ATTRIBUTE.equals(attributeName));
2034     }
2035 
2036     private void calculateBase() {
2037         final List<HtmlElement> baseElements = getDocumentElement().getStaticElementsByTagName("base");
2038 
2039         base_ = null;
2040         for (final HtmlElement baseElement : baseElements) {
2041             if (baseElement instanceof HtmlBase base) {
2042                 if (base_ != null) {
2043                     notifyIncorrectness("Multiple 'base' detected, only the first is used.");
2044                     break;
2045                 }
2046                 base_ = base;
2047             }
2048         }
2049     }
2050 
2051     /**
2052      * Loads the content of the contained frames. This is done after the page is completely loaded, to allow script
2053      * contained in the frames to reference elements from the page located after the closing &lt;/frame&gt; tag.
2054      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
2055      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to {@code true}
2056      */
2057     void loadFrames() throws FailingHttpStatusCodeException {
2058         for (final BaseFrameElement frameElement : new ArrayList<>(frameElements_)) {
2059             // test if the frame should really be loaded:
2060             // if a script has already changed its content, it should be skipped
2061             // use == and not equals(...) to identify initial content (versus URL set to "about:blank")
2062             if (frameElement.getEnclosedWindow() != null
2063                     && UrlUtils.URL_ABOUT_BLANK == frameElement.getEnclosedPage().getUrl()
2064                     && !frameElement.isContentLoaded()) {
2065                 frameElement.loadInnerPage();
2066             }
2067         }
2068     }
2069 
2070     /**
2071      * Gives a basic representation for debugging purposes.
2072      * @return a basic representation
2073      */
2074     @Override
2075     public String toString() {
2076         final StringBuilder builder = new StringBuilder()
2077             .append("HtmlPage(")
2078             .append(getUrl())
2079             .append(")@")
2080             .append(hashCode());
2081         return builder.toString();
2082     }
2083 
2084     /**
2085      * Gets the meta tag for a given {@code http-equiv} value.
2086      * @param httpEquiv the {@code http-equiv} value
2087      * @return a list of {@link HtmlMeta}
2088      */
2089     protected List<HtmlMeta> getMetaTags(final String httpEquiv) {
2090         if (getDocumentElement() == null) {
2091             return Collections.emptyList(); // weird case, for instance if document.documentElement has been removed
2092         }
2093         final List<HtmlMeta> tags = getDocumentElement().getStaticElementsByTagName("meta");
2094         final List<HtmlMeta> foundTags = new ArrayList<>();
2095         for (final HtmlMeta htmlMeta : tags) {
2096             if (httpEquiv.equalsIgnoreCase(htmlMeta.getHttpEquivAttribute())) {
2097                 foundTags.add(htmlMeta);
2098             }
2099         }
2100         return foundTags;
2101     }
2102 
2103     /**
2104      * Creates a clone of this instance, and clears cached state to be not shared with the original.
2105      *
2106      * @return a clone of this instance
2107      */
2108     @Override
2109     protected HtmlPage clone() {
2110         final HtmlPage result = (HtmlPage) super.clone();
2111         result.elementWithFocus_ = null;
2112 
2113         result.idMap_ = new ConcurrentHashMap<>();
2114         result.nameMap_ = new ConcurrentHashMap<>();
2115         result.mappedElementsBuilt_ = false;
2116 
2117         return result;
2118     }
2119 
2120     /**
2121      * {@inheritDoc}
2122      */
2123     @Override
2124     public HtmlPage cloneNode(final boolean deep) {
2125         // we need the ScriptObject clone before cloning the kids.
2126         final HtmlPage result = (HtmlPage) super.cloneNode(false);
2127         if (getWebClient().isJavaScriptEnabled()) {
2128             final HtmlUnitScriptable jsObjClone = getScriptableObject().clone();
2129             jsObjClone.setDomNode(result);
2130         }
2131 
2132         // if deep, clone the kids too, and re initialize parts of the clone
2133         if (deep) {
2134             // this was previously synchronized but that makes not sense, why
2135             // lock the source against a copy only one has a reference too,
2136             // because result is a local reference
2137             result.attributeListeners_ = null;
2138 
2139             result.selectionRanges_ = new ArrayList<>(3);
2140             // the original one is synchronized so we should do that here too, shouldn't we?
2141             result.afterLoadActions_ = Collections.synchronizedList(new ArrayList<>());
2142             result.frameElements_ = new ArrayList<>();
2143             for (DomNode child = getFirstChild(); child != null; child = child.getNextSibling()) {
2144                 result.appendChild(child.cloneNode(true));
2145             }
2146         }
2147         return result;
2148     }
2149 
2150     /**
2151      * Adds an HtmlAttributeChangeListener to the listener list.
2152      * The listener is registered for all attributes of all HtmlElements contained in this page.
2153      *
2154      * @param listener the attribute change listener to be added
2155      * @see #removeHtmlAttributeChangeListener(HtmlAttributeChangeListener)
2156      */
2157     public void addHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
2158         WebAssert.notNull("listener", listener);
2159         synchronized (lock_) {
2160             if (attributeListeners_ == null) {
2161                 attributeListeners_ = new LinkedHashSet<>();
2162             }
2163             attributeListeners_.add(listener);
2164         }
2165     }
2166 
2167     /**
2168      * Removes an HtmlAttributeChangeListener from the listener list.
2169      * This method should be used to remove HtmlAttributeChangeListener that were registered
2170      * for all attributes of all HtmlElements contained in this page.
2171      *
2172      * @param listener the attribute change listener to be removed
2173      * @see #addHtmlAttributeChangeListener(HtmlAttributeChangeListener)
2174      */
2175     public void removeHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
2176         WebAssert.notNull("listener", listener);
2177         synchronized (lock_) {
2178             if (attributeListeners_ != null) {
2179                 attributeListeners_.remove(listener);
2180             }
2181         }
2182     }
2183 
2184     /**
2185      * Notifies all registered listeners for the given event to add an attribute.
2186      * @param event the event to fire
2187      */
2188     void fireHtmlAttributeAdded(final HtmlAttributeChangeEvent event) {
2189         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2190         if (listeners != null) {
2191             for (final HtmlAttributeChangeListener listener : listeners) {
2192                 listener.attributeAdded(event);
2193             }
2194         }
2195     }
2196 
2197     /**
2198      * Notifies all registered listeners for the given event to replace an attribute.
2199      * @param event the event to fire
2200      */
2201     void fireHtmlAttributeReplaced(final HtmlAttributeChangeEvent event) {
2202         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2203         if (listeners != null) {
2204             for (final HtmlAttributeChangeListener listener : listeners) {
2205                 listener.attributeReplaced(event);
2206             }
2207         }
2208     }
2209 
2210     /**
2211      * Notifies all registered listeners for the given event to remove an attribute.
2212      * @param event the event to fire
2213      */
2214     void fireHtmlAttributeRemoved(final HtmlAttributeChangeEvent event) {
2215         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2216         if (listeners != null) {
2217             for (final HtmlAttributeChangeListener listener : listeners) {
2218                 listener.attributeRemoved(event);
2219             }
2220         }
2221     }
2222 
2223     private List<HtmlAttributeChangeListener> safeGetAttributeListeners() {
2224         synchronized (lock_) {
2225             if (attributeListeners_ != null) {
2226                 return new ArrayList<>(attributeListeners_);
2227             }
2228             return null;
2229         }
2230     }
2231 
2232     /**
2233      * {@inheritDoc}
2234      */
2235     @Override
2236     protected void checkChildHierarchy(final org.w3c.dom.Node newChild) throws DOMException {
2237         if (newChild instanceof Element) {
2238             if (getDocumentElement() != null) {
2239                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2240                     "The Document may only have a single child Element.");
2241             }
2242         }
2243         else if (newChild instanceof DocumentType) {
2244             if (getDoctype() != null) {
2245                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2246                     "The Document may only have a single child DocumentType.");
2247             }
2248         }
2249         else if (!(newChild instanceof Comment || newChild instanceof ProcessingInstruction)) {
2250             throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2251                 "The Document may not have a child of this type: " + newChild.getNodeType());
2252         }
2253         super.checkChildHierarchy(newChild);
2254     }
2255 
2256     /**
2257      * Returns {@code true} if an HTML parser is operating on this page, adding content to it.
2258      * @return {@code true} if an HTML parser is operating on this page, adding content to it
2259      */
2260     public boolean isBeingParsed() {
2261         return parserCount_ > 0;
2262     }
2263 
2264     /**
2265      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2266      *
2267      * Called by the HTML parser to let the page know that it has started parsing some content for this page.
2268      */
2269     public void registerParsingStart() {
2270         parserCount_++;
2271     }
2272 
2273     /**
2274      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2275      *
2276      * Called by the HTML parser to let the page know that it has finished parsing some content for this page.
2277      */
2278     public void registerParsingEnd() {
2279         parserCount_--;
2280     }
2281 
2282     /**
2283      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2284      *
2285      * Returns {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2286      * to this page. Non-inline content is content that is parsed for the page, but not in the
2287      * same stream as the page itself -- basically anything other than <code>document.write()</code>
2288      * or <code>document.writeln()</code>: <code>innerHTML</code>, <code>outerHTML</code>,
2289      * <code>document.createElement()</code>, etc.
2290      *
2291      * @return {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2292      *         to this page
2293      */
2294     public boolean isParsingHtmlSnippet() {
2295         return snippetParserCount_ > 0;
2296     }
2297 
2298     /**
2299      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2300      *
2301      * Called by the HTML parser to let the page know that it has started parsing a non-inline HTML snippet.
2302      */
2303     public void registerSnippetParsingStart() {
2304         snippetParserCount_++;
2305     }
2306 
2307     /**
2308      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2309      *
2310      * Called by the HTML parser to let the page know that it has finished parsing a non-inline HTML snippet.
2311      */
2312     public void registerSnippetParsingEnd() {
2313         snippetParserCount_--;
2314     }
2315 
2316     /**
2317      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2318      *
2319      * Returns {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2320      * to this page. Inline content is content inserted into the parser stream dynamically
2321      * while the page is being parsed (i.e. <code>document.write()</code> or <code>document.writeln()</code>).
2322      *
2323      * @return {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2324      *         to this page
2325      */
2326     public boolean isParsingInlineHtmlSnippet() {
2327         return inlineSnippetParserCount_ > 0;
2328     }
2329 
2330     /**
2331      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2332      *
2333      * Called by the HTML parser to let the page know that it has started parsing an inline HTML snippet.
2334      */
2335     public void registerInlineSnippetParsingStart() {
2336         inlineSnippetParserCount_++;
2337     }
2338 
2339     /**
2340      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2341      *
2342      * Called by the HTML parser to let the page know that it has finished parsing an inline HTML snippet.
2343      */
2344     public void registerInlineSnippetParsingEnd() {
2345         inlineSnippetParserCount_--;
2346     }
2347 
2348     /**
2349      * Refreshes the page by sending the same parameters as previously sent to get this page.
2350      * @return the newly loaded page.
2351      * @throws IOException if an IO problem occurs
2352      */
2353     public Page refresh() throws IOException {
2354         return getWebClient().getPage(getWebResponse().getWebRequest());
2355     }
2356 
2357     /**
2358      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2359      * <p>
2360      * Parses the given string as would it belong to the content being parsed
2361      * at the current parsing position
2362      * </p>
2363      * @param string the HTML code to write in place
2364      */
2365     public void writeInParsedStream(final String string) {
2366         getDOMBuilder().pushInputString(string);
2367     }
2368 
2369     /**
2370      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2371      *
2372      * Sets the builder to allow page to send content from document.write(ln) calls.
2373      * @param htmlUnitDOMBuilder the builder
2374      */
2375     public void setDOMBuilder(final HTMLParserDOMBuilder htmlUnitDOMBuilder) {
2376         domBuilder_ = htmlUnitDOMBuilder;
2377     }
2378 
2379     /**
2380      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2381      *
2382      * Returns the current builder.
2383      * @return the current builder
2384      */
2385     public HTMLParserDOMBuilder getDOMBuilder() {
2386         return domBuilder_;
2387     }
2388 
2389     /**
2390      * <p>Returns all namespaces defined in the root element of this page.</p>
2391      * <p>The default namespace has a key of an empty string.</p>
2392      * @return all namespaces defined in the root element of this page
2393      */
2394     public Map<String, String> getNamespaces() {
2395         final org.w3c.dom.NamedNodeMap attributes = getDocumentElement().getAttributes();
2396         final Map<String, String> namespaces = new HashMap<>();
2397         for (int i = 0; i < attributes.getLength(); i++) {
2398             final Attr attr = (Attr) attributes.item(i);
2399             String name = attr.getName();
2400             if (name.startsWith("xmlns")) {
2401                 int startPos = 5;
2402                 if (name.length() > 5 && name.charAt(5) == ':') {
2403                     startPos = 6;
2404                 }
2405                 name = name.substring(startPos);
2406                 namespaces.put(name, attr.getValue());
2407             }
2408         }
2409         return namespaces;
2410     }
2411 
2412     /**
2413      * {@inheritDoc}
2414      */
2415     @Override
2416     public void setDocumentType(final DocumentType type) {
2417         super.setDocumentType(type);
2418     }
2419 
2420     /**
2421      * Saves the current page, with all images, to the specified location.
2422      * The default behavior removes all script elements.
2423      *
2424      * @param file file to write this page into
2425      * @throws IOException If an error occurs
2426      */
2427     public void save(final File file) throws IOException {
2428         new XmlSerializer().save(this, file);
2429     }
2430 
2431     /**
2432      * Returns whether the current page mode is in {@code quirks mode} or in {@code standards mode}.
2433      * @return true for {@code quirks mode}, false for {@code standards mode}
2434      */
2435     public boolean isQuirksMode() {
2436         return "BackCompat".equals(((HTMLDocument) getScriptableObject()).getCompatMode());
2437     }
2438 
2439     /**
2440      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2441      * {@inheritDoc}
2442      */
2443     @Override
2444     public boolean isAttachedToPage() {
2445         return true;
2446     }
2447 
2448     /**
2449      * {@inheritDoc}
2450      */
2451     @Override
2452     public boolean isHtmlPage() {
2453         return true;
2454     }
2455 
2456     /**
2457      * The base URL used to resolve relative URLs.
2458      * @return the base URL
2459      */
2460     public URL getBaseURL() {
2461         URL baseUrl;
2462         if (base_ == null) {
2463             baseUrl = getUrl();
2464             final WebWindow window = getEnclosingWindow();
2465             final boolean frame = window != null && window != window.getTopWindow();
2466             if (frame) {
2467                 final boolean frameSrcIsNotSet = baseUrl == UrlUtils.URL_ABOUT_BLANK;
2468                 final boolean frameSrcIsJs = "javascript".equals(baseUrl.getProtocol());
2469                 if (frameSrcIsNotSet || frameSrcIsJs) {
2470                     baseUrl = window.getTopWindow().getEnclosedPage().getWebResponse()
2471                         .getWebRequest().getUrl();
2472                 }
2473             }
2474             else if (baseUrl_ != null) {
2475                 baseUrl = baseUrl_;
2476             }
2477         }
2478         else {
2479             final String href = base_.getHrefAttribute().trim();
2480             if (org.htmlunit.util.StringUtils.isEmptyOrNull(href)) {
2481                 baseUrl = getUrl();
2482             }
2483             else {
2484                 final URL url = getUrl();
2485                 try {
2486                     if (href.startsWith("http://") || href.startsWith("https://")) {
2487                         baseUrl = new URL(href);
2488                     }
2489                     else if (href.startsWith("//")) {
2490                         baseUrl = new URL("%s:%s".formatted(url.getProtocol(), href));
2491                     }
2492                     else if (href.length() > 0 && href.charAt(0) == '/') {
2493                         final int port = Window.getPort(url);
2494                         baseUrl = new URL("%s://%s:%d%s".formatted(url.getProtocol(), url.getHost(), port, href));
2495                     }
2496                     else if (url.toString().endsWith("/")) {
2497                         baseUrl = new URL("%s%s".formatted(url, href));
2498                     }
2499                     else {
2500                         baseUrl = new URL(UrlUtils.resolveUrl(url, href));
2501                     }
2502                 }
2503                 catch (final MalformedURLException e) {
2504                     notifyIncorrectness("Invalid base url: \"" + href + "\", ignoring it");
2505                     baseUrl = url;
2506                 }
2507             }
2508         }
2509 
2510         return baseUrl;
2511     }
2512 
2513     /**
2514      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2515      *
2516      * Adds an {@link AutoCloseable}, which would be closed during the {@link #cleanUp()}.
2517      * @param autoCloseable the autoclosable
2518      */
2519     public void addAutoCloseable(final AutoCloseable autoCloseable) {
2520         if (autoCloseable == null) {
2521             return;
2522         }
2523 
2524         if (autoCloseableList_ == null) {
2525             autoCloseableList_ = new ArrayList<>();
2526         }
2527         autoCloseableList_.add(autoCloseable);
2528     }
2529 
2530     /**
2531      * {@inheritDoc}
2532      */
2533     @Override
2534     public boolean handles(final Event event) {
2535         if (Event.TYPE_BLUR.equals(event.getType()) || Event.TYPE_FOCUS.equals(event.getType())) {
2536             return true;
2537         }
2538         return super.handles(event);
2539     }
2540 
2541     /**
2542      * Sets the {@link ElementFromPointHandler}.
2543      * @param elementFromPointHandler the handler
2544      */
2545     public void setElementFromPointHandler(final ElementFromPointHandler elementFromPointHandler) {
2546         elementFromPointHandler_ = elementFromPointHandler;
2547     }
2548 
2549     /**
2550      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2551      *
2552      * Returns the element for the specified x coordinate and the specified y coordinate.
2553      *
2554      * @param x the x offset, in pixels
2555      * @param y the y offset, in pixels
2556      * @return the element for the specified x coordinate and the specified y coordinate
2557      */
2558     public HtmlElement getElementFromPoint(final int x, final int y) {
2559         if (elementFromPointHandler_ == null) {
2560             if (LOG.isWarnEnabled()) {
2561                 LOG.warn("ElementFromPointHandler was not specicifed for " + this);
2562             }
2563             if (x <= 0 || y <= 0) {
2564                 return null;
2565             }
2566             return getBody();
2567         }
2568         return elementFromPointHandler_.getElementFromPoint(this, x, y);
2569     }
2570 
2571     /**
2572      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2573      * event handlers.
2574      *
2575      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2576      * @return true if the specified element now has the focus
2577      * @see #getFocusedElement()
2578      */
2579     public boolean setFocusedElement(final DomElement newElement) {
2580         return setFocusedElement(newElement, false);
2581     }
2582 
2583     /**
2584      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2585      * event handlers.
2586      *
2587      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2588      * @param windowActivated - whether the enclosing window got focus resulting in specified element getting focus
2589      * @return true if the specified element now has the focus
2590      * @see #getFocusedElement()
2591      */
2592     public boolean setFocusedElement(final DomElement newElement, final boolean windowActivated) {
2593         if (elementWithFocus_ == newElement && !windowActivated) {
2594             // nothing to do
2595             return true;
2596         }
2597 
2598         final DomElement oldFocusedElement = elementWithFocus_;
2599         elementWithFocus_ = null;
2600 
2601         if (!windowActivated) {
2602             if (oldFocusedElement != null) {
2603                 oldFocusedElement.removeFocus();
2604                 oldFocusedElement.fireEvent(Event.TYPE_BLUR);
2605 
2606                 oldFocusedElement.fireEvent(Event.TYPE_FOCUS_OUT);
2607             }
2608         }
2609 
2610         elementWithFocus_ = newElement;
2611 
2612         // use newElement in the code below because element elementWithFocus_
2613         // might be changed by another thread
2614         if (newElement != null) {
2615             newElement.focus();
2616             newElement.fireEvent(Event.TYPE_FOCUS);
2617 
2618             newElement.fireEvent(Event.TYPE_FOCUS_IN);
2619         }
2620 
2621         // If a page reload happened as a result of the focus change then obviously this
2622         // element will not have the focus because its page has gone away.
2623         return this == getEnclosingWindow().getEnclosedPage();
2624     }
2625 
2626     /**
2627      * Returns the element with the focus or null if no element has the focus.
2628      * @return the element with focus or null
2629      * @see #setFocusedElement(DomElement)
2630      */
2631     public DomElement getFocusedElement() {
2632         return elementWithFocus_;
2633     }
2634 
2635     /**
2636      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2637      *
2638      * Sets the element with focus.
2639      * @param elementWithFocus the element with focus
2640      */
2641     public void setElementWithFocus(final DomElement elementWithFocus) {
2642         elementWithFocus_ = elementWithFocus;
2643     }
2644 
2645     /**
2646      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2647      * Returns the currently active element.
2648      *
2649      * @return the element that currently has focus, or the document's
2650      *         {@code body} element if no element has focus, or {@code null}
2651      *         if the document has no {@code body} element
2652      */
2653     public HtmlElement getActiveElement() {
2654         final DomElement activeElement = getFocusedElement();
2655         if (activeElement instanceof HtmlElement element) {
2656             return element;
2657         }
2658 
2659         final HtmlElement body = getBody();
2660         if (body != null) {
2661             return body;
2662         }
2663         return null;
2664     }
2665 
2666     /**
2667      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2668      *
2669      * <p>Returns the page's current selection ranges.</p>
2670      *
2671      * @return the page's current selection ranges
2672      */
2673     public List<SimpleRange> getSelectionRanges() {
2674         return selectionRanges_;
2675     }
2676 
2677     /**
2678      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2679      *
2680      * <p>Makes the specified selection range the *only* selection range on this page.</p>
2681      *
2682      * @param selectionRange the selection range
2683      */
2684     public void setSelectionRange(final SimpleRange selectionRange) {
2685         selectionRanges_.clear();
2686         selectionRanges_.add(selectionRange);
2687     }
2688 
2689     /**
2690      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2691      *
2692      * Execute a Function in the given context.
2693      *
2694      * @param function the JavaScript Function to call
2695      * @param thisObject the "this" object to be used during invocation
2696      * @param args the arguments to pass into the call
2697      * @param htmlElement the HTML element for which this script is being executed
2698      *        This element will be the context during the JavaScript execution. If null,
2699      *        the context will default to the page.
2700      * @return a ScriptResult which will contain both the current page (which may be different from
2701      *        the previous page) and a JavaScript result object.
2702      */
2703     public ScriptResult executeJavaScriptFunction(final Object function, final Object thisObject,
2704             final Object[] args, final DomNode htmlElement) {
2705         if (!getWebClient().isJavaScriptEnabled()) {
2706             return new ScriptResult(null);
2707         }
2708 
2709         final JavaScriptEngine engine = (JavaScriptEngine) getWebClient().getJavaScriptEngine();
2710         final Object result = engine.callFunction(this,
2711                                 (Function) function, (Scriptable) thisObject, args, htmlElement);
2712 
2713         return new ScriptResult(result);
2714     }
2715 
2716     private void writeObject(final ObjectOutputStream oos) throws IOException {
2717         oos.defaultWriteObject();
2718         oos.writeObject(originalCharset_ == null ? null : originalCharset_.name());
2719     }
2720 
2721     private void readObject(final ObjectInputStream ois) throws ClassNotFoundException, IOException {
2722         ois.defaultReadObject();
2723         final String charsetName = (String) ois.readObject();
2724         if (charsetName != null) {
2725             originalCharset_ = Charset.forName(charsetName);
2726         }
2727     }
2728 
2729     /**
2730      * {@inheritDoc}
2731      */
2732     @Override
2733     public void setNodeValue(final String value) {
2734         // Default behavior is to do nothing, overridden in some subclasses
2735     }
2736 
2737     /**
2738      * {@inheritDoc}
2739      */
2740     @Override
2741     public void setPrefix(final String prefix) {
2742         // Empty.
2743     }
2744 
2745     /**
2746      * {@inheritDoc}
2747      */
2748     @Override
2749     public void clearComputedStyles() {
2750         if (computedStylesCache_ != null) {
2751             computedStylesCache_.clear();
2752         }
2753     }
2754 
2755     /**
2756      * {@inheritDoc}
2757      */
2758     @Override
2759     public void clearComputedStyles(final DomElement element) {
2760         if (computedStylesCache_ != null) {
2761             computedStylesCache_.remove(element);
2762         }
2763     }
2764 
2765     /**
2766      * {@inheritDoc}
2767      */
2768     @Override
2769     public void clearComputedStylesUpToRoot(final DomElement element) {
2770         if (computedStylesCache_ != null) {
2771             computedStylesCache_.remove(element);
2772 
2773             DomNode parent = element.getParentNode();
2774             while (parent != null) {
2775                 computedStylesCache_.remove(parent);
2776                 parent = parent.getParentNode();
2777             }
2778         }
2779     }
2780 
2781     /**
2782      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2783      *
2784      * @param element the element to clear its cache
2785      * @param normalizedPseudo the pseudo attribute
2786      * @return the cached ComputedCssStyleDeclaration object or null
2787      */
2788     public ComputedCssStyleDeclaration getStyleFromCache(final DomElement element,
2789             final String normalizedPseudo) {
2790         return getCssPropertiesCache().get(element, normalizedPseudo);
2791     }
2792 
2793     /**
2794      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2795      *
2796      * Caches a ComputedCssStyleDeclaration object.
2797      * @param element the element to clear its cache
2798      * @param normalizedPseudo the pseudo attribute
2799      * @param style the ComputedCssStyleDeclaration to cache
2800      */
2801     public void putStyleIntoCache(final DomElement element, final String normalizedPseudo,
2802             final ComputedCssStyleDeclaration style) {
2803         getCssPropertiesCache().put(element, normalizedPseudo, style);
2804     }
2805 
2806     /**
2807      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2808      *
2809      * @return a list of all styles from this page (&lt;style&gt; and &lt;link rel=stylesheet&gt;).
2810      *         This returns an empty list if css support is disabled in the web client options.
2811      */
2812     public List<CssStyleSheet> getStyleSheets() {
2813         final List<CssStyleSheet> styles = new ArrayList<>();
2814         if (getWebClient().getOptions().isCssEnabled()) {
2815             for (final HtmlElement htmlElement : getHtmlElementDescendants()) {
2816                 if (htmlElement instanceof HtmlStyle style) {
2817                     styles.add(style.getSheet());
2818                     continue;
2819                 }
2820 
2821                 if (htmlElement instanceof HtmlLink link) {
2822                     if (link.isStyleSheetLink()) {
2823                         styles.add(link.getSheet());
2824                     }
2825                 }
2826             }
2827         }
2828         return styles;
2829     }
2830 
2831     /**
2832      * Returns the computed styles cache for this page.
2833      *
2834      * @return the computed styles cache for this page
2835      */
2836     private ComputedStylesCache getCssPropertiesCache() {
2837         if (computedStylesCache_ == null) {
2838             computedStylesCache_ = new ComputedStylesCache();
2839 
2840             // maintain the style cache
2841             final DomHtmlAttributeChangeListenerImpl listener = new DomHtmlAttributeChangeListenerImpl();
2842             addDomChangeListener(listener);
2843             addHtmlAttributeChangeListener(listener);
2844         }
2845         return computedStylesCache_;
2846     }
2847 
2848     /**
2849      * <p>Listens for changes anywhere in the document and evicts cached computed styles whenever something relevant
2850      * changes. Note that the very lazy way of doing this (completely clearing the cache every time something happens)
2851      * results in very meager performance gains. In order to get good (but still correct) performance, we need to be
2852      * a little smarter.</p>
2853      *
2854      * <p>CSS 2.1 has the following <a href="http://www.w3.org/TR/CSS21/selector.html">selector types</a> (where "SN" is
2855      * shorthand for "the selected node"):</p>
2856      *
2857      * <ol>
2858      *   <li><em>Universal</em> (i.e. "*"): Affected by the removal of SN from the document.</li>
2859      *   <li><em>Type</em> (i.e. "div"): Affected by the removal of SN from the document.</li>
2860      *   <li><em>Descendant</em> (i.e. "div span"): Affected by changes to SN or to any of its ancestors.</li>
2861      *   <li><em>Child</em> (i.e. "div &gt; span"): Affected by changes to SN or to its parent.</li>
2862      *   <li><em>Adjacent Sibling</em> (i.e. "table + p"): Affected by changes to SN or its previous sibling.</li>
2863      *   <li><em>Attribute</em> (i.e. "div.up, div[class~=up]"): Affected by changes to an attribute of SN.</li>
2864      *   <li><em>ID</em> (i.e. "#header"): Affected by changes to the <code>id</code> attribute of SN.</li>
2865      *   <li><em>Pseudo-Elements and Pseudo-Classes</em> (i.e. "p:first-child"): Affected by changes to parent.</li>
2866      * </ol>
2867      *
2868      * <p>Together, these rules dictate that the smart (but still lazy) way of removing elements from the computed style
2869      * cache is as follows -- whenever a node changes in any way, the cache needs to be cleared of styles for nodes
2870      * which:</p>
2871      *
2872      * <ul>
2873      *   <li>are actually the same node as the node that changed</li>
2874      *   <li>are siblings of the node that changed</li>
2875      *   <li>are descendants of the node that changed</li>
2876      * </ul>
2877      *
2878      * <p>Additionally, whenever a <code>style</code> node or a <code>link</code> node
2879      * with <code>rel=stylesheet</code> is added or
2880      * removed, all elements should be removed from the computed style cache.</p>
2881      */
2882     private class DomHtmlAttributeChangeListenerImpl implements DomChangeListener, HtmlAttributeChangeListener {
2883 
2884         /**
2885          * Ctor.
2886          */
2887         DomHtmlAttributeChangeListenerImpl() {
2888             super();
2889         }
2890 
2891         /**
2892          * {@inheritDoc}
2893          */
2894         @Override
2895         public void nodeAdded(final DomChangeEvent event) {
2896             nodeChanged(event.getChangedNode(), null);
2897         }
2898 
2899         /**
2900          * {@inheritDoc}
2901          */
2902         @Override
2903         public void nodeDeleted(final DomChangeEvent event) {
2904             nodeChanged(event.getChangedNode(), null);
2905         }
2906 
2907         /**
2908          * {@inheritDoc}
2909          */
2910         @Override
2911         public void attributeAdded(final HtmlAttributeChangeEvent event) {
2912             nodeChanged(event.getHtmlElement(), event.getName());
2913         }
2914 
2915         /**
2916          * {@inheritDoc}
2917          */
2918         @Override
2919         public void attributeRemoved(final HtmlAttributeChangeEvent event) {
2920             nodeChanged(event.getHtmlElement(), event.getName());
2921         }
2922 
2923         /**
2924          * {@inheritDoc}
2925          */
2926         @Override
2927         public void attributeReplaced(final HtmlAttributeChangeEvent event) {
2928             nodeChanged(event.getHtmlElement(), event.getName());
2929         }
2930 
2931         private void nodeChanged(final DomNode changedNode, final String attribName) {
2932             // If a stylesheet was changed, all of our calculations could be off; clear the cache.
2933             if (changedNode instanceof HtmlStyle) {
2934                 clearComputedStyles();
2935                 return;
2936             }
2937             if (changedNode instanceof HtmlLink link) {
2938                 if (link.isStyleSheetLink()) {
2939                     clearComputedStyles();
2940                     return;
2941                 }
2942             }
2943 
2944             // Apparently it wasn't a stylesheet that changed; be semi-smart about what we evict and when.
2945             // null means that a node was added/removed; we always have to take care of this for the parents
2946             final boolean clearParents = attribName == null || ATTRIBUTES_AFFECTING_PARENT.contains(attribName);
2947             if (computedStylesCache_ != null) {
2948                 computedStylesCache_.nodeChanged(changedNode, clearParents);
2949             }
2950         }
2951     }
2952 
2953     /**
2954      * Cache computed styles when possible, because their calculation is very expensive.
2955      * We use a weak hash map because we don't want this cache to be the only reason
2956      * nodes are kept around in the JVM, if all other references to them are gone.
2957      */
2958     private static final class ComputedStylesCache implements Serializable {
2959         private transient WeakHashMap<DomElement, Map<String, ComputedCssStyleDeclaration>>
2960                     computedStyles_ = new WeakHashMap<>();
2961 
2962         /**
2963          * Ctor.
2964          */
2965         ComputedStylesCache() {
2966             super();
2967         }
2968 
2969         public synchronized ComputedCssStyleDeclaration get(final DomElement element,
2970                 final String normalizedPseudo) {
2971             final Map<String, ComputedCssStyleDeclaration> elementMap = computedStyles_.get(element);
2972             if (elementMap != null) {
2973                 return elementMap.get(normalizedPseudo);
2974             }
2975             return null;
2976         }
2977 
2978         public synchronized void put(final DomElement element,
2979                 final String normalizedPseudo, final ComputedCssStyleDeclaration style) {
2980             final Map<String, ComputedCssStyleDeclaration>
2981                     elementMap = computedStyles_.computeIfAbsent(element, k -> new WeakHashMap<>());
2982             elementMap.put(normalizedPseudo, style);
2983         }
2984 
2985         public synchronized void nodeChanged(final DomNode changed, final boolean clearParents) {
2986             final Iterator<Map.Entry<DomElement, Map<String, ComputedCssStyleDeclaration>>>
2987                     i = computedStyles_.entrySet().iterator();
2988             while (i.hasNext()) {
2989                 final Map.Entry<DomElement, Map<String, ComputedCssStyleDeclaration>> entry = i.next();
2990                 final DomElement node = entry.getKey();
2991                 if (changed == node
2992                     || changed.getParentNode() == node.getParentNode()
2993                     || changed.isAncestorOf(node)
2994                     || clearParents && node.isAncestorOf(changed)) {
2995                     i.remove();
2996                 }
2997             }
2998 
2999             // maybe this is a better solution but I have to think a bit more about this
3000             //
3001             //            if (computedStyles_.isEmpty()) {
3002             //                return;
3003             //            }
3004             //
3005             //            // remove all siblings
3006             //            DomNode parent = changed.getParentNode();
3007             //            if (parent != null) {
3008             //                for (DomNode sibling : parent.getChildNodes()) {
3009             //                    computedStyles_.remove(sibling.getScriptableObject());
3010             //                }
3011             //
3012             //                if (clearParents) {
3013             //                    // remove all parents
3014             //                    while (parent != null) {
3015             //                        computedStyles_.remove(parent.getScriptableObject());
3016             //                        parent = parent.getParentNode();
3017             //                    }
3018             //                }
3019             //            }
3020             //
3021             //            // remove changed itself and all descendants
3022             //            computedStyles_.remove(changed.getScriptableObject());
3023             //            for (DomNode descendant : changed.getDescendants()) {
3024             //                computedStyles_.remove(descendant.getScriptableObject());
3025             //            }
3026         }
3027 
3028         public synchronized void clear() {
3029             computedStyles_.clear();
3030         }
3031 
3032         public synchronized Map<String, ComputedCssStyleDeclaration> remove(final DomNode element) {
3033             return computedStyles_.remove(element);
3034         }
3035 
3036         private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
3037             in.defaultReadObject();
3038             computedStyles_ = new WeakHashMap<>();
3039         }
3040     }
3041 
3042     private static final class MappedElementIndexEntry implements Serializable {
3043         private final ArrayList<DomElement> elements_;
3044         private boolean sorted_;
3045 
3046         MappedElementIndexEntry() {
3047             // we do not expect to many elements having the same id/name
3048             elements_ = new ArrayList<>(2);
3049             sorted_ = true;
3050         }
3051 
3052         void add(final DomElement element) {
3053             if (elements_.indexOf(element) == -1) {
3054                 elements_.add(element);
3055                 sorted_ = elements_.size() < 2;
3056             }
3057         }
3058 
3059         DomElement first() {
3060             if (elements_.isEmpty()) {
3061                 return null;
3062             }
3063 
3064             if (sorted_) {
3065                 return elements_.get(0);
3066             }
3067 
3068             elements_.sort(DOCUMENT_POSITION_COMPERATOR);
3069             sorted_ = true;
3070 
3071             return elements_.get(0);
3072         }
3073 
3074         List<DomElement> elements() {
3075             if (sorted_) {
3076                 return elements_;
3077             }
3078 
3079             elements_.sort(DOCUMENT_POSITION_COMPERATOR);
3080             sorted_ = true;
3081 
3082             return elements_;
3083         }
3084 
3085         void remove(final DomElement element) {
3086             elements_.remove(element);
3087             sorted_ = elements_.size() < 2;
3088         }
3089     }
3090 }