View Javadoc
1   /*
2    * Copyright (c) 2002-2025 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit;
16  
17  import static java.nio.charset.StandardCharsets.ISO_8859_1;
18  import static java.nio.charset.StandardCharsets.UTF_8;
19  import static org.htmlunit.BrowserVersionFeatures.HTTP_HEADER_CH_UA;
20  import static org.htmlunit.BrowserVersionFeatures.HTTP_HEADER_PRIORITY;
21  
22  import java.io.BufferedInputStream;
23  import java.io.File;
24  import java.io.IOException;
25  import java.io.InputStream;
26  import java.io.ObjectInputStream;
27  import java.io.Serializable;
28  import java.lang.ref.WeakReference;
29  import java.net.MalformedURLException;
30  import java.net.URL;
31  import java.net.URLConnection;
32  import java.net.URLDecoder;
33  import java.nio.charset.Charset;
34  import java.nio.file.Files;
35  import java.util.ArrayList;
36  import java.util.Collections;
37  import java.util.ConcurrentModificationException;
38  import java.util.Date;
39  import java.util.HashMap;
40  import java.util.HashSet;
41  import java.util.Iterator;
42  import java.util.LinkedHashMap;
43  import java.util.LinkedHashSet;
44  import java.util.List;
45  import java.util.Locale;
46  import java.util.Map;
47  import java.util.Objects;
48  import java.util.Optional;
49  import java.util.Set;
50  import java.util.concurrent.ConcurrentLinkedDeque;
51  import java.util.concurrent.Executor;
52  import java.util.concurrent.ExecutorService;
53  import java.util.concurrent.Executors;
54  import java.util.concurrent.ThreadFactory;
55  import java.util.concurrent.ThreadPoolExecutor;
56  
57  import org.apache.commons.lang3.StringUtils;
58  import org.apache.commons.logging.Log;
59  import org.apache.commons.logging.LogFactory;
60  import org.apache.http.NoHttpResponseException;
61  import org.apache.http.client.CredentialsProvider;
62  import org.apache.http.cookie.MalformedCookieException;
63  import org.htmlunit.attachment.Attachment;
64  import org.htmlunit.attachment.AttachmentHandler;
65  import org.htmlunit.csp.Policy;
66  import org.htmlunit.csp.url.URI;
67  import org.htmlunit.css.ComputedCssStyleDeclaration;
68  import org.htmlunit.cssparser.parser.CSSErrorHandler;
69  import org.htmlunit.cssparser.parser.javacc.CSS3Parser;
70  import org.htmlunit.html.BaseFrameElement;
71  import org.htmlunit.html.DomElement;
72  import org.htmlunit.html.DomNode;
73  import org.htmlunit.html.FrameWindow;
74  import org.htmlunit.html.FrameWindow.PageDenied;
75  import org.htmlunit.html.HtmlElement;
76  import org.htmlunit.html.HtmlInlineFrame;
77  import org.htmlunit.html.HtmlPage;
78  import org.htmlunit.html.XHtmlPage;
79  import org.htmlunit.html.parser.HTMLParser;
80  import org.htmlunit.html.parser.HTMLParserListener;
81  import org.htmlunit.http.HttpStatus;
82  import org.htmlunit.http.HttpUtils;
83  import org.htmlunit.httpclient.HttpClientConverter;
84  import org.htmlunit.javascript.AbstractJavaScriptEngine;
85  import org.htmlunit.javascript.DefaultJavaScriptErrorListener;
86  import org.htmlunit.javascript.HtmlUnitScriptable;
87  import org.htmlunit.javascript.JavaScriptEngine;
88  import org.htmlunit.javascript.JavaScriptErrorListener;
89  import org.htmlunit.javascript.background.JavaScriptJobManager;
90  import org.htmlunit.javascript.host.Location;
91  import org.htmlunit.javascript.host.Window;
92  import org.htmlunit.javascript.host.dom.Node;
93  import org.htmlunit.javascript.host.event.Event;
94  import org.htmlunit.javascript.host.file.Blob;
95  import org.htmlunit.javascript.host.html.HTMLIFrameElement;
96  import org.htmlunit.protocol.data.DataURLConnection;
97  import org.htmlunit.util.Cookie;
98  import org.htmlunit.util.HeaderUtils;
99  import org.htmlunit.util.MimeType;
100 import org.htmlunit.util.NameValuePair;
101 import org.htmlunit.util.UrlUtils;
102 import org.htmlunit.websocket.JettyWebSocketAdapter.JettyWebSocketAdapterFactory;
103 import org.htmlunit.websocket.WebSocketAdapter;
104 import org.htmlunit.websocket.WebSocketAdapterFactory;
105 import org.htmlunit.websocket.WebSocketListener;
106 import org.htmlunit.webstart.WebStartHandler;
107 
108 /**
109  * The main starting point in HtmlUnit: this class simulates a web browser.
110  * <p>
111  * A standard usage of HtmlUnit will start with using the {@link #getPage(String)} method
112  * (or {@link #getPage(URL)}) to load a first {@link Page}
113  * and will continue with further processing on this page depending on its type.
114  * </p>
115  * <b>Example:</b><br>
116  * <br>
117  * <code>
118  * final WebClient webClient = new WebClient();<br>
119  * final {@link HtmlPage} startPage = webClient.getPage("http://htmlunit.sf.net");<br>
120  * assertEquals("HtmlUnit - Welcome to HtmlUnit", startPage.{@link HtmlPage#getTitleText() getTitleText}());
121  * </code>
122  * <p>
123  * Note: a {@link WebClient} instance is <b>not thread safe</b>. It is intended to be used from a single thread.
124  * </p>
125  * @author <a href="mailto:mbowler@GargoyleSoftware.com">Mike Bowler</a>
126  * @author <a href="mailto:gudujarlson@sf.net">Mike J. Bresnahan</a>
127  * @author Dominique Broeglin
128  * @author Noboru Sinohara
129  * @author <a href="mailto:chen_jun@users.sourceforge.net">Chen Jun</a>
130  * @author David K. Taylor
131  * @author <a href="mailto:cse@dynabean.de">Christian Sell</a>
132  * @author <a href="mailto:bcurren@esomnie.com">Ben Curren</a>
133  * @author Marc Guillemot
134  * @author Chris Erskine
135  * @author Daniel Gredler
136  * @author Sergey Gorelkin
137  * @author Hans Donner
138  * @author Paul King
139  * @author Ahmed Ashour
140  * @author Bruce Chapman
141  * @author Sudhan Moghe
142  * @author Martin Tamme
143  * @author Amit Manjhi
144  * @author Nicolas Belisle
145  * @author Ronald Brill
146  * @author Frank Danek
147  * @author Joerg Werner
148  * @author Anton Demydenko
149  * @author Sergio Moreno
150  * @author Lai Quang Duong
151  * @author René Schwietzke
152  * @author Sven Strickroth
153  */
154 @SuppressWarnings("PMD.TooManyFields")
155 public class WebClient implements Serializable, AutoCloseable {
156 
157     /** Logging support. */
158     private static final Log LOG = LogFactory.getLog(WebClient.class);
159 
160     /** Like the Firefox default value for {@code network.http.redirection-limit}. */
161     private static final int ALLOWED_REDIRECTIONS_SAME_URL = 20;
162     private static final WebResponseData RESPONSE_DATA_NO_HTTP_RESPONSE = new WebResponseData(
163             0, "No HTTP Response", Collections.emptyList());
164 
165     /**
166      * These response headers are not copied from a 304 response to the cached
167      * response headers. This list is based on Chromium http_response_headers.cc
168      */
169     private static final String[] DISCARDING_304_RESPONSE_HEADER_NAMES = {
170         "connection",
171         "proxy-connection",
172         "keep-alive",
173         "www-authenticate",
174         "proxy-authenticate",
175         "proxy-authorization",
176         "te",
177         "trailer",
178         "transfer-encoding",
179         "upgrade",
180         "content-location",
181         "content-md5",
182         "etag",
183         "content-encoding",
184         "content-range",
185         "content-type",
186         "content-length",
187         "x-frame-options",
188         "x-xss-protection",
189     };
190 
191     private static final String[] DISCARDING_304_HEADER_PREFIXES = {
192         "x-content-",
193         "x-webkit-"
194     };
195 
196     private transient WebConnection webConnection_;
197     private CredentialsProvider credentialsProvider_ = new DefaultCredentialsProvider();
198     private CookieManager cookieManager_ = new CookieManager();
199     private WebSocketAdapterFactory webSocketAdapterFactory_;
200     private transient AbstractJavaScriptEngine<?> scriptEngine_;
201     private transient List<LoadJob> loadQueue_;
202     private final Map<String, String> requestHeaders_ = Collections.synchronizedMap(new HashMap<>(89));
203     private IncorrectnessListener incorrectnessListener_ = new IncorrectnessListenerImpl();
204     private WebConsole webConsole_;
205     private transient ExecutorService executor_;
206 
207     private AlertHandler alertHandler_;
208     private ConfirmHandler confirmHandler_;
209     private PromptHandler promptHandler_;
210     private StatusHandler statusHandler_;
211     private AttachmentHandler attachmentHandler_;
212     private ClipboardHandler clipboardHandler_;
213     private PrintHandler printHandler_;
214     private WebStartHandler webStartHandler_;
215     private FrameContentHandler frameContentHandler_;
216 
217     private AjaxController ajaxController_ = new AjaxController();
218 
219     private final BrowserVersion browserVersion_;
220     private PageCreator pageCreator_ = new DefaultPageCreator();
221 
222     // we need a separate one to be sure the one is always informed as first
223     // one. Only then we can make sure our state is consistent when the others
224     // are informed.
225     private CurrentWindowTracker currentWindowTracker_;
226     private final Set<WebWindowListener> webWindowListeners_ = new HashSet<>(5);
227 
228     private final List<TopLevelWindow> topLevelWindows_ =
229             Collections.synchronizedList(new ArrayList<>()); // top-level windows
230     private final List<WebWindow> windows_ = Collections.synchronizedList(new ArrayList<>()); // all windows
231     private transient List<WeakReference<JavaScriptJobManager>> jobManagers_ =
232             Collections.synchronizedList(new ArrayList<>());
233     private WebWindow currentWindow_;
234 
235     private HTMLParserListener htmlParserListener_;
236     private CSSErrorHandler cssErrorHandler_ = new DefaultCssErrorHandler();
237     private OnbeforeunloadHandler onbeforeunloadHandler_;
238     private Cache cache_ = new Cache();
239 
240     // mini pool to save resource when parsing CSS
241     private transient CSS3ParserPool css3ParserPool_ = new CSS3ParserPool();
242 
243     /** target "_blank". */
244     public static final String TARGET_BLANK = "_blank";
245 
246     /** target "_self". */
247     public static final String TARGET_SELF = "_self";
248 
249     /** target "_parent". */
250     private static final String TARGET_PARENT = "_parent";
251     /** target "_top". */
252     private static final String TARGET_TOP = "_top";
253 
254     private ScriptPreProcessor scriptPreProcessor_;
255 
256     private RefreshHandler refreshHandler_ = new NiceRefreshHandler(2);
257     private JavaScriptErrorListener javaScriptErrorListener_ = new DefaultJavaScriptErrorListener();
258 
259     private final WebClientOptions options_ = new WebClientOptions();
260     private final boolean javaScriptEngineEnabled_;
261     private final StorageHolder storageHolder_ = new StorageHolder();
262 
263     /**
264      * Creates a web client instance using the browser version returned by
265      * {@link BrowserVersion#getDefault()}.
266      */
267     public WebClient() {
268         this(BrowserVersion.getDefault());
269     }
270 
271     /**
272      * Creates a web client instance using the specified {@link BrowserVersion}.
273      * @param browserVersion the browser version to simulate
274      */
275     public WebClient(final BrowserVersion browserVersion) {
276         this(browserVersion, null, -1);
277     }
278 
279     /**
280      * Creates an instance that will use the specified {@link BrowserVersion} and proxy server.
281      * @param browserVersion the browser version to simulate
282      * @param proxyHost the server that will act as proxy or null for no proxy
283      * @param proxyPort the port to use on the proxy server
284      */
285     public WebClient(final BrowserVersion browserVersion, final String proxyHost, final int proxyPort) {
286         this(browserVersion, true, proxyHost, proxyPort, null);
287     }
288 
289     /**
290      * Creates an instance that will use the specified {@link BrowserVersion} and proxy server.
291      * @param browserVersion the browser version to simulate
292      * @param proxyHost the server that will act as proxy or null for no proxy
293      * @param proxyPort the port to use on the proxy server
294      * @param proxyScheme the scheme http/https
295      */
296     public WebClient(final BrowserVersion browserVersion,
297             final String proxyHost, final int proxyPort, final String proxyScheme) {
298         this(browserVersion, true, proxyHost, proxyPort, proxyScheme);
299     }
300 
301     /**
302      * Creates an instance that will use the specified {@link BrowserVersion} and proxy server.
303      * @param browserVersion the browser version to simulate
304      * @param javaScriptEngineEnabled set to false if the simulated browser should not support javaScript
305      * @param proxyHost the server that will act as proxy or null for no proxy
306      * @param proxyPort the port to use on the proxy server
307      */
308     public WebClient(final BrowserVersion browserVersion, final boolean javaScriptEngineEnabled,
309             final String proxyHost, final int proxyPort) {
310         this(browserVersion, javaScriptEngineEnabled, proxyHost, proxyPort, null);
311     }
312 
313     /**
314      * Creates an instance that will use the specified {@link BrowserVersion} and proxy server.
315      * @param browserVersion the browser version to simulate
316      * @param javaScriptEngineEnabled set to false if the simulated browser should not support javaScript
317      * @param proxyHost the server that will act as proxy or null for no proxy
318      * @param proxyPort the port to use on the proxy server
319      * @param proxyScheme the scheme http/https
320      */
321     public WebClient(final BrowserVersion browserVersion, final boolean javaScriptEngineEnabled,
322             final String proxyHost, final int proxyPort, final String proxyScheme) {
323         WebAssert.notNull("browserVersion", browserVersion);
324 
325         browserVersion_ = browserVersion;
326         javaScriptEngineEnabled_ = javaScriptEngineEnabled;
327 
328         if (proxyHost == null) {
329             getOptions().setProxyConfig(new ProxyConfig());
330         }
331         else {
332             getOptions().setProxyConfig(new ProxyConfig(proxyHost, proxyPort, proxyScheme));
333         }
334 
335         webConnection_ = new HttpWebConnection(this); // this has to be done after the browser version was set
336         if (javaScriptEngineEnabled_) {
337             scriptEngine_ = new JavaScriptEngine(this);
338         }
339         loadQueue_ = new ArrayList<>();
340 
341         webSocketAdapterFactory_ = new JettyWebSocketAdapterFactory();
342 
343         // The window must be constructed AFTER the script engine.
344         currentWindowTracker_ = new CurrentWindowTracker(this, true);
345         currentWindow_ = new TopLevelWindow("", this);
346     }
347 
348     /**
349      * Our simple impl of a ThreadFactory (decorator) to be able to name
350      * our threads.
351      */
352     private static final class ThreadNamingFactory implements ThreadFactory {
353         private static int ID_ = 1;
354         private final ThreadFactory baseFactory_;
355 
356         ThreadNamingFactory(final ThreadFactory aBaseFactory) {
357             baseFactory_ = aBaseFactory;
358         }
359 
360         @Override
361         public Thread newThread(final Runnable aRunnable) {
362             final Thread thread = baseFactory_.newThread(aRunnable);
363             thread.setName("WebClient Thread " + ID_++);
364             return thread;
365         }
366     }
367 
368     /**
369      * Returns the object that will resolve all URL requests.
370      *
371      * @return the connection that will be used
372      */
373     public WebConnection getWebConnection() {
374         return webConnection_;
375     }
376 
377     /**
378      * Sets the object that will resolve all URL requests.
379      *
380      * @param webConnection the new web connection
381      */
382     public void setWebConnection(final WebConnection webConnection) {
383         WebAssert.notNull("webConnection", webConnection);
384         webConnection_ = webConnection;
385     }
386 
387     /**
388      * Send a request to a server and return a Page that represents the
389      * response from the server. This page will be used to populate the provided window.
390      * <p>
391      * The returned {@link Page} will be created by the {@link PageCreator}
392      * configured by {@link #setPageCreator(PageCreator)}, if any.
393      * <p>
394      * The {@link DefaultPageCreator} will create a {@link Page} depending on the content type of the HTTP response,
395      * basically {@link HtmlPage} for HTML content, {@link org.htmlunit.xml.XmlPage} for XML content,
396      * {@link TextPage} for other text content and {@link UnexpectedPage} for anything else.
397      *
398      * @param webWindow the WebWindow to load the result of the request into
399      * @param webRequest the web request
400      * @param <P> the page type
401      * @return the page returned by the server when the specified request was made in the specified window
402      * @throws IOException if an IO error occurs
403      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
404      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
405      *
406      * @see WebRequest
407      */
408     public <P extends Page> P getPage(final WebWindow webWindow, final WebRequest webRequest)
409             throws IOException, FailingHttpStatusCodeException {
410         return getPage(webWindow, webRequest, true);
411     }
412 
413     /**
414      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
415      *
416      * Send a request to a server and return a Page that represents the
417      * response from the server. This page will be used to populate the provided window.
418      * <p>
419      * The returned {@link Page} will be created by the {@link PageCreator}
420      * configured by {@link #setPageCreator(PageCreator)}, if any.
421      * <p>
422      * The {@link DefaultPageCreator} will create a {@link Page} depending on the content type of the HTTP response,
423      * basically {@link HtmlPage} for HTML content, {@link org.htmlunit.xml.XmlPage} for XML content,
424      * {@link TextPage} for other text content and {@link UnexpectedPage} for anything else.
425      *
426      * @param webWindow the WebWindow to load the result of the request into
427      * @param webRequest the web request
428      * @param addToHistory true if the page should be part of the history
429      * @param <P> the page type
430      * @return the page returned by the server when the specified request was made in the specified window
431      * @throws IOException if an IO error occurs
432      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
433      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
434      *
435      * @see WebRequest
436      */
437     @SuppressWarnings("unchecked")
438     <P extends Page> P getPage(final WebWindow webWindow, final WebRequest webRequest,
439             final boolean addToHistory)
440         throws IOException, FailingHttpStatusCodeException {
441 
442         final Page page = webWindow.getEnclosedPage();
443 
444         if (page != null) {
445             final URL prev = page.getUrl();
446             final URL current = webRequest.getUrl();
447             if (UrlUtils.sameFile(current, prev)
448                         && current.getRef() != null
449                         && !Objects.equals(current.getRef(), prev.getRef())) {
450                 // We're just navigating to an anchor within the current page.
451                 page.getWebResponse().getWebRequest().setUrl(current);
452                 if (addToHistory) {
453                     webWindow.getHistory().addPage(page);
454                 }
455 
456                 // clear the cache because the anchors are now matched by
457                 // the target pseudo style
458                 if (page instanceof HtmlPage) {
459                     ((HtmlPage) page).clearComputedStyles();
460                 }
461 
462                 final Window window = webWindow.getScriptableObject();
463                 if (window != null) { // js enabled
464                     window.getLocation().setHash(current.getRef());
465                 }
466                 return (P) page;
467             }
468 
469             if (page.isHtmlPage()) {
470                 final HtmlPage htmlPage = (HtmlPage) page;
471                 if (!htmlPage.isOnbeforeunloadAccepted()) {
472                     LOG.debug("The registered OnbeforeunloadHandler rejected to load a new page.");
473                     return (P) page;
474                 }
475             }
476         }
477 
478         if (LOG.isDebugEnabled()) {
479             LOG.debug("Get page for window named '" + webWindow.getName() + "', using " + webRequest);
480         }
481 
482         WebResponse webResponse;
483         final String protocol = webRequest.getUrl().getProtocol();
484         if ("javascript".equals(protocol)) {
485             webResponse = makeWebResponseForJavaScriptUrl(webWindow, webRequest.getUrl(), webRequest.getCharset());
486             if (webWindow.getEnclosedPage() != null && webWindow.getEnclosedPage().getWebResponse() == webResponse) {
487                 // a javascript:... url with result of type undefined didn't changed the page
488                 return (P) webWindow.getEnclosedPage();
489             }
490         }
491         else {
492             try {
493                 webResponse = loadWebResponse(webRequest);
494             }
495             catch (final NoHttpResponseException e) {
496                 webResponse = new WebResponse(RESPONSE_DATA_NO_HTTP_RESPONSE, webRequest, 0);
497             }
498         }
499 
500         printContentIfNecessary(webResponse);
501         loadWebResponseInto(webResponse, webWindow);
502 
503         // start execution here
504         // note: we have to do this also if the server reports an error!
505         //       e.g. if the server returns a 404 error page that includes javascript
506         if (scriptEngine_ != null) {
507             scriptEngine_.registerWindowAndMaybeStartEventLoop(webWindow);
508         }
509 
510         // check and report problems if needed
511         throwFailingHttpStatusCodeExceptionIfNecessary(webResponse);
512         return (P) webWindow.getEnclosedPage();
513     }
514 
515     /**
516      * Convenient method to build a URL and load it into the current WebWindow as it would be done
517      * by {@link #getPage(WebWindow, WebRequest)}.
518      * @param url the URL of the new content; in contrast to real browsers plain file url's are not supported.
519      *        You have to use the 'file', 'data', 'blob', 'http' or 'https' protocol.
520      * @param <P> the page type
521      * @return the new page
522      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
523      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
524      * @throws IOException if an IO problem occurs
525      * @throws MalformedURLException if no URL can be created from the provided string
526      */
527     public <P extends Page> P getPage(final String url) throws IOException, FailingHttpStatusCodeException,
528         MalformedURLException {
529         return getPage(UrlUtils.toUrlUnsafe(url));
530     }
531 
532     /**
533      * Convenient method to load a URL into the current top WebWindow as it would be done
534      * by {@link #getPage(WebWindow, WebRequest)}.
535      * @param url the URL of the new content; in contrast to real browsers plain file url's are not supported.
536      *        You have to use the 'file', 'data', 'blob', 'http' or 'https' protocol.
537      * @param <P> the page type
538      * @return the new page
539      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
540      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
541      * @throws IOException if an IO problem occurs
542      */
543     public <P extends Page> P getPage(final URL url) throws IOException, FailingHttpStatusCodeException {
544         final WebRequest request = new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader(),
545                                                           getBrowserVersion().getAcceptEncodingHeader());
546         request.setCharset(UTF_8);
547         return getPage(getCurrentWindow().getTopWindow(), request);
548     }
549 
550     /**
551      * Convenient method to load a web request into the current top WebWindow.
552      * @param request the request parameters
553      * @param <P> the page type
554      * @return the new page
555      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
556      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true.
557      * @throws IOException if an IO problem occurs
558      * @see #getPage(WebWindow,WebRequest)
559      */
560     public <P extends Page> P getPage(final WebRequest request) throws IOException,
561         FailingHttpStatusCodeException {
562         return getPage(getCurrentWindow().getTopWindow(), request);
563     }
564 
565     /**
566      * <p>Creates a page based on the specified response and inserts it into the specified window. All page
567      * initialization and event notification is handled here.</p>
568      *
569      * <p>Note that if the page created is an attachment page, and an {@link AttachmentHandler} has been
570      * registered with this client, the page is <b>not</b> loaded into the specified window; in this case,
571      * the page is loaded into a new window, and attachment handling is delegated to the registered
572      * <code>AttachmentHandler</code>.</p>
573      *
574      * @param webResponse the response that will be used to create the new page
575      * @param webWindow the window that the new page will be placed within
576      * @throws IOException if an IO error occurs
577      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
578      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
579      * @return the newly created page
580      * @see #setAttachmentHandler(AttachmentHandler)
581      */
582     public Page loadWebResponseInto(final WebResponse webResponse, final WebWindow webWindow)
583         throws IOException, FailingHttpStatusCodeException {
584         return loadWebResponseInto(webResponse, webWindow, null);
585     }
586 
587     /**
588      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
589      *
590      * <p>Creates a page based on the specified response and inserts it into the specified window. All page
591      * initialization and event notification is handled here.</p>
592      *
593      * <p>Note that if the page created is an attachment page, and an {@link AttachmentHandler} has been
594      * registered with this client, the page is <b>not</b> loaded into the specified window; in this case,
595      * the page is loaded into a new window, and attachment handling is delegated to the registered
596      * <code>AttachmentHandler</code>.</p>
597      *
598      * @param webResponse the response that will be used to create the new page
599      * @param webWindow the window that the new page will be placed within
600      * @param forceAttachmentWithFilename if not {@code null}, handle this as an attachment with the specified name
601      *        or if an empty string ("") use the filename provided in the response
602      * @throws IOException if an IO error occurs
603      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
604      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to true
605      * @return the newly created page
606      * @see #setAttachmentHandler(AttachmentHandler)
607      */
608     public Page loadWebResponseInto(final WebResponse webResponse, final WebWindow webWindow,
609             String forceAttachmentWithFilename)
610             throws IOException, FailingHttpStatusCodeException {
611         WebAssert.notNull("webResponse", webResponse);
612         WebAssert.notNull("webWindow", webWindow);
613 
614         if (webResponse.getStatusCode() == HttpStatus.NO_CONTENT_204) {
615             return webWindow.getEnclosedPage();
616         }
617 
618         if (webStartHandler_ != null && "application/x-java-jnlp-file".equals(webResponse.getContentType())) {
619             webStartHandler_.handleJnlpResponse(webResponse);
620             return webWindow.getEnclosedPage();
621         }
622 
623         if (attachmentHandler_ != null
624                 && (forceAttachmentWithFilename != null || attachmentHandler_.isAttachment(webResponse))) {
625 
626             // check content disposition header for nothing provided
627             if (StringUtils.isEmpty(forceAttachmentWithFilename)) {
628                 final String disp = webResponse.getResponseHeaderValue(HttpHeader.CONTENT_DISPOSITION);
629                 forceAttachmentWithFilename = Attachment.getSuggestedFilename(disp);
630             }
631 
632             if (attachmentHandler_.handleAttachment(webResponse,
633                         StringUtils.isEmpty(forceAttachmentWithFilename) ? null : forceAttachmentWithFilename)) {
634                 // the handling is done by the attachment handler;
635                 // do not open a new window
636                 return webWindow.getEnclosedPage();
637             }
638 
639             final WebWindow w = openWindow(null, null, webWindow);
640             final Page page = pageCreator_.createPage(webResponse, w);
641             attachmentHandler_.handleAttachment(page,
642                                 StringUtils.isEmpty(forceAttachmentWithFilename) ? null : forceAttachmentWithFilename);
643             return page;
644         }
645 
646         final Page oldPage = webWindow.getEnclosedPage();
647         if (oldPage != null) {
648             // Remove the old page before create new one.
649             oldPage.cleanUp();
650         }
651 
652         Page newPage = null;
653         FrameWindow.PageDenied pageDenied = PageDenied.NONE;
654         if (windows_.contains(webWindow)) {
655             if (webWindow instanceof FrameWindow) {
656                 final String contentSecurityPolicy =
657                         webResponse.getResponseHeaderValue(HttpHeader.CONTENT_SECURIRY_POLICY);
658                 if (StringUtils.isNotBlank(contentSecurityPolicy)) {
659                     final URL origin = UrlUtils.getUrlWithoutPathRefQuery(
660                             ((FrameWindow) webWindow).getEnclosingPage().getUrl());
661                     final URL source = UrlUtils.getUrlWithoutPathRefQuery(webResponse.getWebRequest().getUrl());
662                     final Policy policy = Policy.parseSerializedCSP(contentSecurityPolicy,
663                                                     Policy.PolicyErrorConsumer.ignored);
664                     if (!policy.allowsFrameAncestor(
665                             Optional.of(URI.parseURI(source.toExternalForm()).orElse(null)),
666                             Optional.of(URI.parseURI(origin.toExternalForm()).orElse(null)))) {
667                         pageDenied = PageDenied.BY_CONTENT_SECURIRY_POLICY;
668 
669                         if (LOG.isWarnEnabled()) {
670                             LOG.warn("Load denied by Content-Security-Policy: '" + contentSecurityPolicy + "' - "
671                                     + webResponse.getWebRequest().getUrl() + "' does not permit framing.");
672                         }
673                     }
674                 }
675 
676                 if (pageDenied == PageDenied.NONE) {
677                     final String xFrameOptions = webResponse.getResponseHeaderValue(HttpHeader.X_FRAME_OPTIONS);
678                     if ("DENY".equalsIgnoreCase(xFrameOptions)) {
679                         pageDenied = PageDenied.BY_X_FRAME_OPTIONS;
680 
681                         if (LOG.isWarnEnabled()) {
682                             LOG.warn("Load denied by X-Frame-Options: DENY; - '"
683                                     + webResponse.getWebRequest().getUrl() + "' does not permit framing.");
684                         }
685                     }
686                 }
687             }
688 
689             if (pageDenied == PageDenied.NONE) {
690                 newPage = pageCreator_.createPage(webResponse, webWindow);
691             }
692             else {
693                 try {
694                     final WebResponse aboutBlank = loadWebResponse(WebRequest.newAboutBlankRequest());
695                     newPage = pageCreator_.createPage(aboutBlank, webWindow);
696                     // TODO - maybe we have to attach to original request/response to the page
697 
698                     ((FrameWindow) webWindow).setPageDenied(pageDenied);
699                 }
700                 catch (final IOException ignored) {
701                     // ignore
702                 }
703             }
704 
705             if (windows_.contains(webWindow)) {
706                 fireWindowContentChanged(new WebWindowEvent(webWindow, WebWindowEvent.CHANGE, oldPage, newPage));
707 
708                 // The page being loaded may already have been replaced by another page via JavaScript code.
709                 if (webWindow.getEnclosedPage() == newPage) {
710                     newPage.initialize();
711                     // hack: onload should be fired the same way for all type of pages
712                     // here is a hack to handle non HTML pages
713                     if (isJavaScriptEnabled()
714                             && webWindow instanceof FrameWindow && !newPage.isHtmlPage()) {
715                         final FrameWindow fw = (FrameWindow) webWindow;
716                         final BaseFrameElement frame = fw.getFrameElement();
717                         if (frame.hasEventHandlers("onload")) {
718                             if (LOG.isDebugEnabled()) {
719                                 LOG.debug("Executing onload handler for " + frame);
720                             }
721                             final Event event = new Event(frame, Event.TYPE_LOAD);
722                             ((Node) frame.getScriptableObject()).executeEventLocally(event);
723                         }
724                     }
725                 }
726             }
727         }
728         return newPage;
729     }
730 
731     /**
732      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span>
733      *
734      * <p>Logs the response's content if its status code indicates a request failure and
735      * {@link WebClientOptions#isPrintContentOnFailingStatusCode()} returns {@code true}.
736      *
737      * @param webResponse the response whose content may be logged
738      */
739     public void printContentIfNecessary(final WebResponse webResponse) {
740         if (getOptions().isPrintContentOnFailingStatusCode()
741                 && !webResponse.isSuccess() && LOG.isInfoEnabled()) {
742             final String contentType = webResponse.getContentType();
743             LOG.info("statusCode=[" + webResponse.getStatusCode() + "] contentType=[" + contentType + "]");
744             LOG.info(webResponse.getContentAsString());
745         }
746     }
747 
748     /**
749      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span>
750      *
751      * <p>Throws a {@link FailingHttpStatusCodeException} if the request's status code indicates a request
752      * failure and {@link WebClientOptions#isThrowExceptionOnFailingStatusCode()} returns {@code true}.
753      *
754      * @param webResponse the response which may trigger a {@link FailingHttpStatusCodeException}
755      */
756     public void throwFailingHttpStatusCodeExceptionIfNecessary(final WebResponse webResponse) {
757         if (getOptions().isThrowExceptionOnFailingStatusCode() && !webResponse.isSuccessOrUseProxyOrNotModified()) {
758             throw new FailingHttpStatusCodeException(webResponse);
759         }
760     }
761 
762     /**
763      * Adds a header which will be sent with EVERY request from this client.
764      * This list is empty per default; use this to add specific headers for your
765      * case.
766      * @param name the name of the header to add
767      * @param value the value of the header to add
768      * @see #removeRequestHeader(String)
769      */
770     public void addRequestHeader(final String name, final String value) {
771         if (HttpHeader.COOKIE_LC.equalsIgnoreCase(name)) {
772             throw new IllegalArgumentException("Do not add 'Cookie' header, use .getCookieManager() instead");
773         }
774         requestHeaders_.put(name, value);
775     }
776 
777     /**
778      * Removes a header from being sent with EVERY request from this client.
779      * This list is empty per default; use this method to remove specific headers
780      * your have added using {{@link #addRequestHeader(String, String)} before.<br>
781      * You can't use this to avoid sending standard headers like "Accept-Language"
782      * or "Sec-Fetch-Dest".
783      * @param name the name of the header to remove
784      * @see #addRequestHeader
785      */
786     public void removeRequestHeader(final String name) {
787         requestHeaders_.remove(name);
788     }
789 
790     /**
791      * Sets the credentials provider that will provide authentication information when
792      * trying to access protected information on a web server. This information is
793      * required when the server is using Basic HTTP authentication, NTLM authentication,
794      * or Digest authentication.
795      * @param credentialsProvider the new credentials provider to use to authenticate
796      */
797     public void setCredentialsProvider(final CredentialsProvider credentialsProvider) {
798         WebAssert.notNull("credentialsProvider", credentialsProvider);
799         credentialsProvider_ = credentialsProvider;
800     }
801 
802     /**
803      * Returns the credentials provider for this client instance. By default, this
804      * method returns an instance of {@link DefaultCredentialsProvider}.
805      * @return the credentials provider for this client instance
806      */
807     public CredentialsProvider getCredentialsProvider() {
808         return credentialsProvider_;
809     }
810 
811     /**
812      * This method is intended for testing only - use at your own risk.
813      * @return the current JavaScript engine (never {@code null})
814      */
815     public AbstractJavaScriptEngine<?> getJavaScriptEngine() {
816         return scriptEngine_;
817     }
818 
819     /**
820      * This method is intended for testing only - use at your own risk.
821      *
822      * @param engine the new script engine to use
823      */
824     public void setJavaScriptEngine(final AbstractJavaScriptEngine<?> engine) {
825         if (engine == null) {
826             throw new IllegalArgumentException("Can't set JavaScriptEngine to null");
827         }
828         scriptEngine_ = engine;
829     }
830 
831     /**
832      * Returns the cookie manager used by this web client.
833      * @return the cookie manager used by this web client
834      */
835     public CookieManager getCookieManager() {
836         return cookieManager_;
837     }
838 
839     /**
840      * Sets the cookie manager used by this web client.
841      * @param cookieManager the cookie manager used by this web client
842      */
843     public void setCookieManager(final CookieManager cookieManager) {
844         WebAssert.notNull("cookieManager", cookieManager);
845         cookieManager_ = cookieManager;
846     }
847 
848     /**
849      * Sets the alert handler for this webclient.
850      * @param alertHandler the new alerthandler or null if none is specified
851      */
852     public void setAlertHandler(final AlertHandler alertHandler) {
853         alertHandler_ = alertHandler;
854     }
855 
856     /**
857      * Returns the alert handler for this webclient.
858      * @return the alert handler or null if one hasn't been set
859      */
860     public AlertHandler getAlertHandler() {
861         return alertHandler_;
862     }
863 
864     /**
865      * Sets the handler that will be executed when the JavaScript method Window.confirm() is called.
866      * @param handler the new handler or null if no handler is to be used
867      */
868     public void setConfirmHandler(final ConfirmHandler handler) {
869         confirmHandler_ = handler;
870     }
871 
872     /**
873      * Returns the confirm handler.
874      * @return the confirm handler or null if one hasn't been set
875      */
876     public ConfirmHandler getConfirmHandler() {
877         return confirmHandler_;
878     }
879 
880     /**
881      * Sets the handler that will be executed when the JavaScript method Window.prompt() is called.
882      * @param handler the new handler or null if no handler is to be used
883      */
884     public void setPromptHandler(final PromptHandler handler) {
885         promptHandler_ = handler;
886     }
887 
888     /**
889      * Returns the prompt handler.
890      * @return the prompt handler or null if one hasn't been set
891      */
892     public PromptHandler getPromptHandler() {
893         return promptHandler_;
894     }
895 
896     /**
897      * Sets the status handler for this webclient.
898      * @param statusHandler the new status handler or null if none is specified
899      */
900     public void setStatusHandler(final StatusHandler statusHandler) {
901         statusHandler_ = statusHandler;
902     }
903 
904     /**
905      * Returns the status handler for this {@link WebClient}.
906      * @return the status handler or null if one hasn't been set
907      */
908     public StatusHandler getStatusHandler() {
909         return statusHandler_;
910     }
911 
912     /**
913      * Returns the executor for this {@link WebClient}.
914      * @return the executor
915      */
916     public synchronized Executor getExecutor() {
917         if (executor_ == null) {
918             final ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) Executors.newCachedThreadPool();
919             threadPoolExecutor.setThreadFactory(new ThreadNamingFactory(threadPoolExecutor.getThreadFactory()));
920             // threadPoolExecutor.prestartAllCoreThreads();
921             executor_ = threadPoolExecutor;
922         }
923 
924         return executor_;
925     }
926 
927     /**
928      * Changes the ExecutorService for this {@link WebClient}.
929      * You have to call this before the first use of the executor, otherwise
930      * an IllegalStateExceptions is thrown.
931      * @param executor the new Executor.
932      */
933     public synchronized void setExecutor(final ExecutorService executor) {
934         if (executor_ != null) {
935             throw new IllegalStateException("Can't change the executor after first use.");
936         }
937 
938         executor_ = executor;
939     }
940 
941     /**
942      * Sets the javascript error listener for this {@link WebClient}.
943      * When setting to null, the {@link DefaultJavaScriptErrorListener} is used.
944      * @param javaScriptErrorListener the new JavaScriptErrorListener or null if none is specified
945      */
946     public void setJavaScriptErrorListener(final JavaScriptErrorListener javaScriptErrorListener) {
947         if (javaScriptErrorListener == null) {
948             javaScriptErrorListener_ = new DefaultJavaScriptErrorListener();
949         }
950         else {
951             javaScriptErrorListener_ = javaScriptErrorListener;
952         }
953     }
954 
955     /**
956      * Returns the javascript error listener for this {@link WebClient}.
957      * @return the javascript error listener or null if one hasn't been set
958      */
959     public JavaScriptErrorListener getJavaScriptErrorListener() {
960         return javaScriptErrorListener_;
961     }
962 
963     /**
964      * Returns the current browser version.
965      * @return the current browser version
966      */
967     public BrowserVersion getBrowserVersion() {
968         return browserVersion_;
969     }
970 
971     /**
972      * Returns the "current" window for this client. This window (or its top window) will be used
973      * when <code>getPage(...)</code> is called without specifying a window.
974      * @return the "current" window for this client
975      */
976     public WebWindow getCurrentWindow() {
977         return currentWindow_;
978     }
979 
980     /**
981      * Sets the "current" window for this client. This is the window that will be used when
982      * <code>getPage(...)</code> is called without specifying a window.
983      * @param window the new "current" window for this client
984      */
985     public void setCurrentWindow(final WebWindow window) {
986         WebAssert.notNull("window", window);
987         if (currentWindow_ == window) {
988             return;
989         }
990         // onBlur event is triggered for focused element of old current window
991         if (currentWindow_ != null && !currentWindow_.isClosed()) {
992             final Page enclosedPage = currentWindow_.getEnclosedPage();
993             if (enclosedPage != null && enclosedPage.isHtmlPage()) {
994                 final DomElement focusedElement = ((HtmlPage) enclosedPage).getFocusedElement();
995                 if (focusedElement != null) {
996                     focusedElement.fireEvent(Event.TYPE_BLUR);
997                 }
998             }
999         }
1000         currentWindow_ = window;
1001 
1002         // when marking an iframe window as current we have no need to move the focus
1003         final boolean isIFrame = currentWindow_ instanceof FrameWindow
1004                 && ((FrameWindow) currentWindow_).getFrameElement() instanceof HtmlInlineFrame;
1005         if (!isIFrame) {
1006             //1. activeElement becomes focused element for new current window
1007             //2. onFocus event is triggered for focusedElement of new current window
1008             final Page enclosedPage = currentWindow_.getEnclosedPage();
1009             if (enclosedPage != null && enclosedPage.isHtmlPage()) {
1010                 final HtmlPage enclosedHtmlPage = (HtmlPage) enclosedPage;
1011                 final HtmlElement activeElement = enclosedHtmlPage.getActiveElement();
1012                 if (activeElement != null) {
1013                     enclosedHtmlPage.setFocusedElement(activeElement, true);
1014                 }
1015             }
1016         }
1017     }
1018 
1019     /**
1020      * Adds a listener for {@link WebWindowEvent}s. All events from all windows associated with this
1021      * client will be sent to the specified listener.
1022      * @param listener a listener
1023      */
1024     public void addWebWindowListener(final WebWindowListener listener) {
1025         WebAssert.notNull("listener", listener);
1026         webWindowListeners_.add(listener);
1027     }
1028 
1029     /**
1030      * Removes a listener for {@link WebWindowEvent}s.
1031      * @param listener a listener
1032      */
1033     public void removeWebWindowListener(final WebWindowListener listener) {
1034         WebAssert.notNull("listener", listener);
1035         webWindowListeners_.remove(listener);
1036     }
1037 
1038     private void fireWindowContentChanged(final WebWindowEvent event) {
1039         if (currentWindowTracker_ != null) {
1040             currentWindowTracker_.webWindowContentChanged(event);
1041         }
1042         for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
1043             listener.webWindowContentChanged(event);
1044         }
1045     }
1046 
1047     private void fireWindowOpened(final WebWindowEvent event) {
1048         if (currentWindowTracker_ != null) {
1049             currentWindowTracker_.webWindowOpened(event);
1050         }
1051         for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
1052             listener.webWindowOpened(event);
1053         }
1054     }
1055 
1056     private void fireWindowClosed(final WebWindowEvent event) {
1057         if (currentWindowTracker_ != null) {
1058             currentWindowTracker_.webWindowClosed(event);
1059         }
1060 
1061         for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
1062             listener.webWindowClosed(event);
1063         }
1064 
1065         // to open a new top level window if all others are gone
1066         if (currentWindowTracker_ != null) {
1067             currentWindowTracker_.afterWebWindowClosedListenersProcessed(event);
1068         }
1069     }
1070 
1071     /**
1072      * Open a new window with the specified name. If the URL is non-null then attempt to load
1073      * a page from that location and put it in the new window.
1074      *
1075      * @param url the URL to load content from or null if no content is to be loaded
1076      * @param windowName the name of the new window
1077      * @return the new window
1078      */
1079     public WebWindow openWindow(final URL url, final String windowName) {
1080         WebAssert.notNull("windowName", windowName);
1081         return openWindow(url, windowName, getCurrentWindow());
1082     }
1083 
1084     /**
1085      * Open a new window with the specified name. If the URL is non-null then attempt to load
1086      * a page from that location and put it in the new window.
1087      *
1088      * @param url the URL to load content from or null if no content is to be loaded
1089      * @param windowName the name of the new window
1090      * @param opener the web window that is calling openWindow
1091      * @return the new window
1092      */
1093     public WebWindow openWindow(final URL url, final String windowName, final WebWindow opener) {
1094         final WebWindow window = openTargetWindow(opener, windowName, TARGET_BLANK);
1095         if (url == null) {
1096             initializeEmptyWindow(window, window.getEnclosedPage());
1097         }
1098         else {
1099             try {
1100                 final WebRequest request = new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader(),
1101                                                                 getBrowserVersion().getAcceptEncodingHeader());
1102                 request.setCharset(UTF_8);
1103 
1104                 final Page openerPage = opener.getEnclosedPage();
1105                 if (openerPage != null && openerPage.getUrl() != null) {
1106                     request.setRefererHeader(openerPage.getUrl());
1107                 }
1108                 getPage(window, request);
1109             }
1110             catch (final IOException e) {
1111                 LOG.error("Error loading content into window", e);
1112             }
1113         }
1114         return window;
1115     }
1116 
1117     /**
1118      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1119      *
1120      * Open the window with the specified name. The name may be a special
1121      * target name of _self, _parent, _top, or _blank. An empty or null
1122      * name is set to the default. The special target names are relative to
1123      * the opener window.
1124      *
1125      * @param opener the web window that is calling openWindow
1126      * @param windowName the name of the new window
1127      * @param defaultName the default target if no name is given
1128      * @return the new window
1129      */
1130     public WebWindow openTargetWindow(
1131             final WebWindow opener, final String windowName, final String defaultName) {
1132 
1133         WebAssert.notNull("opener", opener);
1134         WebAssert.notNull("defaultName", defaultName);
1135 
1136         String windowToOpen = windowName;
1137         if (windowToOpen == null || windowToOpen.isEmpty()) {
1138             windowToOpen = defaultName;
1139         }
1140 
1141         WebWindow webWindow = resolveWindow(opener, windowToOpen);
1142 
1143         if (webWindow == null) {
1144             if (TARGET_BLANK.equals(windowToOpen)) {
1145                 windowToOpen = "";
1146             }
1147             webWindow = new TopLevelWindow(windowToOpen, this);
1148         }
1149 
1150         if (webWindow instanceof TopLevelWindow && webWindow != opener.getTopWindow()) {
1151             ((TopLevelWindow) webWindow).setOpener(opener);
1152         }
1153 
1154         return webWindow;
1155     }
1156 
1157     private WebWindow resolveWindow(final WebWindow opener, final String name) {
1158         if (name == null || name.isEmpty() || TARGET_SELF.equals(name)) {
1159             return opener;
1160         }
1161 
1162         if (TARGET_PARENT.equals(name)) {
1163             return opener.getParentWindow();
1164         }
1165 
1166         if (TARGET_TOP.equals(name)) {
1167             return opener.getTopWindow();
1168         }
1169 
1170         if (TARGET_BLANK.equals(name)) {
1171             return null;
1172         }
1173 
1174         // first search for frame windows inside our window hierarchy
1175         WebWindow window = opener;
1176         while (true) {
1177             final Page page = window.getEnclosedPage();
1178             if (page != null && page.isHtmlPage()) {
1179                 try {
1180                     final FrameWindow frame = ((HtmlPage) page).getFrameByName(name);
1181                     final HtmlUnitScriptable scriptable = frame.getFrameElement().getScriptableObject();
1182                     if (scriptable instanceof HTMLIFrameElement) {
1183                         ((HTMLIFrameElement) scriptable).onRefresh();
1184                     }
1185                     return frame;
1186                 }
1187                 catch (final ElementNotFoundException expected) {
1188                     // Fall through
1189                 }
1190             }
1191 
1192             if (window == window.getParentWindow()) {
1193                 // TODO: should getParentWindow() return null on top windows?
1194                 break;
1195             }
1196             window = window.getParentWindow();
1197         }
1198 
1199         try {
1200             return getWebWindowByName(name);
1201         }
1202         catch (final WebWindowNotFoundException expected) {
1203             // Fall through - a new window will be created below
1204         }
1205         return null;
1206     }
1207 
1208     /**
1209      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
1210      *
1211      * Opens a new dialog window.
1212      * @param url the URL of the document to load and display
1213      * @param opener the web window that is opening the dialog
1214      * @param dialogArguments the object to make available inside the dialog via <code>window.dialogArguments</code>
1215      * @return the new dialog window
1216      * @throws IOException if there is an IO error
1217      */
1218     public DialogWindow openDialogWindow(final URL url, final WebWindow opener, final Object dialogArguments)
1219         throws IOException {
1220 
1221         WebAssert.notNull("url", url);
1222         WebAssert.notNull("opener", opener);
1223 
1224         final DialogWindow window = new DialogWindow(this, dialogArguments);
1225 
1226         final HtmlPage openerPage = (HtmlPage) opener.getEnclosedPage();
1227         final WebRequest request = new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader(),
1228                                                         getBrowserVersion().getAcceptEncodingHeader());
1229         request.setCharset(UTF_8);
1230 
1231         if (openerPage != null) {
1232             request.setRefererHeader(openerPage.getUrl());
1233         }
1234 
1235         getPage(window, request);
1236 
1237         return window;
1238     }
1239 
1240     /**
1241      * Sets the object that will be used to create pages. Set this if you want
1242      * to customize the type of page that is returned for a given content type.
1243      *
1244      * @param pageCreator the new page creator
1245      */
1246     public void setPageCreator(final PageCreator pageCreator) {
1247         WebAssert.notNull("pageCreator", pageCreator);
1248         pageCreator_ = pageCreator;
1249     }
1250 
1251     /**
1252      * Returns the current page creator.
1253      *
1254      * @return the page creator
1255      */
1256     public PageCreator getPageCreator() {
1257         return pageCreator_;
1258     }
1259 
1260     /**
1261      * Returns the first {@link WebWindow} that matches the specified name.
1262      *
1263      * @param name the name to search for
1264      * @return the {@link WebWindow} with the specified name
1265      * @throws WebWindowNotFoundException if the {@link WebWindow} can't be found
1266      * @see #getWebWindows()
1267      * @see #getTopLevelWindows()
1268      */
1269     public WebWindow getWebWindowByName(final String name) throws WebWindowNotFoundException {
1270         WebAssert.notNull("name", name);
1271 
1272         for (final WebWindow webWindow : windows_) {
1273             if (name.equals(webWindow.getName())) {
1274                 return webWindow;
1275             }
1276         }
1277 
1278         throw new WebWindowNotFoundException(name);
1279     }
1280 
1281     /**
1282      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1283      *
1284      * Initializes a new web window for JavaScript.
1285      * @param webWindow the new WebWindow
1286      * @param page the page that will become the enclosing page
1287      */
1288     public void initialize(final WebWindow webWindow, final Page page) {
1289         WebAssert.notNull("webWindow", webWindow);
1290 
1291         if (isJavaScriptEngineEnabled()) {
1292             scriptEngine_.initialize(webWindow, page);
1293         }
1294     }
1295 
1296     /**
1297      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1298      *
1299      * Initializes a new empty window for JavaScript.
1300      *
1301      * @param webWindow the new WebWindow
1302      * @param page the page that will become the enclosing page
1303      */
1304     public void initializeEmptyWindow(final WebWindow webWindow, final Page page) {
1305         WebAssert.notNull("webWindow", webWindow);
1306 
1307         if (isJavaScriptEngineEnabled()) {
1308             initialize(webWindow, page);
1309             ((Window) webWindow.getScriptableObject()).initialize();
1310         }
1311     }
1312 
1313     /**
1314      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1315      *
1316      * Adds a new window to the list of available windows.
1317      *
1318      * @param webWindow the new WebWindow
1319      */
1320     public void registerWebWindow(final WebWindow webWindow) {
1321         WebAssert.notNull("webWindow", webWindow);
1322         if (windows_.add(webWindow)) {
1323             fireWindowOpened(new WebWindowEvent(webWindow, WebWindowEvent.OPEN, webWindow.getEnclosedPage(), null));
1324         }
1325         // register JobManager here but don't deregister in deregisterWebWindow as it can live longer
1326         jobManagers_.add(new WeakReference<>(webWindow.getJobManager()));
1327     }
1328 
1329     /**
1330      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1331      *
1332      * Removes a window from the list of available windows.
1333      *
1334      * @param webWindow the window to remove
1335      */
1336     public void deregisterWebWindow(final WebWindow webWindow) {
1337         WebAssert.notNull("webWindow", webWindow);
1338         if (windows_.remove(webWindow)) {
1339             fireWindowClosed(new WebWindowEvent(webWindow, WebWindowEvent.CLOSE, webWindow.getEnclosedPage(), null));
1340         }
1341     }
1342 
1343     /**
1344      * Expands a relative URL relative to the specified base. In most situations
1345      * this is the same as <code>new URL(baseUrl, relativeUrl)</code> but
1346      * there are some cases that URL doesn't handle correctly. See
1347      * <a href="http://www.faqs.org/rfcs/rfc1808.html">RFC1808</a>
1348      * regarding Relative Uniform Resource Locators for more information.
1349      *
1350      * @param baseUrl the base URL
1351      * @param relativeUrl the relative URL
1352      * @return the expansion of the specified base and relative URLs
1353      * @throws MalformedURLException if an error occurred when creating a URL object
1354      */
1355     public static URL expandUrl(final URL baseUrl, final String relativeUrl) throws MalformedURLException {
1356         final String newUrl = UrlUtils.resolveUrl(baseUrl, relativeUrl);
1357         return UrlUtils.toUrlUnsafe(newUrl);
1358     }
1359 
1360     private WebResponse makeWebResponseForDataUrl(final WebRequest webRequest) throws IOException {
1361         final URL url = webRequest.getUrl();
1362         final DataURLConnection connection;
1363         connection = new DataURLConnection(url);
1364 
1365         final List<NameValuePair> responseHeaders = new ArrayList<>();
1366         responseHeaders.add(new NameValuePair(HttpHeader.CONTENT_TYPE_LC,
1367             connection.getMediaType() + ";charset=" + connection.getCharset()));
1368 
1369         try (InputStream is = connection.getInputStream()) {
1370             final DownloadedContent downloadedContent =
1371                     HttpWebConnection.downloadContent(is,
1372                             getOptions().getMaxInMemory(),
1373                             getOptions().getTempFileDirectory());
1374             final WebResponseData data = new WebResponseData(downloadedContent, 200, "OK", responseHeaders);
1375             return new WebResponse(data, url, webRequest.getHttpMethod(), 0);
1376         }
1377     }
1378 
1379     private static WebResponse makeWebResponseForAboutUrl(final WebRequest webRequest) throws MalformedURLException {
1380         final URL url = webRequest.getUrl();
1381         final String urlString = url.toExternalForm();
1382         if (UrlUtils.ABOUT_BLANK.equalsIgnoreCase(urlString)) {
1383             return new StringWebResponse("", UrlUtils.URL_ABOUT_BLANK);
1384         }
1385 
1386         final String urlWithoutQuery = StringUtils.substringBefore(urlString, "?");
1387         if (!"blank".equalsIgnoreCase(StringUtils.substringAfter(urlWithoutQuery, UrlUtils.ABOUT_SCHEME))) {
1388             throw new MalformedURLException(url + " is not supported, only about:blank is supported at the moment.");
1389         }
1390         return new StringWebResponse("", url);
1391     }
1392 
1393     /**
1394      * Builds a WebResponse for a file URL.
1395      * This first implementation is basic.
1396      * It assumes that the file contains an HTML page encoded with the specified encoding.
1397      * @param webRequest the request
1398      * @return the web response
1399      * @throws IOException if an IO problem occurs
1400      */
1401     private WebResponse makeWebResponseForFileUrl(final WebRequest webRequest) throws IOException {
1402         URL cleanUrl = webRequest.getUrl();
1403         if (cleanUrl.getQuery() != null) {
1404             // Get rid of the query portion before trying to load the file.
1405             cleanUrl = UrlUtils.getUrlWithNewQuery(cleanUrl, null);
1406         }
1407         if (cleanUrl.getRef() != null) {
1408             // Get rid of the ref portion before trying to load the file.
1409             cleanUrl = UrlUtils.getUrlWithNewRef(cleanUrl, null);
1410         }
1411 
1412         final WebResponse fromCache = getCache().getCachedResponse(webRequest);
1413         if (fromCache != null) {
1414             return new WebResponseFromCache(fromCache, webRequest);
1415         }
1416 
1417         String fileUrl = cleanUrl.toExternalForm();
1418         fileUrl = URLDecoder.decode(fileUrl, UTF_8.name());
1419         final File file = new File(fileUrl.substring(5));
1420         if (!file.exists()) {
1421             // construct 404
1422             final List<NameValuePair> compiledHeaders = new ArrayList<>();
1423             compiledHeaders.add(new NameValuePair(HttpHeader.CONTENT_TYPE, MimeType.TEXT_HTML));
1424             final WebResponseData responseData =
1425                 new WebResponseData(
1426                         org.htmlunit.util.StringUtils
1427                             .toByteArray("File: " + file.getAbsolutePath(), UTF_8),
1428                     404, "Not Found", compiledHeaders);
1429             return new WebResponse(responseData, webRequest, 0);
1430         }
1431 
1432         final String contentType = guessContentType(file);
1433 
1434         final DownloadedContent content = new DownloadedContent.OnFile(file, false);
1435         final List<NameValuePair> compiledHeaders = new ArrayList<>();
1436         compiledHeaders.add(new NameValuePair(HttpHeader.CONTENT_TYPE, contentType));
1437         compiledHeaders.add(new NameValuePair(HttpHeader.LAST_MODIFIED,
1438                 HttpUtils.formatDate(new Date(file.lastModified()))));
1439         final WebResponseData responseData = new WebResponseData(content, 200, "OK", compiledHeaders);
1440         final WebResponse webResponse = new WebResponse(responseData, webRequest, 0);
1441         getCache().cacheIfPossible(webRequest, webResponse, null);
1442         return webResponse;
1443     }
1444 
1445     private WebResponse makeWebResponseForBlobUrl(final WebRequest webRequest) {
1446         final Window window = getCurrentWindow().getScriptableObject();
1447         final Blob fileOrBlob = window.getDocument().resolveBlobUrl(webRequest.getUrl().toString());
1448         if (fileOrBlob == null) {
1449             throw JavaScriptEngine.typeError("Cannot load data from " + webRequest.getUrl());
1450         }
1451 
1452         final List<NameValuePair> headers = new ArrayList<>();
1453         final String type = fileOrBlob.getType();
1454         if (!StringUtils.isEmpty(type)) {
1455             headers.add(new NameValuePair(HttpHeader.CONTENT_TYPE, fileOrBlob.getType()));
1456         }
1457         if (fileOrBlob instanceof org.htmlunit.javascript.host.file.File) {
1458             final org.htmlunit.javascript.host.file.File file = (org.htmlunit.javascript.host.file.File) fileOrBlob;
1459             final String fileName = file.getName();
1460             if (!StringUtils.isEmpty(fileName)) {
1461                 // https://datatracker.ietf.org/doc/html/rfc6266#autoid-10
1462                 headers.add(new NameValuePair(HttpHeader.CONTENT_DISPOSITION, "inline; filename=\"" + fileName + "\""));
1463             }
1464         }
1465 
1466         final DownloadedContent content = new DownloadedContent.InMemory(fileOrBlob.getBytes());
1467         final WebResponseData responseData = new WebResponseData(content, 200, "OK", headers);
1468         return new WebResponse(responseData, webRequest, 0);
1469     }
1470 
1471     /**
1472      * Tries to guess the content type of the file.<br>
1473      * This utility could be located in a helper class but we can compare this functionality
1474      * for instance with the "Helper Applications" settings of Mozilla and therefore see it as a
1475      * property of the "browser".
1476      * @param file the file
1477      * @return "application/octet-stream" if nothing could be guessed
1478      */
1479     public String guessContentType(final File file) {
1480         final String fileName = file.getName();
1481         final String fileNameLC = fileName.toLowerCase(Locale.ROOT);
1482         if (fileNameLC.endsWith(".xhtml")) {
1483             // Java's mime type map returns application/xml in JDK8.
1484             return MimeType.APPLICATION_XHTML;
1485         }
1486 
1487         // Java's mime type map does not know these in JDK8.
1488         if (fileNameLC.endsWith(".js")) {
1489             return MimeType.TEXT_JAVASCRIPT;
1490         }
1491 
1492         if (fileNameLC.endsWith(".css")) {
1493             return MimeType.TEXT_CSS;
1494         }
1495 
1496         String contentType = null;
1497         if (!fileNameLC.endsWith(".php")) {
1498             contentType = URLConnection.guessContentTypeFromName(fileName);
1499         }
1500         if (contentType == null) {
1501             try (InputStream inputStream = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
1502                 contentType = URLConnection.guessContentTypeFromStream(inputStream);
1503             }
1504             catch (final IOException ignored) {
1505                 // Ignore silently.
1506             }
1507         }
1508         if (contentType == null) {
1509             contentType = MimeType.APPLICATION_OCTET_STREAM;
1510         }
1511         return contentType;
1512     }
1513 
1514     private WebResponse makeWebResponseForJavaScriptUrl(final WebWindow webWindow, final URL url,
1515         final Charset charset) throws FailingHttpStatusCodeException, IOException {
1516 
1517         HtmlPage page = null;
1518         if (webWindow instanceof FrameWindow) {
1519             final FrameWindow frameWindow = (FrameWindow) webWindow;
1520             page = (HtmlPage) frameWindow.getEnclosedPage();
1521         }
1522         else {
1523             final Page currentPage = webWindow.getEnclosedPage();
1524             if (currentPage instanceof HtmlPage) {
1525                 page = (HtmlPage) currentPage;
1526             }
1527         }
1528 
1529         if (page == null) {
1530             page = getPage(webWindow, WebRequest.newAboutBlankRequest());
1531         }
1532         final ScriptResult r = page.executeJavaScript(url.toExternalForm(), "JavaScript URL", 1);
1533         if (r.getJavaScriptResult() == null || ScriptResult.isUndefined(r)) {
1534             // No new WebResponse to produce.
1535             return webWindow.getEnclosedPage().getWebResponse();
1536         }
1537 
1538         final String contentString = r.getJavaScriptResult().toString();
1539         final StringWebResponse response = new StringWebResponse(contentString, charset, url);
1540         response.setFromJavascript(true);
1541         return response;
1542     }
1543 
1544     /**
1545      * Loads a {@link WebResponse} from the server.
1546      * @param webRequest the request
1547      * @throws IOException if an IO problem occurs
1548      * @return the WebResponse
1549      */
1550     public WebResponse loadWebResponse(final WebRequest webRequest) throws IOException {
1551         final String protocol = webRequest.getUrl().getProtocol();
1552         switch (protocol) {
1553             case UrlUtils.ABOUT:
1554                 return makeWebResponseForAboutUrl(webRequest);
1555 
1556             case "file":
1557                 return makeWebResponseForFileUrl(webRequest);
1558 
1559             case "data":
1560                 return makeWebResponseForDataUrl(webRequest);
1561 
1562             case "blob":
1563                 return makeWebResponseForBlobUrl(webRequest);
1564 
1565             case "http":
1566             case "https":
1567                 return loadWebResponseFromWebConnection(webRequest, ALLOWED_REDIRECTIONS_SAME_URL);
1568 
1569             default:
1570                 throw new IOException("Unsupported protocol '" + protocol + "'");
1571         }
1572     }
1573 
1574     /**
1575      * Loads a {@link WebResponse} from the server through the WebConnection.
1576      * @param webRequest the request
1577      * @param allowedRedirects the number of allowed redirects remaining
1578      * @throws IOException if an IO problem occurs
1579      * @return the resultant {@link WebResponse}
1580      */
1581     private WebResponse loadWebResponseFromWebConnection(final WebRequest webRequest,
1582         final int allowedRedirects) throws IOException {
1583 
1584         URL url = webRequest.getUrl();
1585         final HttpMethod method = webRequest.getHttpMethod();
1586         final List<NameValuePair> parameters = webRequest.getRequestParameters();
1587 
1588         WebAssert.notNull("url", url);
1589         WebAssert.notNull("method", method);
1590         WebAssert.notNull("parameters", parameters);
1591 
1592         url = UrlUtils.encodeUrl(url, webRequest.getCharset());
1593         webRequest.setUrl(url);
1594 
1595         if (LOG.isDebugEnabled()) {
1596             LOG.debug("Load response for " + method + " " + url.toExternalForm());
1597         }
1598 
1599         // If the request settings don't specify a custom proxy, use the default client proxy...
1600         if (webRequest.getProxyHost() == null) {
1601             final ProxyConfig proxyConfig = getOptions().getProxyConfig();
1602             if (proxyConfig.getProxyAutoConfigUrl() != null) {
1603                 if (!UrlUtils.sameFile(new URL(proxyConfig.getProxyAutoConfigUrl()), url)) {
1604                     String content = proxyConfig.getProxyAutoConfigContent();
1605                     if (content == null) {
1606                         content = getPage(proxyConfig.getProxyAutoConfigUrl())
1607                             .getWebResponse().getContentAsString();
1608                         proxyConfig.setProxyAutoConfigContent(content);
1609                     }
1610                     final String allValue = JavaScriptEngine.evaluateProxyAutoConfig(getBrowserVersion(), content, url);
1611                     if (LOG.isDebugEnabled()) {
1612                         LOG.debug("Proxy Auto-Config: value '" + allValue + "' for URL " + url);
1613                     }
1614                     String value = allValue.split(";")[0].trim();
1615                     if (value.startsWith("PROXY")) {
1616                         value = value.substring(6);
1617                         final int colonIndex = value.indexOf(':');
1618                         webRequest.setSocksProxy(false);
1619                         webRequest.setProxyHost(value.substring(0, colonIndex));
1620                         webRequest.setProxyPort(Integer.parseInt(value.substring(colonIndex + 1)));
1621                     }
1622                     else if (value.startsWith("SOCKS")) {
1623                         value = value.substring(6);
1624                         final int colonIndex = value.indexOf(':');
1625                         webRequest.setSocksProxy(true);
1626                         webRequest.setProxyHost(value.substring(0, colonIndex));
1627                         webRequest.setProxyPort(Integer.parseInt(value.substring(colonIndex + 1)));
1628                     }
1629                 }
1630             }
1631             // ...unless the host needs to bypass the configured client proxy!
1632             else if (!proxyConfig.shouldBypassProxy(webRequest.getUrl().getHost())) {
1633                 webRequest.setProxyHost(proxyConfig.getProxyHost());
1634                 webRequest.setProxyPort(proxyConfig.getProxyPort());
1635                 webRequest.setProxyScheme(proxyConfig.getProxyScheme());
1636                 webRequest.setSocksProxy(proxyConfig.isSocksProxy());
1637             }
1638         }
1639 
1640         // Add the headers that are sent with every request.
1641         addDefaultHeaders(webRequest);
1642 
1643         // Retrieve the response, either from the cache or from the server.
1644         final WebResponse fromCache = getCache().getCachedResponse(webRequest);
1645         final WebResponse webResponse = getWebResponseOrUseCached(webRequest, fromCache);
1646 
1647         // Continue according to the HTTP status code.
1648         final int status = webResponse.getStatusCode();
1649         if (status == HttpStatus.USE_PROXY_305) {
1650             getIncorrectnessListener().notify("Ignoring HTTP status code [305] 'Use Proxy'", this);
1651         }
1652         else if (status >= HttpStatus.MOVED_PERMANENTLY_301
1653             && status <= HttpStatus.PERMANENT_REDIRECT_308
1654             && status != HttpStatus.NOT_MODIFIED_304
1655             && getOptions().isRedirectEnabled()) {
1656 
1657             final URL newUrl;
1658             String locationString = null;
1659             try {
1660                 locationString = webResponse.getResponseHeaderValue("Location");
1661                 if (locationString == null) {
1662                     return webResponse;
1663                 }
1664                 locationString = new String(locationString.getBytes(ISO_8859_1), UTF_8);
1665                 newUrl = expandUrl(url, locationString);
1666             }
1667             catch (final MalformedURLException e) {
1668                 getIncorrectnessListener().notify("Got a redirect status code [" + status + " "
1669                     + webResponse.getStatusMessage()
1670                     + "] but the location is not a valid URL [" + locationString
1671                     + "]. Skipping redirection processing.", this);
1672                 return webResponse;
1673             }
1674 
1675             if (LOG.isDebugEnabled()) {
1676                 LOG.debug("Got a redirect status code [" + status + "] new location = [" + locationString + "]");
1677             }
1678 
1679             if (allowedRedirects == 0) {
1680                 throw new FailingHttpStatusCodeException("Too much redirect for "
1681                     + webResponse.getWebRequest().getUrl(), webResponse);
1682             }
1683 
1684             if (status == HttpStatus.MOVED_PERMANENTLY_301
1685                     || status == HttpStatus.FOUND_302
1686                     || status == HttpStatus.SEE_OTHER_303) {
1687                 final WebRequest wrs = new WebRequest(newUrl, HttpMethod.GET);
1688                 wrs.setCharset(webRequest.getCharset());
1689 
1690                 if (HttpMethod.HEAD == webRequest.getHttpMethod()) {
1691                     wrs.setHttpMethod(HttpMethod.HEAD);
1692                 }
1693                 for (final Map.Entry<String, String> entry : webRequest.getAdditionalHeaders().entrySet()) {
1694                     wrs.setAdditionalHeader(entry.getKey(), entry.getValue());
1695                 }
1696                 return loadWebResponseFromWebConnection(wrs, allowedRedirects - 1);
1697             }
1698             else if (status == HttpStatus.TEMPORARY_REDIRECT_307
1699                         || status == HttpStatus.PERMANENT_REDIRECT_308) {
1700                 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/307
1701                 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/308
1702                 // reuse method and body
1703                 final WebRequest wrs = new WebRequest(newUrl, webRequest.getHttpMethod());
1704                 wrs.setCharset(webRequest.getCharset());
1705                 if (webRequest.getRequestBody() != null) {
1706                     if (HttpMethod.POST == webRequest.getHttpMethod()
1707                             || HttpMethod.PUT == webRequest.getHttpMethod()
1708                             || HttpMethod.PATCH == webRequest.getHttpMethod()) {
1709                         wrs.setRequestBody(webRequest.getRequestBody());
1710                         wrs.setEncodingType(webRequest.getEncodingType());
1711                     }
1712                 }
1713                 else {
1714                     wrs.setRequestParameters(parameters);
1715                 }
1716 
1717                 for (final Map.Entry<String, String> entry : webRequest.getAdditionalHeaders().entrySet()) {
1718                     wrs.setAdditionalHeader(entry.getKey(), entry.getValue());
1719                 }
1720 
1721                 return loadWebResponseFromWebConnection(wrs, allowedRedirects - 1);
1722             }
1723         }
1724 
1725         if (fromCache == null) {
1726             getCache().cacheIfPossible(webRequest, webResponse, null);
1727         }
1728         return webResponse;
1729     }
1730 
1731     /**
1732      * Returns the cached response provided for the request if usable otherwise makes the
1733      * request and returns the response.
1734      * @param webRequest the request
1735      * @param cached a previous cached response for the request, or {@code null}
1736      */
1737     private WebResponse getWebResponseOrUseCached(
1738             final WebRequest webRequest, final WebResponse cached) throws IOException {
1739         if (cached == null) {
1740             return getWebConnection().getResponse(webRequest);
1741         }
1742 
1743         if (!HeaderUtils.containsNoCache(cached)) {
1744             return new WebResponseFromCache(cached, webRequest);
1745         }
1746 
1747         // implementation based on rfc9111 https://www.rfc-editor.org/rfc/rfc9111#name-validation
1748         if (HeaderUtils.containsETag(cached)) {
1749             webRequest.setAdditionalHeader(HttpHeader.IF_NONE_MATCH, cached.getResponseHeaderValue(HttpHeader.ETAG));
1750         }
1751         if (HeaderUtils.containsLastModified(cached)) {
1752             webRequest.setAdditionalHeader(HttpHeader.IF_MODIFIED_SINCE,
1753                     cached.getResponseHeaderValue(HttpHeader.LAST_MODIFIED));
1754         }
1755 
1756         final WebResponse webResponse = getWebConnection().getResponse(webRequest);
1757 
1758         if (webResponse.getStatusCode() >= HttpStatus.INTERNAL_SERVER_ERROR_500) {
1759             return new WebResponseFromCache(cached, webRequest);
1760         }
1761 
1762         if (webResponse.getStatusCode() == HttpStatus.NOT_MODIFIED_304) {
1763             final Map<String, NameValuePair> header2NameValuePair = new LinkedHashMap<>();
1764             for (final NameValuePair pair : cached.getResponseHeaders()) {
1765                 header2NameValuePair.put(pair.getName(), pair);
1766             }
1767             for (final NameValuePair pair : webResponse.getResponseHeaders()) {
1768                 if (preferHeaderFrom304Response(pair.getName())) {
1769                     header2NameValuePair.put(pair.getName(), pair);
1770                 }
1771             }
1772             // WebResponse headers is unmodifiableList so we cannot update it directly
1773             // instead, create a new WebResponseFromCache with updated headers
1774             // then use it to replace the old cached value
1775             final WebResponse updatedCached =
1776                     new WebResponseFromCache(cached, new ArrayList<>(header2NameValuePair.values()), webRequest);
1777             getCache().cacheIfPossible(webRequest, updatedCached, null);
1778             return updatedCached;
1779         }
1780 
1781         getCache().cacheIfPossible(webRequest, webResponse, null);
1782         return webResponse;
1783     }
1784 
1785     /**
1786      * Returns true if the value of the specified header in a 304 Not Modified response should be
1787      * adopted over any previously cached value.
1788      */
1789     private static boolean preferHeaderFrom304Response(final String name) {
1790         final String lcName = name.toLowerCase(Locale.ROOT);
1791         for (final String header : DISCARDING_304_RESPONSE_HEADER_NAMES) {
1792             if (lcName.equals(header)) {
1793                 return false;
1794             }
1795         }
1796         for (final String prefix : DISCARDING_304_HEADER_PREFIXES) {
1797             if (lcName.startsWith(prefix)) {
1798                 return false;
1799             }
1800         }
1801         return true;
1802     }
1803 
1804     /**
1805      * Adds the headers that are sent with every request to the specified {@link WebRequest} instance.
1806      * @param wrs the <code>WebRequestSettings</code> instance to modify
1807      */
1808     private void addDefaultHeaders(final WebRequest wrs) {
1809         // Add user-specified headers to the web request if not present there yet.
1810         requestHeaders_.forEach((name, value) -> {
1811             if (!wrs.isAdditionalHeader(name)) {
1812                 wrs.setAdditionalHeader(name, value);
1813             }
1814         });
1815 
1816         // Add standard HtmlUnit headers to the web request if still not present there yet.
1817         if (!wrs.isAdditionalHeader(HttpHeader.ACCEPT_LANGUAGE)) {
1818             wrs.setAdditionalHeader(HttpHeader.ACCEPT_LANGUAGE, getBrowserVersion().getAcceptLanguageHeader());
1819         }
1820 
1821         if (!wrs.isAdditionalHeader(HttpHeader.SEC_FETCH_DEST)) {
1822             wrs.setAdditionalHeader(HttpHeader.SEC_FETCH_DEST, "document");
1823         }
1824         if (!wrs.isAdditionalHeader(HttpHeader.SEC_FETCH_MODE)) {
1825             wrs.setAdditionalHeader(HttpHeader.SEC_FETCH_MODE, "navigate");
1826         }
1827         if (!wrs.isAdditionalHeader(HttpHeader.SEC_FETCH_SITE)) {
1828             wrs.setAdditionalHeader(HttpHeader.SEC_FETCH_SITE, "same-origin");
1829         }
1830         if (!wrs.isAdditionalHeader(HttpHeader.SEC_FETCH_USER)) {
1831             wrs.setAdditionalHeader(HttpHeader.SEC_FETCH_USER, "?1");
1832         }
1833         if (getBrowserVersion().hasFeature(HTTP_HEADER_PRIORITY)
1834                 && !wrs.isAdditionalHeader(HttpHeader.PRIORITY)) {
1835             wrs.setAdditionalHeader(HttpHeader.PRIORITY, "u=0, i");
1836         }
1837 
1838         if (getBrowserVersion().hasFeature(HTTP_HEADER_CH_UA)
1839                 && !wrs.isAdditionalHeader(HttpHeader.SEC_CH_UA)) {
1840             wrs.setAdditionalHeader(HttpHeader.SEC_CH_UA, getBrowserVersion().getSecClientHintUserAgentHeader());
1841         }
1842         if (getBrowserVersion().hasFeature(HTTP_HEADER_CH_UA)
1843                 && !wrs.isAdditionalHeader(HttpHeader.SEC_CH_UA_MOBILE)) {
1844             wrs.setAdditionalHeader(HttpHeader.SEC_CH_UA_MOBILE, "?0");
1845         }
1846         if (getBrowserVersion().hasFeature(HTTP_HEADER_CH_UA)
1847                 && !wrs.isAdditionalHeader(HttpHeader.SEC_CH_UA_PLATFORM)) {
1848             wrs.setAdditionalHeader(HttpHeader.SEC_CH_UA_PLATFORM,
1849                     getBrowserVersion().getSecClientHintUserAgentPlatformHeader());
1850         }
1851 
1852         if (!wrs.isAdditionalHeader(HttpHeader.UPGRADE_INSECURE_REQUESTS)) {
1853             wrs.setAdditionalHeader(HttpHeader.UPGRADE_INSECURE_REQUESTS, "1");
1854         }
1855     }
1856 
1857     /**
1858      * Returns an immutable list of open web windows (whether they are top level windows or not).
1859      * This is a snapshot; future changes are not reflected by this list.
1860      * <p>
1861      * The list is ordered by age, the oldest one first.
1862      *
1863      * @return an immutable list of open web windows (whether they are top level windows or not)
1864      * @see #getWebWindowByName(String)
1865      * @see #getTopLevelWindows()
1866      */
1867     public List<WebWindow> getWebWindows() {
1868         return Collections.unmodifiableList(new ArrayList<>(windows_));
1869     }
1870 
1871     /**
1872      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1873      *
1874      * Returns true if the list of WebWindows contains the provided one.
1875      * This method is there to improve the performance of some internal checks because
1876      * calling getWebWindows().contains(.) creates some objects without any need.
1877      *
1878      * @param webWindow the window to check
1879      * @return true or false
1880      */
1881     public boolean containsWebWindow(final WebWindow webWindow) {
1882         return windows_.contains(webWindow);
1883     }
1884 
1885     /**
1886      * Returns an immutable list of open top level windows.
1887      * This is a snapshot; future changes are not reflected by this list.
1888      * <p>
1889      * The list is ordered by age, the oldest one first.
1890      *
1891      * @return an immutable list of open top level windows
1892      * @see #getWebWindowByName(String)
1893      * @see #getWebWindows()
1894      */
1895     public List<TopLevelWindow> getTopLevelWindows() {
1896         return Collections.unmodifiableList(new ArrayList<>(topLevelWindows_));
1897     }
1898 
1899     /**
1900      * Sets the handler to be used whenever a refresh is triggered. Refer
1901      * to the documentation for {@link RefreshHandler} for more details.
1902      * @param handler the new handler
1903      */
1904     public void setRefreshHandler(final RefreshHandler handler) {
1905         if (handler == null) {
1906             refreshHandler_ = new NiceRefreshHandler(2);
1907         }
1908         else {
1909             refreshHandler_ = handler;
1910         }
1911     }
1912 
1913     /**
1914      * Returns the current refresh handler.
1915      * The default refresh handler is a {@link NiceRefreshHandler NiceRefreshHandler(2)}.
1916      * @return the current RefreshHandler
1917      */
1918     public RefreshHandler getRefreshHandler() {
1919         return refreshHandler_;
1920     }
1921 
1922     /**
1923      * Sets the script pre processor for this {@link WebClient}.
1924      * @param scriptPreProcessor the new preprocessor or null if none is specified
1925      */
1926     public void setScriptPreProcessor(final ScriptPreProcessor scriptPreProcessor) {
1927         scriptPreProcessor_ = scriptPreProcessor;
1928     }
1929 
1930     /**
1931      * Returns the script pre processor for this {@link WebClient}.
1932      * @return the pre processor or null of one hasn't been set
1933      */
1934     public ScriptPreProcessor getScriptPreProcessor() {
1935         return scriptPreProcessor_;
1936     }
1937 
1938     /**
1939      * Sets the listener for messages generated by the HTML parser.
1940      * @param listener the new listener, {@code null} if messages should be totally ignored
1941      */
1942     public void setHTMLParserListener(final HTMLParserListener listener) {
1943         htmlParserListener_ = listener;
1944     }
1945 
1946     /**
1947      * Gets the configured listener for messages generated by the HTML parser.
1948      * @return {@code null} if no listener is defined (default value)
1949      */
1950     public HTMLParserListener getHTMLParserListener() {
1951         return htmlParserListener_;
1952     }
1953 
1954     /**
1955      * Returns the CSS error handler used by this web client when CSS problems are encountered.
1956      * @return the CSS error handler used by this web client when CSS problems are encountered
1957      * @see DefaultCssErrorHandler
1958      * @see SilentCssErrorHandler
1959      */
1960     public CSSErrorHandler getCssErrorHandler() {
1961         return cssErrorHandler_;
1962     }
1963 
1964     /**
1965      * Sets the CSS error handler used by this web client when CSS problems are encountered.
1966      * @param cssErrorHandler the CSS error handler used by this web client when CSS problems are encountered
1967      * @see DefaultCssErrorHandler
1968      * @see SilentCssErrorHandler
1969      */
1970     public void setCssErrorHandler(final CSSErrorHandler cssErrorHandler) {
1971         WebAssert.notNull("cssErrorHandler", cssErrorHandler);
1972         cssErrorHandler_ = cssErrorHandler;
1973     }
1974 
1975     /**
1976      * Sets the number of milliseconds that a script is allowed to execute before being terminated.
1977      * A value of 0 or less means no timeout.
1978      *
1979      * @param timeout the timeout value, in milliseconds
1980      */
1981     public void setJavaScriptTimeout(final long timeout) {
1982         scriptEngine_.setJavaScriptTimeout(timeout);
1983     }
1984 
1985     /**
1986      * Returns the number of milliseconds that a script is allowed to execute before being terminated.
1987      * A value of 0 or less means no timeout.
1988      *
1989      * @return the timeout value, in milliseconds
1990      */
1991     public long getJavaScriptTimeout() {
1992         return scriptEngine_.getJavaScriptTimeout();
1993     }
1994 
1995     /**
1996      * Gets the current listener for encountered incorrectness (except HTML parsing messages that
1997      * are handled by the HTML parser listener). Default value is an instance of
1998      * {@link IncorrectnessListenerImpl}.
1999      * @return the current listener (not {@code null})
2000      */
2001     public IncorrectnessListener getIncorrectnessListener() {
2002         return incorrectnessListener_;
2003     }
2004 
2005     /**
2006      * Returns the current HTML incorrectness listener.
2007      * @param listener the new value (not {@code null})
2008      */
2009     public void setIncorrectnessListener(final IncorrectnessListener listener) {
2010         if (listener == null) {
2011             throw new IllegalArgumentException("Null is not a valid IncorrectnessListener");
2012         }
2013         incorrectnessListener_ = listener;
2014     }
2015 
2016     /**
2017      * Returns the WebConsole.
2018      * @return the web console
2019      */
2020     public WebConsole getWebConsole() {
2021         if (webConsole_ == null) {
2022             webConsole_ = new WebConsole();
2023         }
2024         return webConsole_;
2025     }
2026 
2027     /**
2028      * Gets the current AJAX controller.
2029      * @return the controller
2030      */
2031     public AjaxController getAjaxController() {
2032         return ajaxController_;
2033     }
2034 
2035     /**
2036      * Sets the current AJAX controller.
2037      * @param newValue the controller
2038      */
2039     public void setAjaxController(final AjaxController newValue) {
2040         if (newValue == null) {
2041             throw new IllegalArgumentException("Null is not a valid AjaxController");
2042         }
2043         ajaxController_ = newValue;
2044     }
2045 
2046     /**
2047      * Sets the attachment handler.
2048      * @param handler the new attachment handler
2049      */
2050     public void setAttachmentHandler(final AttachmentHandler handler) {
2051         attachmentHandler_ = handler;
2052     }
2053 
2054     /**
2055      * Returns the current attachment handler.
2056      * @return the current attachment handler
2057      */
2058     public AttachmentHandler getAttachmentHandler() {
2059         return attachmentHandler_;
2060     }
2061 
2062     /**
2063      * Sets the WebStart handler.
2064      * @param handler the new WebStart handler
2065      */
2066     public void setWebStartHandler(final WebStartHandler handler) {
2067         webStartHandler_ = handler;
2068     }
2069 
2070     /**
2071      * Returns the current WebStart handler.
2072      * @return the current WebStart handler
2073      */
2074     public WebStartHandler getWebStartHandler() {
2075         return webStartHandler_;
2076     }
2077 
2078     /**
2079      * Returns the current clipboard handler.
2080      * @return the current clipboard handler
2081      */
2082     public ClipboardHandler getClipboardHandler() {
2083         return clipboardHandler_;
2084     }
2085 
2086     /**
2087      * Sets the clipboard handler.
2088      * @param handler the new clipboard handler
2089      */
2090     public void setClipboardHandler(final ClipboardHandler handler) {
2091         clipboardHandler_ = handler;
2092     }
2093 
2094     /**
2095      * Returns the current {@link PrintHandler}.
2096      * @return the current {@link PrintHandler} or null if print
2097      *         requests are ignored
2098      */
2099     public PrintHandler getPrintHandler() {
2100         return printHandler_;
2101     }
2102 
2103     /**
2104      * Sets the {@link PrintHandler} to be used if Windoe.print() is called
2105      * (<a href="https://html.spec.whatwg.org/multipage/timers-and-user-prompts.html#printing">Printing Spec</a>).
2106      *
2107      * @param handler the new {@link PrintHandler} or null if you like to
2108      *        ignore print requests (default is null)
2109      */
2110     public void setPrintHandler(final PrintHandler handler) {
2111         printHandler_ = handler;
2112     }
2113 
2114     /**
2115      * Returns the current FrameContent handler.
2116      * @return the current FrameContent handler
2117      */
2118     public FrameContentHandler getFrameContentHandler() {
2119         return frameContentHandler_;
2120     }
2121 
2122     /**
2123      * Sets the FrameContent handler.
2124      * @param handler the new FrameContent handler
2125      */
2126     public void setFrameContentHandler(final FrameContentHandler handler) {
2127         frameContentHandler_ = handler;
2128     }
2129 
2130     /**
2131      * Sets the onbeforeunload handler for this {@link WebClient}.
2132      * @param onbeforeunloadHandler the new onbeforeunloadHandler or null if none is specified
2133      */
2134     public void setOnbeforeunloadHandler(final OnbeforeunloadHandler onbeforeunloadHandler) {
2135         onbeforeunloadHandler_ = onbeforeunloadHandler;
2136     }
2137 
2138     /**
2139      * Returns the onbeforeunload handler for this {@link WebClient}.
2140      * @return the onbeforeunload handler or null if one hasn't been set
2141      */
2142     public OnbeforeunloadHandler getOnbeforeunloadHandler() {
2143         return onbeforeunloadHandler_;
2144     }
2145 
2146     /**
2147      * Gets the cache currently being used.
2148      * @return the cache (may not be null)
2149      */
2150     public Cache getCache() {
2151         return cache_;
2152     }
2153 
2154     /**
2155      * Sets the cache to use.
2156      * @param cache the new cache (must not be {@code null})
2157      */
2158     public void setCache(final Cache cache) {
2159         if (cache == null) {
2160             throw new IllegalArgumentException("cache should not be null!");
2161         }
2162         cache_ = cache;
2163     }
2164 
2165     /**
2166      * Keeps track of the current window. Inspired by WebTest's logic to track the current response.
2167      */
2168     private static final class CurrentWindowTracker implements WebWindowListener, Serializable {
2169         private final WebClient webClient_;
2170         private final boolean ensureOneTopLevelWindow_;
2171 
2172         CurrentWindowTracker(final WebClient webClient, final boolean ensureOneTopLevelWindow) {
2173             webClient_ = webClient;
2174             ensureOneTopLevelWindow_ = ensureOneTopLevelWindow;
2175         }
2176 
2177         /**
2178          * {@inheritDoc}
2179          */
2180         @Override
2181         public void webWindowClosed(final WebWindowEvent event) {
2182             final WebWindow window = event.getWebWindow();
2183             if (window instanceof TopLevelWindow) {
2184                 webClient_.topLevelWindows_.remove(window);
2185                 if (window == webClient_.getCurrentWindow()) {
2186                     if (!webClient_.topLevelWindows_.isEmpty()) {
2187                         // The current window is now the previous top-level window.
2188                         webClient_.setCurrentWindow(
2189                                 webClient_.topLevelWindows_.get(webClient_.topLevelWindows_.size() - 1));
2190                     }
2191                 }
2192             }
2193             else if (window == webClient_.getCurrentWindow()) {
2194                 // The current window is now the last top-level window.
2195                 if (webClient_.topLevelWindows_.isEmpty()) {
2196                     webClient_.setCurrentWindow(null);
2197                 }
2198                 else {
2199                     webClient_.setCurrentWindow(
2200                             webClient_.topLevelWindows_.get(webClient_.topLevelWindows_.size() - 1));
2201                 }
2202             }
2203         }
2204 
2205         /**
2206          * Postprocessing to make sure we have always one top level window open.
2207          */
2208         public void afterWebWindowClosedListenersProcessed(final WebWindowEvent event) {
2209             if (!ensureOneTopLevelWindow_) {
2210                 return;
2211             }
2212 
2213             if (webClient_.topLevelWindows_.isEmpty()) {
2214                 // Must always have at least window, and there are no top-level windows left; must create one.
2215                 final TopLevelWindow newWindow = new TopLevelWindow("", webClient_);
2216                 webClient_.setCurrentWindow(newWindow);
2217             }
2218         }
2219 
2220         /**
2221          * {@inheritDoc}
2222          */
2223         @Override
2224         public void webWindowContentChanged(final WebWindowEvent event) {
2225             final WebWindow window = event.getWebWindow();
2226             boolean use = false;
2227             if (window instanceof DialogWindow) {
2228                 use = true;
2229             }
2230             else if (window instanceof TopLevelWindow) {
2231                 use = event.getOldPage() == null;
2232             }
2233             else if (window instanceof FrameWindow) {
2234                 final FrameWindow fw = (FrameWindow) window;
2235                 final String enclosingPageState = fw.getEnclosingPage().getDocumentElement().getReadyState();
2236                 final URL frameUrl = fw.getEnclosedPage().getUrl();
2237                 if (!DomNode.READY_STATE_COMPLETE.equals(enclosingPageState) || frameUrl == UrlUtils.URL_ABOUT_BLANK) {
2238                     return;
2239                 }
2240 
2241                 // now looks at the visibility of the frame window
2242                 final BaseFrameElement frameElement = fw.getFrameElement();
2243                 if (webClient_.isJavaScriptEnabled() && frameElement.isDisplayed()) {
2244                     final ComputedCssStyleDeclaration style = fw.getComputedStyle(frameElement, null);
2245                     use = style.getCalculatedWidth(false, false) != 0
2246                             && style.getCalculatedHeight(false, false) != 0;
2247                 }
2248             }
2249             if (use) {
2250                 webClient_.setCurrentWindow(window);
2251             }
2252         }
2253 
2254         /**
2255          * {@inheritDoc}
2256          */
2257         @Override
2258         public void webWindowOpened(final WebWindowEvent event) {
2259             final WebWindow window = event.getWebWindow();
2260             if (window instanceof TopLevelWindow) {
2261                 final TopLevelWindow tlw = (TopLevelWindow) window;
2262                 webClient_.topLevelWindows_.add(tlw);
2263             }
2264             // Page is not loaded yet, don't set it now as current window.
2265         }
2266     }
2267 
2268     /**
2269      * Closes all opened windows, stopping all background JavaScript processing.
2270      * The WebClient is not really usable after this - you have to create a new one or
2271      * use WebClient.reset() instead.
2272      * <p>
2273      * {@inheritDoc}
2274      */
2275     @Override
2276     public void close() {
2277         // avoid attaching new windows to the js engine
2278         if (scriptEngine_ != null) {
2279             scriptEngine_.prepareShutdown();
2280         }
2281 
2282         // stop the CurrentWindowTracker from making sure there is still one window available
2283         currentWindowTracker_ = new CurrentWindowTracker(this, false);
2284 
2285         // Hint: a new TopLevelWindow may be opened by some JS script while we are closing the others
2286         // but the prepareShutdown() call will prevent the new window form getting js support
2287         List<WebWindow> windows = new ArrayList<>(windows_);
2288         for (final WebWindow window : windows) {
2289             if (window instanceof TopLevelWindow) {
2290                 final TopLevelWindow topLevelWindow = (TopLevelWindow) window;
2291 
2292                 try {
2293                     topLevelWindow.close(true);
2294                 }
2295                 catch (final Exception e) {
2296                     LOG.error("Exception while closing a TopLevelWindow", e);
2297                 }
2298             }
2299             else if (window instanceof DialogWindow) {
2300                 final DialogWindow dialogWindow = (DialogWindow) window;
2301 
2302                 try {
2303                     dialogWindow.close();
2304                 }
2305                 catch (final Exception e) {
2306                     LOG.error("Exception while closing a DialogWindow", e);
2307                 }
2308             }
2309         }
2310 
2311         // second round, none of the remaining windows should be registered to
2312         // the js engine because of prepareShutdown()
2313         windows = new ArrayList<>(windows_);
2314         for (final WebWindow window : windows) {
2315             if (window instanceof TopLevelWindow) {
2316                 final TopLevelWindow topLevelWindow = (TopLevelWindow) window;
2317 
2318                 try {
2319                     topLevelWindow.close(true);
2320                 }
2321                 catch (final Exception e) {
2322                     LOG.error("Exception while closing a TopLevelWindow", e);
2323                 }
2324             }
2325             else if (window instanceof DialogWindow) {
2326                 final DialogWindow dialogWindow = (DialogWindow) window;
2327 
2328                 try {
2329                     dialogWindow.close();
2330                 }
2331                 catch (final Exception e) {
2332                     LOG.error("Exception while closing a DialogWindow", e);
2333                 }
2334             }
2335         }
2336 
2337         // now both lists have to be empty
2338         if (!topLevelWindows_.isEmpty()) {
2339             LOG.error("Sill " + topLevelWindows_.size() + " top level windows are open. Please report this error!");
2340             topLevelWindows_.clear();
2341         }
2342 
2343         if (!windows_.isEmpty()) {
2344             LOG.error("Sill " + windows_.size() + " windows are open. Please report this error!");
2345             windows_.clear();
2346         }
2347         currentWindow_ = null;
2348 
2349         ThreadDeath toThrow = null;
2350         if (scriptEngine_ != null) {
2351             try {
2352                 scriptEngine_.shutdown();
2353             }
2354             catch (final ThreadDeath ex) {
2355                 // make sure the following cleanup is performed to avoid resource leaks
2356                 toThrow = ex;
2357             }
2358             catch (final Exception e) {
2359                 LOG.error("Exception while shutdown the scriptEngine", e);
2360             }
2361         }
2362         scriptEngine_ = null;
2363 
2364         if (webConnection_ != null) {
2365             try {
2366                 webConnection_.close();
2367             }
2368             catch (final Exception e) {
2369                 LOG.error("Exception while closing the connection", e);
2370             }
2371         }
2372         webConnection_ = null;
2373 
2374         synchronized (this) {
2375             if (executor_ != null) {
2376                 try {
2377                     executor_.shutdownNow();
2378                 }
2379                 catch (final Exception e) {
2380                     LOG.error("Exception while shutdown the executor service", e);
2381                 }
2382             }
2383         }
2384         executor_ = null;
2385 
2386         cache_.clear();
2387         if (toThrow != null) {
2388             throw toThrow;
2389         }
2390     }
2391 
2392     /**
2393      * <p><span style="color:red">Experimental API: May be changed in next release
2394      * and may not yet work perfectly!</span></p>
2395      *
2396      * <p>This shuts down the whole client and restarts with a new empty window.
2397      * Cookies and other states are preserved.
2398      */
2399     public void reset() {
2400         close();
2401 
2402         // this has to be done after the browser version was set
2403         webConnection_ = new HttpWebConnection(this);
2404         if (javaScriptEngineEnabled_) {
2405             scriptEngine_ = new JavaScriptEngine(this);
2406         }
2407 
2408         // The window must be constructed AFTER the script engine.
2409         currentWindowTracker_ = new CurrentWindowTracker(this, true);
2410         currentWindow_ = new TopLevelWindow("", this);
2411     }
2412 
2413     /**
2414      * <p><span style="color:red">Experimental API: May be changed in next release
2415      * and may not yet work perfectly!</span></p>
2416      *
2417      * <p>This method blocks until all background JavaScript tasks have finished executing. Background
2418      * JavaScript tasks are JavaScript tasks scheduled for execution via <code>window.setTimeout</code>,
2419      * <code>window.setInterval</code> or asynchronous <code>XMLHttpRequest</code>.</p>
2420      *
2421      * <p>If a job is scheduled to begin executing after <code>(now + timeoutMillis)</code>, this method will
2422      * wait for <code>timeoutMillis</code> milliseconds and then return a value greater than <code>0</code>. This
2423      * method will never block longer than <code>timeoutMillis</code> milliseconds.</p>
2424      *
2425      * <p>Use this method instead of {@link #waitForBackgroundJavaScriptStartingBefore(long)} if you
2426      * don't know when your background JavaScript is supposed to start executing, but you're fairly sure
2427      * that you know how long it should take to finish executing.</p>
2428      *
2429      * @param timeoutMillis the maximum amount of time to wait (in milliseconds)
2430      * @return the number of background JavaScript jobs still executing or waiting to be executed when this
2431      *         method returns; will be <code>0</code> if there are no jobs left to execute
2432      */
2433     public int waitForBackgroundJavaScript(final long timeoutMillis) {
2434         int count = 0;
2435         final long endTime = System.currentTimeMillis() + timeoutMillis;
2436         for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
2437             final JavaScriptJobManager jobManager;
2438             final WeakReference<JavaScriptJobManager> reference;
2439             try {
2440                 reference = i.next();
2441                 jobManager = reference.get();
2442                 if (jobManager == null) {
2443                     i.remove();
2444                     continue;
2445                 }
2446             }
2447             catch (final ConcurrentModificationException e) {
2448                 i = jobManagers_.iterator();
2449                 count = 0;
2450                 continue;
2451             }
2452 
2453             final long newTimeout = endTime - System.currentTimeMillis();
2454             count += jobManager.waitForJobs(newTimeout);
2455         }
2456         if (count != getAggregateJobCount()) {
2457             final long newTimeout = endTime - System.currentTimeMillis();
2458             return waitForBackgroundJavaScript(newTimeout);
2459         }
2460         return count;
2461     }
2462 
2463     /**
2464      * <p><span style="color:red">Experimental API: May be changed in next release
2465      * and may not yet work perfectly!</span></p>
2466      *
2467      * <p>This method blocks until all background JavaScript tasks scheduled to start executing before
2468      * <code>(now + delayMillis)</code> have finished executing. Background JavaScript tasks are JavaScript
2469      * tasks scheduled for execution via <code>window.setTimeout</code>, <code>window.setInterval</code> or
2470      * asynchronous <code>XMLHttpRequest</code>.</p>
2471      *
2472      * <p>If there is no background JavaScript task currently executing, and there is no background JavaScript
2473      * task scheduled to start executing within the specified time, this method returns immediately -- even
2474      * if there are tasks scheduled to be executed after <code>(now + delayMillis)</code>.</p>
2475      *
2476      * <p>Note that the total time spent executing a background JavaScript task is never known ahead of
2477      * time, so this method makes no guarantees as to how long it will block.</p>
2478      *
2479      * <p>Use this method instead of {@link #waitForBackgroundJavaScript(long)} if you know roughly when
2480      * your background JavaScript is supposed to start executing, but you're not necessarily sure how long
2481      * it will take to execute.</p>
2482      *
2483      * @param delayMillis the delay which determines the background tasks to wait for (in milliseconds)
2484      * @return the number of background JavaScript jobs still executing or waiting to be executed when this
2485      *         method returns; will be <code>0</code> if there are no jobs left to execute
2486      */
2487     public int waitForBackgroundJavaScriptStartingBefore(final long delayMillis) {
2488         int count = 0;
2489         final long endTime = System.currentTimeMillis() + delayMillis;
2490         for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
2491             final JavaScriptJobManager jobManager;
2492             final WeakReference<JavaScriptJobManager> reference;
2493             try {
2494                 reference = i.next();
2495                 jobManager = reference.get();
2496                 if (jobManager == null) {
2497                     i.remove();
2498                     continue;
2499                 }
2500             }
2501             catch (final ConcurrentModificationException e) {
2502                 i = jobManagers_.iterator();
2503                 count = 0;
2504                 continue;
2505             }
2506             final long newDelay = endTime - System.currentTimeMillis();
2507             count += jobManager.waitForJobsStartingBefore(newDelay);
2508         }
2509         if (count != getAggregateJobCount()) {
2510             final long newDelay = endTime - System.currentTimeMillis();
2511             return waitForBackgroundJavaScriptStartingBefore(newDelay);
2512         }
2513         return count;
2514     }
2515 
2516     /**
2517      * Returns the aggregate background JavaScript job count across all windows.
2518      * @return the aggregate background JavaScript job count across all windows
2519      */
2520     private int getAggregateJobCount() {
2521         int count = 0;
2522         for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
2523             final JavaScriptJobManager jobManager;
2524             final WeakReference<JavaScriptJobManager> reference;
2525             try {
2526                 reference = i.next();
2527                 jobManager = reference.get();
2528                 if (jobManager == null) {
2529                     i.remove();
2530                     continue;
2531                 }
2532             }
2533             catch (final ConcurrentModificationException e) {
2534                 i = jobManagers_.iterator();
2535                 count = 0;
2536                 continue;
2537             }
2538             final int jobCount = jobManager.getJobCount();
2539             count += jobCount;
2540         }
2541         return count;
2542     }
2543 
2544     /**
2545      * When we deserialize, re-initializie transient fields.
2546      * @param in the object input stream
2547      * @throws IOException if an error occurs
2548      * @throws ClassNotFoundException if an error occurs
2549      */
2550     private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
2551         in.defaultReadObject();
2552 
2553         webConnection_ = new HttpWebConnection(this);
2554         scriptEngine_ = new JavaScriptEngine(this);
2555         jobManagers_ = Collections.synchronizedList(new ArrayList<>());
2556         loadQueue_ = new ArrayList<>();
2557         css3ParserPool_ = new CSS3ParserPool();
2558     }
2559 
2560     private static class LoadJob {
2561         private final WebWindow requestingWindow_;
2562         private final String target_;
2563         private final WebResponse response_;
2564         private final WeakReference<Page> originalPage_;
2565         private final WebRequest request_;
2566         private final String forceAttachmentWithFilename_;
2567 
2568         // we can't us the WebRequest from the WebResponse because
2569         // we need the original request e.g. after a redirect
2570         LoadJob(final WebRequest request, final WebResponse response,
2571                 final WebWindow requestingWindow, final String target, final String forceAttachmentWithFilename) {
2572             request_ = request;
2573             requestingWindow_ = requestingWindow;
2574             target_ = target;
2575             response_ = response;
2576             originalPage_ = new WeakReference<>(requestingWindow.getEnclosedPage());
2577             forceAttachmentWithFilename_ = forceAttachmentWithFilename;
2578         }
2579 
2580         public boolean isOutdated() {
2581             if (target_ != null && !target_.isEmpty()) {
2582                 return false;
2583             }
2584 
2585             if (requestingWindow_.isClosed()) {
2586                 return true;
2587             }
2588 
2589             if (requestingWindow_.getEnclosedPage() != originalPage_.get()) {
2590                 return true;
2591             }
2592 
2593             return false;
2594         }
2595     }
2596 
2597     /**
2598      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2599      *
2600      * Perform the downloads and stores it for loading later into a window.
2601      * In the future downloads should be performed in parallel in separated threads.
2602      * TODO: refactor it before next release.
2603      * @param requestingWindow the window from which the request comes
2604      * @param target the name of the target window
2605      * @param request the request to perform
2606      * @param checkHash if true check for hashChenage
2607      * @param forceAttachmentWithFilename if not {@code null} the AttachmentHandler isAttachment() method is not called,
2608      *        the response has to be handled as attachment in any case
2609      * @param description information about the origin of the request. Useful for debugging.
2610      */
2611     public void download(final WebWindow requestingWindow, final String target,
2612         final WebRequest request, final boolean checkHash,
2613         final String forceAttachmentWithFilename, final String description) {
2614 
2615         final WebWindow targetWindow = resolveWindow(requestingWindow, target);
2616         final URL url = request.getUrl();
2617 
2618         if (targetWindow != null && HttpMethod.POST != request.getHttpMethod()) {
2619             final Page page = targetWindow.getEnclosedPage();
2620             if (page != null) {
2621                 if (page.isHtmlPage() && !((HtmlPage) page).isOnbeforeunloadAccepted()) {
2622                     return;
2623                 }
2624 
2625                 if (checkHash) {
2626                     final URL current = page.getUrl();
2627                     final boolean justHashJump =
2628                             HttpMethod.GET == request.getHttpMethod()
2629                             && UrlUtils.sameFile(url, current)
2630                             && null != url.getRef();
2631 
2632                     if (justHashJump) {
2633                         processOnlyHashChange(targetWindow, url);
2634                         return;
2635                     }
2636                 }
2637             }
2638         }
2639 
2640         synchronized (loadQueue_) {
2641             // verify if this load job doesn't already exist
2642             for (final LoadJob otherLoadJob : loadQueue_) {
2643                 if (otherLoadJob.response_ == null) {
2644                     continue;
2645                 }
2646                 final WebRequest otherRequest = otherLoadJob.request_;
2647                 final URL otherUrl = otherRequest.getUrl();
2648 
2649                 if (url.getPath().equals(otherUrl.getPath()) // fail fast
2650                     && url.toString().equals(otherUrl.toString())
2651                     && request.getRequestParameters().equals(otherRequest.getRequestParameters())
2652                     && Objects.equals(request.getRequestBody(), otherRequest.getRequestBody())) {
2653                     return; // skip it;
2654                 }
2655             }
2656         }
2657 
2658         final LoadJob loadJob;
2659         try {
2660             WebResponse response;
2661             try {
2662                 response = loadWebResponse(request);
2663             }
2664             catch (final NoHttpResponseException e) {
2665                 LOG.error("NoHttpResponseException while downloading; generating a NoHttpResponse", e);
2666                 response = new WebResponse(RESPONSE_DATA_NO_HTTP_RESPONSE, request, 0);
2667             }
2668             loadJob = new LoadJob(request, response, requestingWindow, target, forceAttachmentWithFilename);
2669         }
2670         catch (final IOException e) {
2671             throw new RuntimeException(e);
2672         }
2673 
2674         synchronized (loadQueue_) {
2675             loadQueue_.add(loadJob);
2676         }
2677     }
2678 
2679     /**
2680      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2681      *
2682      * Loads downloaded responses into the corresponding windows.
2683      * TODO: refactor it before next release.
2684      * @throws IOException in case of exception
2685      * @throws FailingHttpStatusCodeException in case of exception
2686      */
2687     public void loadDownloadedResponses() throws FailingHttpStatusCodeException, IOException {
2688         final List<LoadJob> queue;
2689 
2690         // synchronize access to the loadQueue_,
2691         // to be sure no job is ignored
2692         synchronized (loadQueue_) {
2693             if (loadQueue_.isEmpty()) {
2694                 return;
2695             }
2696             queue = new ArrayList<>(loadQueue_);
2697             loadQueue_.clear();
2698         }
2699 
2700         final HashSet<WebWindow> updatedWindows = new HashSet<>();
2701         for (int i = queue.size() - 1; i >= 0; --i) {
2702             final LoadJob loadJob = queue.get(i);
2703             if (loadJob.isOutdated()) {
2704                 if (LOG.isInfoEnabled()) {
2705                     LOG.info("No usage of download: " + loadJob);
2706                 }
2707                 continue;
2708             }
2709 
2710             final WebWindow window = resolveWindow(loadJob.requestingWindow_, loadJob.target_);
2711             if (updatedWindows.contains(window)) {
2712                 if (LOG.isInfoEnabled()) {
2713                     LOG.info("No usage of download: " + loadJob);
2714                 }
2715                 continue;
2716             }
2717 
2718             final WebWindow win = openTargetWindow(loadJob.requestingWindow_, loadJob.target_, TARGET_SELF);
2719             final Page pageBeforeLoad = win.getEnclosedPage();
2720             loadWebResponseInto(loadJob.response_, win, loadJob.forceAttachmentWithFilename_);
2721 
2722             // start execution here.
2723             if (scriptEngine_ != null) {
2724                 scriptEngine_.registerWindowAndMaybeStartEventLoop(win);
2725             }
2726 
2727             if (pageBeforeLoad != win.getEnclosedPage()) {
2728                 updatedWindows.add(win);
2729             }
2730 
2731             // check and report problems if needed
2732             throwFailingHttpStatusCodeExceptionIfNecessary(loadJob.response_);
2733         }
2734     }
2735 
2736     private static void processOnlyHashChange(final WebWindow window, final URL urlWithOnlyHashChange) {
2737         final Page page = window.getEnclosedPage();
2738         final String oldURL = page.getUrl().toExternalForm();
2739 
2740         // update request url
2741         final WebRequest req = page.getWebResponse().getWebRequest();
2742         req.setUrl(urlWithOnlyHashChange);
2743 
2744         // update location.hash
2745         final Window jsWindow = window.getScriptableObject();
2746         if (null != jsWindow) {
2747             final Location location = jsWindow.getLocation();
2748             location.setHash(oldURL, urlWithOnlyHashChange.getRef());
2749         }
2750 
2751         // add to history
2752         window.getHistory().addPage(page);
2753     }
2754 
2755     /**
2756      * Returns the options object of this WebClient.
2757      * @return the options object
2758      */
2759     public WebClientOptions getOptions() {
2760         return options_;
2761     }
2762 
2763     /**
2764      * Gets the holder for the different storages.
2765      * <p><span style="color:red">Experimental API: May be changed in next release!</span></p>
2766      * @return the holder
2767      */
2768     public StorageHolder getStorageHolder() {
2769         return storageHolder_;
2770     }
2771 
2772     /**
2773      * Returns the currently configured cookies applicable to the specified URL, in an unmodifiable set.
2774      * If disabled, this returns an empty set.
2775      * @param url the URL on which to filter the returned cookies
2776      * @return the currently configured cookies applicable to the specified URL, in an unmodifiable set
2777      */
2778     public synchronized Set<Cookie> getCookies(final URL url) {
2779         final CookieManager cookieManager = getCookieManager();
2780 
2781         if (!cookieManager.isCookiesEnabled()) {
2782             return Collections.emptySet();
2783         }
2784 
2785         final URL normalizedUrl = HttpClientConverter.replaceForCookieIfNecessary(url);
2786 
2787         final String host = normalizedUrl.getHost();
2788         // URLs like "about:blank" don't have cookies and we need to catch these
2789         // cases here before HttpClient complains
2790         if (host.isEmpty()) {
2791             return Collections.emptySet();
2792         }
2793 
2794         // discard expired cookies
2795         cookieManager.clearExpired(new Date());
2796 
2797         final Set<Cookie> matchingCookies = new LinkedHashSet<>();
2798         HttpClientConverter.addMatching(cookieManager.getCookies(), normalizedUrl,
2799                 getBrowserVersion(), matchingCookies);
2800         return Collections.unmodifiableSet(matchingCookies);
2801     }
2802 
2803     /**
2804      * Parses the given cookie and adds this to our cookie store.
2805      * @param cookieString the string to parse
2806      * @param pageUrl the url of the page that likes to set the cookie
2807      * @param origin the requester
2808      */
2809     public void addCookie(final String cookieString, final URL pageUrl, final Object origin) {
2810         final CookieManager cookieManager = getCookieManager();
2811         if (!cookieManager.isCookiesEnabled()) {
2812             if (LOG.isDebugEnabled()) {
2813                 LOG.debug("Skipped adding cookie: '" + cookieString
2814                         + "' because cookies are not enabled for the CookieManager.");
2815             }
2816             return;
2817         }
2818 
2819         try {
2820             final List<Cookie> cookies = HttpClientConverter.parseCookie(cookieString, pageUrl, getBrowserVersion());
2821 
2822             for (final Cookie cookie : cookies) {
2823                 cookieManager.addCookie(cookie);
2824 
2825                 if (LOG.isDebugEnabled()) {
2826                     LOG.debug("Added cookie: '" + cookieString + "'");
2827                 }
2828             }
2829         }
2830         catch (final MalformedCookieException e) {
2831             if (LOG.isDebugEnabled()) {
2832                 LOG.warn("Adding cookie '" + cookieString + "' failed.", e);
2833             }
2834             getIncorrectnessListener().notify("Adding cookie '" + cookieString
2835                         + "' failed; reason: '" + e.getMessage() + "'.", origin);
2836         }
2837     }
2838 
2839     /**
2840      * Returns true if the javaScript support is enabled.
2841      * To disable the javascript support (eg. temporary)
2842      * you have to use the {@link WebClientOptions#setJavaScriptEnabled(boolean)} setter.
2843      * @see #isJavaScriptEngineEnabled()
2844      * @see WebClientOptions#isJavaScriptEnabled()
2845      * @return true if the javaScript engine and the javaScript support is enabled.
2846      */
2847     public boolean isJavaScriptEnabled() {
2848         return javaScriptEngineEnabled_ && getOptions().isJavaScriptEnabled();
2849     }
2850 
2851     /**
2852      * Returns true if the javaScript engine is enabled.
2853      * To disable the javascript engine you have to use the
2854      * {@link WebClient#WebClient(BrowserVersion, boolean, String, int)} constructor.
2855      * @return true if the javaScript engine is enabled.
2856      */
2857     public boolean isJavaScriptEngineEnabled() {
2858         return javaScriptEngineEnabled_;
2859     }
2860 
2861     /**
2862      * Parses the given XHtml code string and loads the resulting XHtmlPage into
2863      * the current window.
2864      *
2865      * @param htmlCode the html code as string
2866      * @return the HtmlPage
2867      * @throws IOException in case of error
2868      */
2869     public HtmlPage loadHtmlCodeIntoCurrentWindow(final String htmlCode) throws IOException {
2870         final HTMLParser htmlParser = getPageCreator().getHtmlParser();
2871         final WebWindow webWindow = getCurrentWindow();
2872 
2873         final StringWebResponse webResponse =
2874                 new StringWebResponse(htmlCode, new URL("https://www.htmlunit.org/dummy.html"));
2875         final HtmlPage page = new HtmlPage(webResponse, webWindow);
2876         webWindow.setEnclosedPage(page);
2877 
2878         htmlParser.parse(this, webResponse, page, false, false);
2879         return page;
2880     }
2881 
2882     /**
2883      * Parses the given XHtml code string and loads the resulting XHtmlPage into
2884      * the current window.
2885      *
2886      * @param xhtmlCode the xhtml code as string
2887      * @return the XHtmlPage
2888      * @throws IOException in case of error
2889      */
2890     public XHtmlPage loadXHtmlCodeIntoCurrentWindow(final String xhtmlCode) throws IOException {
2891         final HTMLParser htmlParser = getPageCreator().getHtmlParser();
2892         final WebWindow webWindow = getCurrentWindow();
2893 
2894         final StringWebResponse webResponse =
2895                 new StringWebResponse(xhtmlCode, new URL("https://www.htmlunit.org/dummy.html"));
2896         final XHtmlPage page = new XHtmlPage(webResponse, webWindow);
2897         webWindow.setEnclosedPage(page);
2898 
2899         htmlParser.parse(this, webResponse, page, true, false);
2900         return page;
2901     }
2902 
2903     /**
2904      * Creates a new {@link WebSocketAdapter}.
2905      *
2906      * @param webSocketListener the {@link WebSocketListener}
2907      * @return a new {@link WebSocketAdapter}
2908      */
2909     public WebSocketAdapter buildWebSocketAdapter(final WebSocketListener webSocketListener) {
2910         return webSocketAdapterFactory_.buildWebSocketAdapter(this, webSocketListener);
2911     }
2912 
2913     /**
2914      * Defines a new factory method to create a new WebSocketAdapter.
2915      *
2916      * @param factory a {@link WebSocketAdapterFactory}
2917      */
2918     public void setWebSocketAdapter(final WebSocketAdapterFactory factory) {
2919         webSocketAdapterFactory_ = factory;
2920     }
2921 
2922     /**
2923      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2924      *
2925      * @return a CSS3Parser that will return to an internal pool for reuse if closed using the
2926      *         try-with-resource concept
2927      */
2928     public PooledCSS3Parser getCSS3Parser() {
2929         return this.css3ParserPool_.get();
2930     }
2931 
2932     /**
2933      * Our pool of CSS3Parsers. If you need a parser, get it from here and use the AutoCloseable
2934      * functionality with a try-with-resource block. If you don't want to do that at all, continue
2935      * to build CSS3Parsers the old fashioned way.
2936      *
2937      * Fetching a parser is thread safe. This API is built to minimize synchronization overhead,
2938      * hence it is possible to miss a returned parser from another thread under heavy pressure,
2939      * but because that is unlikely, we keep it simple and efficient. Caches are not supposed
2940      * to give cutting-edge guarantees.
2941      *
2942      * This concept avoids a resource leak when someone does not close the fetched
2943      * parser because the pool does not know anything about the parser unless
2944      * it returns. We are not running a checkout-checkin concept.
2945      *
2946      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2947      */
2948     static class CSS3ParserPool {
2949         /*
2950          * Our pool. We only hold data when it is available. In addition, synchronization against
2951          * this deque is cheap.
2952          */
2953         private final ConcurrentLinkedDeque<PooledCSS3Parser> parsers_ = new ConcurrentLinkedDeque<>();
2954 
2955         /**
2956          * Fetch a new or recycled CSS3parser. Make sure you use the try-with-resource concept
2957          * to automatically return it after use because a parser creation is expensive.
2958          * We won't get a leak, if you don't do so, but that will remove the advantage.
2959          *
2960          * @return a parser
2961          */
2962         public PooledCSS3Parser get() {
2963             // see if we have one, LIFO
2964             final PooledCSS3Parser parser = parsers_.pollLast();
2965 
2966             // if we don't have one, get us one
2967             return parser != null ? parser.markInUse(this) : new PooledCSS3Parser(this);
2968         }
2969 
2970         /**
2971          * Return a parser. Normally you don't have to use that method explicitly.
2972          * Prefer to user the AutoCloseable interface of the PooledParser by
2973          * using a try-with-resource statement.
2974          *
2975          * @param parser the parser to recycle
2976          */
2977         protected void recycle(final PooledCSS3Parser parser) {
2978             parsers_.addLast(parser);
2979         }
2980     }
2981 
2982     /**
2983      * This is a poolable CSS3Parser which can be reused automatically when closed.
2984      * A regular CSS3Parser is not thread-safe, hence also our pooled parser
2985      * is not thread-safe.
2986      *
2987      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2988      */
2989     public static class PooledCSS3Parser extends CSS3Parser implements AutoCloseable {
2990         /**
2991          * The pool we want to return us to. Because multiple threads can use this, we
2992          * have to ensure that we see the action here.
2993          */
2994         private CSS3ParserPool pool_;
2995 
2996         /**
2997          * Create a new poolable parser.
2998          *
2999          * @param pool the pool the parser should return to when it is closed
3000          */
3001         protected PooledCSS3Parser(final CSS3ParserPool pool) {
3002             super();
3003             this.pool_ = pool;
3004         }
3005 
3006         /**
3007          * Resets the parser's pool state so it can be safely returned again.
3008          *
3009          * @param pool the pool the parser should return to when it is closed
3010          * @return this parser for fluid programming
3011          */
3012         protected PooledCSS3Parser markInUse(final CSS3ParserPool pool) {
3013             // ensure we detect programming mistakes
3014             if (this.pool_ == null) {
3015                 this.pool_ = pool;
3016             }
3017             else {
3018                 throw new IllegalStateException("This PooledParser was not returned to the pool properly");
3019             }
3020 
3021             return this;
3022         }
3023 
3024         /**
3025          * Implements the AutoClosable interface. The return method ensures that
3026          * we are notified when we incorrectly close it twice which indicates a
3027          * programming flow defect.
3028          *
3029          * @throws IllegalStateException in case the parser is closed several times
3030          */
3031         @Override
3032         public void close() {
3033             if (this.pool_ != null) {
3034                 final CSS3ParserPool oldPool = this.pool_;
3035                 // set null first and recycle later to avoid exposing a broken state
3036                 // volatile guarantees visibility
3037                 this.pool_ = null;
3038 
3039                 // return
3040                 oldPool.recycle(this);
3041             }
3042             else {
3043                 throw new IllegalStateException("This PooledParser was returned already");
3044             }
3045         }
3046     }
3047 }