1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package org.htmlunit;
16
17 import static java.nio.charset.StandardCharsets.ISO_8859_1;
18 import static java.nio.charset.StandardCharsets.UTF_8;
19 import static org.htmlunit.BrowserVersionFeatures.HTTP_HEADER_CH_UA;
20 import static org.htmlunit.BrowserVersionFeatures.HTTP_HEADER_PRIORITY;
21
22 import java.io.BufferedInputStream;
23 import java.io.File;
24 import java.io.IOException;
25 import java.io.InputStream;
26 import java.io.ObjectInputStream;
27 import java.io.Serializable;
28 import java.lang.ref.WeakReference;
29 import java.net.MalformedURLException;
30 import java.net.URL;
31 import java.net.URLConnection;
32 import java.net.URLDecoder;
33 import java.nio.charset.Charset;
34 import java.nio.file.Files;
35 import java.util.ArrayList;
36 import java.util.Collections;
37 import java.util.ConcurrentModificationException;
38 import java.util.Date;
39 import java.util.HashMap;
40 import java.util.HashSet;
41 import java.util.Iterator;
42 import java.util.LinkedHashMap;
43 import java.util.LinkedHashSet;
44 import java.util.List;
45 import java.util.Locale;
46 import java.util.Map;
47 import java.util.Objects;
48 import java.util.Optional;
49 import java.util.Set;
50 import java.util.concurrent.ConcurrentLinkedDeque;
51 import java.util.concurrent.Executor;
52 import java.util.concurrent.ExecutorService;
53 import java.util.concurrent.Executors;
54 import java.util.concurrent.ThreadFactory;
55 import java.util.concurrent.ThreadPoolExecutor;
56
57 import org.apache.commons.lang3.StringUtils;
58 import org.apache.commons.logging.Log;
59 import org.apache.commons.logging.LogFactory;
60 import org.apache.http.NoHttpResponseException;
61 import org.apache.http.client.CredentialsProvider;
62 import org.apache.http.cookie.MalformedCookieException;
63 import org.htmlunit.attachment.Attachment;
64 import org.htmlunit.attachment.AttachmentHandler;
65 import org.htmlunit.csp.Policy;
66 import org.htmlunit.csp.url.URI;
67 import org.htmlunit.css.ComputedCssStyleDeclaration;
68 import org.htmlunit.cssparser.parser.CSSErrorHandler;
69 import org.htmlunit.cssparser.parser.javacc.CSS3Parser;
70 import org.htmlunit.html.BaseFrameElement;
71 import org.htmlunit.html.DomElement;
72 import org.htmlunit.html.DomNode;
73 import org.htmlunit.html.FrameWindow;
74 import org.htmlunit.html.FrameWindow.PageDenied;
75 import org.htmlunit.html.HtmlElement;
76 import org.htmlunit.html.HtmlInlineFrame;
77 import org.htmlunit.html.HtmlPage;
78 import org.htmlunit.html.XHtmlPage;
79 import org.htmlunit.html.parser.HTMLParser;
80 import org.htmlunit.html.parser.HTMLParserListener;
81 import org.htmlunit.http.HttpStatus;
82 import org.htmlunit.http.HttpUtils;
83 import org.htmlunit.httpclient.HttpClientConverter;
84 import org.htmlunit.javascript.AbstractJavaScriptEngine;
85 import org.htmlunit.javascript.DefaultJavaScriptErrorListener;
86 import org.htmlunit.javascript.HtmlUnitScriptable;
87 import org.htmlunit.javascript.JavaScriptEngine;
88 import org.htmlunit.javascript.JavaScriptErrorListener;
89 import org.htmlunit.javascript.background.JavaScriptJobManager;
90 import org.htmlunit.javascript.host.Location;
91 import org.htmlunit.javascript.host.Window;
92 import org.htmlunit.javascript.host.dom.Node;
93 import org.htmlunit.javascript.host.event.Event;
94 import org.htmlunit.javascript.host.file.Blob;
95 import org.htmlunit.javascript.host.html.HTMLIFrameElement;
96 import org.htmlunit.protocol.data.DataURLConnection;
97 import org.htmlunit.util.Cookie;
98 import org.htmlunit.util.HeaderUtils;
99 import org.htmlunit.util.MimeType;
100 import org.htmlunit.util.NameValuePair;
101 import org.htmlunit.util.UrlUtils;
102 import org.htmlunit.websocket.JettyWebSocketAdapter.JettyWebSocketAdapterFactory;
103 import org.htmlunit.websocket.WebSocketAdapter;
104 import org.htmlunit.websocket.WebSocketAdapterFactory;
105 import org.htmlunit.websocket.WebSocketListener;
106 import org.htmlunit.webstart.WebStartHandler;
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154 @SuppressWarnings("PMD.TooManyFields")
155 public class WebClient implements Serializable, AutoCloseable {
156
157
158 private static final Log LOG = LogFactory.getLog(WebClient.class);
159
160
161 private static final int ALLOWED_REDIRECTIONS_SAME_URL = 20;
162 private static final WebResponseData RESPONSE_DATA_NO_HTTP_RESPONSE = new WebResponseData(
163 0, "No HTTP Response", Collections.emptyList());
164
165
166
167
168
169 private static final String[] DISCARDING_304_RESPONSE_HEADER_NAMES = {
170 "connection",
171 "proxy-connection",
172 "keep-alive",
173 "www-authenticate",
174 "proxy-authenticate",
175 "proxy-authorization",
176 "te",
177 "trailer",
178 "transfer-encoding",
179 "upgrade",
180 "content-location",
181 "content-md5",
182 "etag",
183 "content-encoding",
184 "content-range",
185 "content-type",
186 "content-length",
187 "x-frame-options",
188 "x-xss-protection",
189 };
190
191 private static final String[] DISCARDING_304_HEADER_PREFIXES = {
192 "x-content-",
193 "x-webkit-"
194 };
195
196 private transient WebConnection webConnection_;
197 private CredentialsProvider credentialsProvider_ = new DefaultCredentialsProvider();
198 private CookieManager cookieManager_ = new CookieManager();
199 private WebSocketAdapterFactory webSocketAdapterFactory_;
200 private transient AbstractJavaScriptEngine<?> scriptEngine_;
201 private transient List<LoadJob> loadQueue_;
202 private final Map<String, String> requestHeaders_ = Collections.synchronizedMap(new HashMap<>(89));
203 private IncorrectnessListener incorrectnessListener_ = new IncorrectnessListenerImpl();
204 private WebConsole webConsole_;
205 private transient ExecutorService executor_;
206
207 private AlertHandler alertHandler_;
208 private ConfirmHandler confirmHandler_;
209 private PromptHandler promptHandler_;
210 private StatusHandler statusHandler_;
211 private AttachmentHandler attachmentHandler_;
212 private ClipboardHandler clipboardHandler_;
213 private PrintHandler printHandler_;
214 private WebStartHandler webStartHandler_;
215 private FrameContentHandler frameContentHandler_;
216
217 private AjaxController ajaxController_ = new AjaxController();
218
219 private final BrowserVersion browserVersion_;
220 private PageCreator pageCreator_ = new DefaultPageCreator();
221
222
223
224
225 private CurrentWindowTracker currentWindowTracker_;
226 private final Set<WebWindowListener> webWindowListeners_ = new HashSet<>(5);
227
228 private final List<TopLevelWindow> topLevelWindows_ =
229 Collections.synchronizedList(new ArrayList<>());
230 private final List<WebWindow> windows_ = Collections.synchronizedList(new ArrayList<>());
231 private transient List<WeakReference<JavaScriptJobManager>> jobManagers_ =
232 Collections.synchronizedList(new ArrayList<>());
233 private WebWindow currentWindow_;
234
235 private HTMLParserListener htmlParserListener_;
236 private CSSErrorHandler cssErrorHandler_ = new DefaultCssErrorHandler();
237 private OnbeforeunloadHandler onbeforeunloadHandler_;
238 private Cache cache_ = new Cache();
239
240
241 private transient CSS3ParserPool css3ParserPool_ = new CSS3ParserPool();
242
243
244 public static final String TARGET_BLANK = "_blank";
245
246
247 public static final String TARGET_SELF = "_self";
248
249
250 private static final String TARGET_PARENT = "_parent";
251
252 private static final String TARGET_TOP = "_top";
253
254 private ScriptPreProcessor scriptPreProcessor_;
255
256 private RefreshHandler refreshHandler_ = new NiceRefreshHandler(2);
257 private JavaScriptErrorListener javaScriptErrorListener_ = new DefaultJavaScriptErrorListener();
258
259 private final WebClientOptions options_ = new WebClientOptions();
260 private final boolean javaScriptEngineEnabled_;
261 private final StorageHolder storageHolder_ = new StorageHolder();
262
263
264
265
266
267 public WebClient() {
268 this(BrowserVersion.getDefault());
269 }
270
271
272
273
274
275 public WebClient(final BrowserVersion browserVersion) {
276 this(browserVersion, null, -1);
277 }
278
279
280
281
282
283
284
285 public WebClient(final BrowserVersion browserVersion, final String proxyHost, final int proxyPort) {
286 this(browserVersion, true, proxyHost, proxyPort, null);
287 }
288
289
290
291
292
293
294
295
296 public WebClient(final BrowserVersion browserVersion,
297 final String proxyHost, final int proxyPort, final String proxyScheme) {
298 this(browserVersion, true, proxyHost, proxyPort, proxyScheme);
299 }
300
301
302
303
304
305
306
307
308 public WebClient(final BrowserVersion browserVersion, final boolean javaScriptEngineEnabled,
309 final String proxyHost, final int proxyPort) {
310 this(browserVersion, javaScriptEngineEnabled, proxyHost, proxyPort, null);
311 }
312
313
314
315
316
317
318
319
320
321 public WebClient(final BrowserVersion browserVersion, final boolean javaScriptEngineEnabled,
322 final String proxyHost, final int proxyPort, final String proxyScheme) {
323 WebAssert.notNull("browserVersion", browserVersion);
324
325 browserVersion_ = browserVersion;
326 javaScriptEngineEnabled_ = javaScriptEngineEnabled;
327
328 if (proxyHost == null) {
329 getOptions().setProxyConfig(new ProxyConfig());
330 }
331 else {
332 getOptions().setProxyConfig(new ProxyConfig(proxyHost, proxyPort, proxyScheme));
333 }
334
335 webConnection_ = new HttpWebConnection(this);
336 if (javaScriptEngineEnabled_) {
337 scriptEngine_ = new JavaScriptEngine(this);
338 }
339 loadQueue_ = new ArrayList<>();
340
341 webSocketAdapterFactory_ = new JettyWebSocketAdapterFactory();
342
343
344 currentWindowTracker_ = new CurrentWindowTracker(this, true);
345 currentWindow_ = new TopLevelWindow("", this);
346 }
347
348
349
350
351
352 private static final class ThreadNamingFactory implements ThreadFactory {
353 private static int ID_ = 1;
354 private final ThreadFactory baseFactory_;
355
356 ThreadNamingFactory(final ThreadFactory aBaseFactory) {
357 baseFactory_ = aBaseFactory;
358 }
359
360 @Override
361 public Thread newThread(final Runnable aRunnable) {
362 final Thread thread = baseFactory_.newThread(aRunnable);
363 thread.setName("WebClient Thread " + ID_++);
364 return thread;
365 }
366 }
367
368
369
370
371
372
373 public WebConnection getWebConnection() {
374 return webConnection_;
375 }
376
377
378
379
380
381
382 public void setWebConnection(final WebConnection webConnection) {
383 WebAssert.notNull("webConnection", webConnection);
384 webConnection_ = webConnection;
385 }
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408 public <P extends Page> P getPage(final WebWindow webWindow, final WebRequest webRequest)
409 throws IOException, FailingHttpStatusCodeException {
410 return getPage(webWindow, webRequest, true);
411 }
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437 @SuppressWarnings("unchecked")
438 <P extends Page> P getPage(final WebWindow webWindow, final WebRequest webRequest,
439 final boolean addToHistory)
440 throws IOException, FailingHttpStatusCodeException {
441
442 final Page page = webWindow.getEnclosedPage();
443
444 if (page != null) {
445 final URL prev = page.getUrl();
446 final URL current = webRequest.getUrl();
447 if (UrlUtils.sameFile(current, prev)
448 && current.getRef() != null
449 && !Objects.equals(current.getRef(), prev.getRef())) {
450
451 page.getWebResponse().getWebRequest().setUrl(current);
452 if (addToHistory) {
453 webWindow.getHistory().addPage(page);
454 }
455
456
457
458 if (page instanceof HtmlPage) {
459 ((HtmlPage) page).clearComputedStyles();
460 }
461
462 final Window window = webWindow.getScriptableObject();
463 if (window != null) {
464 window.getLocation().setHash(current.getRef());
465 }
466 return (P) page;
467 }
468
469 if (page.isHtmlPage()) {
470 final HtmlPage htmlPage = (HtmlPage) page;
471 if (!htmlPage.isOnbeforeunloadAccepted()) {
472 LOG.debug("The registered OnbeforeunloadHandler rejected to load a new page.");
473 return (P) page;
474 }
475 }
476 }
477
478 if (LOG.isDebugEnabled()) {
479 LOG.debug("Get page for window named '" + webWindow.getName() + "', using " + webRequest);
480 }
481
482 WebResponse webResponse;
483 final String protocol = webRequest.getUrl().getProtocol();
484 if ("javascript".equals(protocol)) {
485 webResponse = makeWebResponseForJavaScriptUrl(webWindow, webRequest.getUrl(), webRequest.getCharset());
486 if (webWindow.getEnclosedPage() != null && webWindow.getEnclosedPage().getWebResponse() == webResponse) {
487
488 return (P) webWindow.getEnclosedPage();
489 }
490 }
491 else {
492 try {
493 webResponse = loadWebResponse(webRequest);
494 }
495 catch (final NoHttpResponseException e) {
496 webResponse = new WebResponse(RESPONSE_DATA_NO_HTTP_RESPONSE, webRequest, 0);
497 }
498 }
499
500 printContentIfNecessary(webResponse);
501 loadWebResponseInto(webResponse, webWindow);
502
503
504
505
506 if (scriptEngine_ != null) {
507 scriptEngine_.registerWindowAndMaybeStartEventLoop(webWindow);
508 }
509
510
511 throwFailingHttpStatusCodeExceptionIfNecessary(webResponse);
512 return (P) webWindow.getEnclosedPage();
513 }
514
515
516
517
518
519
520
521
522
523
524
525
526
527 public <P extends Page> P getPage(final String url) throws IOException, FailingHttpStatusCodeException,
528 MalformedURLException {
529 return getPage(UrlUtils.toUrlUnsafe(url));
530 }
531
532
533
534
535
536
537
538
539
540
541
542
543 public <P extends Page> P getPage(final URL url) throws IOException, FailingHttpStatusCodeException {
544 final WebRequest request = new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader(),
545 getBrowserVersion().getAcceptEncodingHeader());
546 request.setCharset(UTF_8);
547 return getPage(getCurrentWindow().getTopWindow(), request);
548 }
549
550
551
552
553
554
555
556
557
558
559
560 public <P extends Page> P getPage(final WebRequest request) throws IOException,
561 FailingHttpStatusCodeException {
562 return getPage(getCurrentWindow().getTopWindow(), request);
563 }
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582 public Page loadWebResponseInto(final WebResponse webResponse, final WebWindow webWindow)
583 throws IOException, FailingHttpStatusCodeException {
584 return loadWebResponseInto(webResponse, webWindow, null);
585 }
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608 public Page loadWebResponseInto(final WebResponse webResponse, final WebWindow webWindow,
609 String forceAttachmentWithFilename)
610 throws IOException, FailingHttpStatusCodeException {
611 WebAssert.notNull("webResponse", webResponse);
612 WebAssert.notNull("webWindow", webWindow);
613
614 if (webResponse.getStatusCode() == HttpStatus.NO_CONTENT_204) {
615 return webWindow.getEnclosedPage();
616 }
617
618 if (webStartHandler_ != null && "application/x-java-jnlp-file".equals(webResponse.getContentType())) {
619 webStartHandler_.handleJnlpResponse(webResponse);
620 return webWindow.getEnclosedPage();
621 }
622
623 if (attachmentHandler_ != null
624 && (forceAttachmentWithFilename != null || attachmentHandler_.isAttachment(webResponse))) {
625
626
627 if (StringUtils.isEmpty(forceAttachmentWithFilename)) {
628 final String disp = webResponse.getResponseHeaderValue(HttpHeader.CONTENT_DISPOSITION);
629 forceAttachmentWithFilename = Attachment.getSuggestedFilename(disp);
630 }
631
632 if (attachmentHandler_.handleAttachment(webResponse,
633 StringUtils.isEmpty(forceAttachmentWithFilename) ? null : forceAttachmentWithFilename)) {
634
635
636 return webWindow.getEnclosedPage();
637 }
638
639 final WebWindow w = openWindow(null, null, webWindow);
640 final Page page = pageCreator_.createPage(webResponse, w);
641 attachmentHandler_.handleAttachment(page,
642 StringUtils.isEmpty(forceAttachmentWithFilename) ? null : forceAttachmentWithFilename);
643 return page;
644 }
645
646 final Page oldPage = webWindow.getEnclosedPage();
647 if (oldPage != null) {
648
649 oldPage.cleanUp();
650 }
651
652 Page newPage = null;
653 FrameWindow.PageDenied pageDenied = PageDenied.NONE;
654 if (windows_.contains(webWindow)) {
655 if (webWindow instanceof FrameWindow) {
656 final String contentSecurityPolicy =
657 webResponse.getResponseHeaderValue(HttpHeader.CONTENT_SECURIRY_POLICY);
658 if (StringUtils.isNotBlank(contentSecurityPolicy)) {
659 final URL origin = UrlUtils.getUrlWithoutPathRefQuery(
660 ((FrameWindow) webWindow).getEnclosingPage().getUrl());
661 final URL source = UrlUtils.getUrlWithoutPathRefQuery(webResponse.getWebRequest().getUrl());
662 final Policy policy = Policy.parseSerializedCSP(contentSecurityPolicy,
663 Policy.PolicyErrorConsumer.ignored);
664 if (!policy.allowsFrameAncestor(
665 Optional.of(URI.parseURI(source.toExternalForm()).orElse(null)),
666 Optional.of(URI.parseURI(origin.toExternalForm()).orElse(null)))) {
667 pageDenied = PageDenied.BY_CONTENT_SECURIRY_POLICY;
668
669 if (LOG.isWarnEnabled()) {
670 LOG.warn("Load denied by Content-Security-Policy: '" + contentSecurityPolicy + "' - "
671 + webResponse.getWebRequest().getUrl() + "' does not permit framing.");
672 }
673 }
674 }
675
676 if (pageDenied == PageDenied.NONE) {
677 final String xFrameOptions = webResponse.getResponseHeaderValue(HttpHeader.X_FRAME_OPTIONS);
678 if ("DENY".equalsIgnoreCase(xFrameOptions)) {
679 pageDenied = PageDenied.BY_X_FRAME_OPTIONS;
680
681 if (LOG.isWarnEnabled()) {
682 LOG.warn("Load denied by X-Frame-Options: DENY; - '"
683 + webResponse.getWebRequest().getUrl() + "' does not permit framing.");
684 }
685 }
686 }
687 }
688
689 if (pageDenied == PageDenied.NONE) {
690 newPage = pageCreator_.createPage(webResponse, webWindow);
691 }
692 else {
693 try {
694 final WebResponse aboutBlank = loadWebResponse(WebRequest.newAboutBlankRequest());
695 newPage = pageCreator_.createPage(aboutBlank, webWindow);
696
697
698 ((FrameWindow) webWindow).setPageDenied(pageDenied);
699 }
700 catch (final IOException ignored) {
701
702 }
703 }
704
705 if (windows_.contains(webWindow)) {
706 fireWindowContentChanged(new WebWindowEvent(webWindow, WebWindowEvent.CHANGE, oldPage, newPage));
707
708
709 if (webWindow.getEnclosedPage() == newPage) {
710 newPage.initialize();
711
712
713 if (isJavaScriptEnabled()
714 && webWindow instanceof FrameWindow && !newPage.isHtmlPage()) {
715 final FrameWindow fw = (FrameWindow) webWindow;
716 final BaseFrameElement frame = fw.getFrameElement();
717 if (frame.hasEventHandlers("onload")) {
718 if (LOG.isDebugEnabled()) {
719 LOG.debug("Executing onload handler for " + frame);
720 }
721 final Event event = new Event(frame, Event.TYPE_LOAD);
722 ((Node) frame.getScriptableObject()).executeEventLocally(event);
723 }
724 }
725 }
726 }
727 }
728 return newPage;
729 }
730
731
732
733
734
735
736
737
738
739 public void printContentIfNecessary(final WebResponse webResponse) {
740 if (getOptions().isPrintContentOnFailingStatusCode()
741 && !webResponse.isSuccess() && LOG.isInfoEnabled()) {
742 final String contentType = webResponse.getContentType();
743 LOG.info("statusCode=[" + webResponse.getStatusCode() + "] contentType=[" + contentType + "]");
744 LOG.info(webResponse.getContentAsString());
745 }
746 }
747
748
749
750
751
752
753
754
755
756 public void throwFailingHttpStatusCodeExceptionIfNecessary(final WebResponse webResponse) {
757 if (getOptions().isThrowExceptionOnFailingStatusCode() && !webResponse.isSuccessOrUseProxyOrNotModified()) {
758 throw new FailingHttpStatusCodeException(webResponse);
759 }
760 }
761
762
763
764
765
766
767
768
769
770 public void addRequestHeader(final String name, final String value) {
771 if (HttpHeader.COOKIE_LC.equalsIgnoreCase(name)) {
772 throw new IllegalArgumentException("Do not add 'Cookie' header, use .getCookieManager() instead");
773 }
774 requestHeaders_.put(name, value);
775 }
776
777
778
779
780
781
782
783
784
785
786 public void removeRequestHeader(final String name) {
787 requestHeaders_.remove(name);
788 }
789
790
791
792
793
794
795
796
797 public void setCredentialsProvider(final CredentialsProvider credentialsProvider) {
798 WebAssert.notNull("credentialsProvider", credentialsProvider);
799 credentialsProvider_ = credentialsProvider;
800 }
801
802
803
804
805
806
807 public CredentialsProvider getCredentialsProvider() {
808 return credentialsProvider_;
809 }
810
811
812
813
814
815 public AbstractJavaScriptEngine<?> getJavaScriptEngine() {
816 return scriptEngine_;
817 }
818
819
820
821
822
823
824 public void setJavaScriptEngine(final AbstractJavaScriptEngine<?> engine) {
825 if (engine == null) {
826 throw new IllegalArgumentException("Can't set JavaScriptEngine to null");
827 }
828 scriptEngine_ = engine;
829 }
830
831
832
833
834
835 public CookieManager getCookieManager() {
836 return cookieManager_;
837 }
838
839
840
841
842
843 public void setCookieManager(final CookieManager cookieManager) {
844 WebAssert.notNull("cookieManager", cookieManager);
845 cookieManager_ = cookieManager;
846 }
847
848
849
850
851
852 public void setAlertHandler(final AlertHandler alertHandler) {
853 alertHandler_ = alertHandler;
854 }
855
856
857
858
859
860 public AlertHandler getAlertHandler() {
861 return alertHandler_;
862 }
863
864
865
866
867
868 public void setConfirmHandler(final ConfirmHandler handler) {
869 confirmHandler_ = handler;
870 }
871
872
873
874
875
876 public ConfirmHandler getConfirmHandler() {
877 return confirmHandler_;
878 }
879
880
881
882
883
884 public void setPromptHandler(final PromptHandler handler) {
885 promptHandler_ = handler;
886 }
887
888
889
890
891
892 public PromptHandler getPromptHandler() {
893 return promptHandler_;
894 }
895
896
897
898
899
900 public void setStatusHandler(final StatusHandler statusHandler) {
901 statusHandler_ = statusHandler;
902 }
903
904
905
906
907
908 public StatusHandler getStatusHandler() {
909 return statusHandler_;
910 }
911
912
913
914
915
916 public synchronized Executor getExecutor() {
917 if (executor_ == null) {
918 final ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) Executors.newCachedThreadPool();
919 threadPoolExecutor.setThreadFactory(new ThreadNamingFactory(threadPoolExecutor.getThreadFactory()));
920
921 executor_ = threadPoolExecutor;
922 }
923
924 return executor_;
925 }
926
927
928
929
930
931
932
933 public synchronized void setExecutor(final ExecutorService executor) {
934 if (executor_ != null) {
935 throw new IllegalStateException("Can't change the executor after first use.");
936 }
937
938 executor_ = executor;
939 }
940
941
942
943
944
945
946 public void setJavaScriptErrorListener(final JavaScriptErrorListener javaScriptErrorListener) {
947 if (javaScriptErrorListener == null) {
948 javaScriptErrorListener_ = new DefaultJavaScriptErrorListener();
949 }
950 else {
951 javaScriptErrorListener_ = javaScriptErrorListener;
952 }
953 }
954
955
956
957
958
959 public JavaScriptErrorListener getJavaScriptErrorListener() {
960 return javaScriptErrorListener_;
961 }
962
963
964
965
966
967 public BrowserVersion getBrowserVersion() {
968 return browserVersion_;
969 }
970
971
972
973
974
975
976 public WebWindow getCurrentWindow() {
977 return currentWindow_;
978 }
979
980
981
982
983
984
985 public void setCurrentWindow(final WebWindow window) {
986 WebAssert.notNull("window", window);
987 if (currentWindow_ == window) {
988 return;
989 }
990
991 if (currentWindow_ != null && !currentWindow_.isClosed()) {
992 final Page enclosedPage = currentWindow_.getEnclosedPage();
993 if (enclosedPage != null && enclosedPage.isHtmlPage()) {
994 final DomElement focusedElement = ((HtmlPage) enclosedPage).getFocusedElement();
995 if (focusedElement != null) {
996 focusedElement.fireEvent(Event.TYPE_BLUR);
997 }
998 }
999 }
1000 currentWindow_ = window;
1001
1002
1003 final boolean isIFrame = currentWindow_ instanceof FrameWindow
1004 && ((FrameWindow) currentWindow_).getFrameElement() instanceof HtmlInlineFrame;
1005 if (!isIFrame) {
1006
1007
1008 final Page enclosedPage = currentWindow_.getEnclosedPage();
1009 if (enclosedPage != null && enclosedPage.isHtmlPage()) {
1010 final HtmlPage enclosedHtmlPage = (HtmlPage) enclosedPage;
1011 final HtmlElement activeElement = enclosedHtmlPage.getActiveElement();
1012 if (activeElement != null) {
1013 enclosedHtmlPage.setFocusedElement(activeElement, true);
1014 }
1015 }
1016 }
1017 }
1018
1019
1020
1021
1022
1023
1024 public void addWebWindowListener(final WebWindowListener listener) {
1025 WebAssert.notNull("listener", listener);
1026 webWindowListeners_.add(listener);
1027 }
1028
1029
1030
1031
1032
1033 public void removeWebWindowListener(final WebWindowListener listener) {
1034 WebAssert.notNull("listener", listener);
1035 webWindowListeners_.remove(listener);
1036 }
1037
1038 private void fireWindowContentChanged(final WebWindowEvent event) {
1039 if (currentWindowTracker_ != null) {
1040 currentWindowTracker_.webWindowContentChanged(event);
1041 }
1042 for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
1043 listener.webWindowContentChanged(event);
1044 }
1045 }
1046
1047 private void fireWindowOpened(final WebWindowEvent event) {
1048 if (currentWindowTracker_ != null) {
1049 currentWindowTracker_.webWindowOpened(event);
1050 }
1051 for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
1052 listener.webWindowOpened(event);
1053 }
1054 }
1055
1056 private void fireWindowClosed(final WebWindowEvent event) {
1057 if (currentWindowTracker_ != null) {
1058 currentWindowTracker_.webWindowClosed(event);
1059 }
1060
1061 for (final WebWindowListener listener : new ArrayList<>(webWindowListeners_)) {
1062 listener.webWindowClosed(event);
1063 }
1064
1065
1066 if (currentWindowTracker_ != null) {
1067 currentWindowTracker_.afterWebWindowClosedListenersProcessed(event);
1068 }
1069 }
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079 public WebWindow openWindow(final URL url, final String windowName) {
1080 WebAssert.notNull("windowName", windowName);
1081 return openWindow(url, windowName, getCurrentWindow());
1082 }
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093 public WebWindow openWindow(final URL url, final String windowName, final WebWindow opener) {
1094 final WebWindow window = openTargetWindow(opener, windowName, TARGET_BLANK);
1095 if (url == null) {
1096 initializeEmptyWindow(window, window.getEnclosedPage());
1097 }
1098 else {
1099 try {
1100 final WebRequest request = new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader(),
1101 getBrowserVersion().getAcceptEncodingHeader());
1102 request.setCharset(UTF_8);
1103
1104 final Page openerPage = opener.getEnclosedPage();
1105 if (openerPage != null && openerPage.getUrl() != null) {
1106 request.setRefererHeader(openerPage.getUrl());
1107 }
1108 getPage(window, request);
1109 }
1110 catch (final IOException e) {
1111 LOG.error("Error loading content into window", e);
1112 }
1113 }
1114 return window;
1115 }
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130 public WebWindow openTargetWindow(
1131 final WebWindow opener, final String windowName, final String defaultName) {
1132
1133 WebAssert.notNull("opener", opener);
1134 WebAssert.notNull("defaultName", defaultName);
1135
1136 String windowToOpen = windowName;
1137 if (windowToOpen == null || windowToOpen.isEmpty()) {
1138 windowToOpen = defaultName;
1139 }
1140
1141 WebWindow webWindow = resolveWindow(opener, windowToOpen);
1142
1143 if (webWindow == null) {
1144 if (TARGET_BLANK.equals(windowToOpen)) {
1145 windowToOpen = "";
1146 }
1147 webWindow = new TopLevelWindow(windowToOpen, this);
1148 }
1149
1150 if (webWindow instanceof TopLevelWindow && webWindow != opener.getTopWindow()) {
1151 ((TopLevelWindow) webWindow).setOpener(opener);
1152 }
1153
1154 return webWindow;
1155 }
1156
1157 private WebWindow resolveWindow(final WebWindow opener, final String name) {
1158 if (name == null || name.isEmpty() || TARGET_SELF.equals(name)) {
1159 return opener;
1160 }
1161
1162 if (TARGET_PARENT.equals(name)) {
1163 return opener.getParentWindow();
1164 }
1165
1166 if (TARGET_TOP.equals(name)) {
1167 return opener.getTopWindow();
1168 }
1169
1170 if (TARGET_BLANK.equals(name)) {
1171 return null;
1172 }
1173
1174
1175 WebWindow window = opener;
1176 while (true) {
1177 final Page page = window.getEnclosedPage();
1178 if (page != null && page.isHtmlPage()) {
1179 try {
1180 final FrameWindow frame = ((HtmlPage) page).getFrameByName(name);
1181 final HtmlUnitScriptable scriptable = frame.getFrameElement().getScriptableObject();
1182 if (scriptable instanceof HTMLIFrameElement) {
1183 ((HTMLIFrameElement) scriptable).onRefresh();
1184 }
1185 return frame;
1186 }
1187 catch (final ElementNotFoundException expected) {
1188
1189 }
1190 }
1191
1192 if (window == window.getParentWindow()) {
1193
1194 break;
1195 }
1196 window = window.getParentWindow();
1197 }
1198
1199 try {
1200 return getWebWindowByName(name);
1201 }
1202 catch (final WebWindowNotFoundException expected) {
1203
1204 }
1205 return null;
1206 }
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218 public DialogWindow openDialogWindow(final URL url, final WebWindow opener, final Object dialogArguments)
1219 throws IOException {
1220
1221 WebAssert.notNull("url", url);
1222 WebAssert.notNull("opener", opener);
1223
1224 final DialogWindow window = new DialogWindow(this, dialogArguments);
1225
1226 final HtmlPage openerPage = (HtmlPage) opener.getEnclosedPage();
1227 final WebRequest request = new WebRequest(url, getBrowserVersion().getHtmlAcceptHeader(),
1228 getBrowserVersion().getAcceptEncodingHeader());
1229 request.setCharset(UTF_8);
1230
1231 if (openerPage != null) {
1232 request.setRefererHeader(openerPage.getUrl());
1233 }
1234
1235 getPage(window, request);
1236
1237 return window;
1238 }
1239
1240
1241
1242
1243
1244
1245
1246 public void setPageCreator(final PageCreator pageCreator) {
1247 WebAssert.notNull("pageCreator", pageCreator);
1248 pageCreator_ = pageCreator;
1249 }
1250
1251
1252
1253
1254
1255
1256 public PageCreator getPageCreator() {
1257 return pageCreator_;
1258 }
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269 public WebWindow getWebWindowByName(final String name) throws WebWindowNotFoundException {
1270 WebAssert.notNull("name", name);
1271
1272 for (final WebWindow webWindow : windows_) {
1273 if (name.equals(webWindow.getName())) {
1274 return webWindow;
1275 }
1276 }
1277
1278 throw new WebWindowNotFoundException(name);
1279 }
1280
1281
1282
1283
1284
1285
1286
1287
1288 public void initialize(final WebWindow webWindow, final Page page) {
1289 WebAssert.notNull("webWindow", webWindow);
1290
1291 if (isJavaScriptEngineEnabled()) {
1292 scriptEngine_.initialize(webWindow, page);
1293 }
1294 }
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304 public void initializeEmptyWindow(final WebWindow webWindow, final Page page) {
1305 WebAssert.notNull("webWindow", webWindow);
1306
1307 if (isJavaScriptEngineEnabled()) {
1308 initialize(webWindow, page);
1309 ((Window) webWindow.getScriptableObject()).initialize();
1310 }
1311 }
1312
1313
1314
1315
1316
1317
1318
1319
1320 public void registerWebWindow(final WebWindow webWindow) {
1321 WebAssert.notNull("webWindow", webWindow);
1322 if (windows_.add(webWindow)) {
1323 fireWindowOpened(new WebWindowEvent(webWindow, WebWindowEvent.OPEN, webWindow.getEnclosedPage(), null));
1324 }
1325
1326 jobManagers_.add(new WeakReference<>(webWindow.getJobManager()));
1327 }
1328
1329
1330
1331
1332
1333
1334
1335
1336 public void deregisterWebWindow(final WebWindow webWindow) {
1337 WebAssert.notNull("webWindow", webWindow);
1338 if (windows_.remove(webWindow)) {
1339 fireWindowClosed(new WebWindowEvent(webWindow, WebWindowEvent.CLOSE, webWindow.getEnclosedPage(), null));
1340 }
1341 }
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355 public static URL expandUrl(final URL baseUrl, final String relativeUrl) throws MalformedURLException {
1356 final String newUrl = UrlUtils.resolveUrl(baseUrl, relativeUrl);
1357 return UrlUtils.toUrlUnsafe(newUrl);
1358 }
1359
1360 private WebResponse makeWebResponseForDataUrl(final WebRequest webRequest) throws IOException {
1361 final URL url = webRequest.getUrl();
1362 final DataURLConnection connection;
1363 connection = new DataURLConnection(url);
1364
1365 final List<NameValuePair> responseHeaders = new ArrayList<>();
1366 responseHeaders.add(new NameValuePair(HttpHeader.CONTENT_TYPE_LC,
1367 connection.getMediaType() + ";charset=" + connection.getCharset()));
1368
1369 try (InputStream is = connection.getInputStream()) {
1370 final DownloadedContent downloadedContent =
1371 HttpWebConnection.downloadContent(is,
1372 getOptions().getMaxInMemory(),
1373 getOptions().getTempFileDirectory());
1374 final WebResponseData data = new WebResponseData(downloadedContent, 200, "OK", responseHeaders);
1375 return new WebResponse(data, url, webRequest.getHttpMethod(), 0);
1376 }
1377 }
1378
1379 private static WebResponse makeWebResponseForAboutUrl(final WebRequest webRequest) throws MalformedURLException {
1380 final URL url = webRequest.getUrl();
1381 final String urlString = url.toExternalForm();
1382 if (UrlUtils.ABOUT_BLANK.equalsIgnoreCase(urlString)) {
1383 return new StringWebResponse("", UrlUtils.URL_ABOUT_BLANK);
1384 }
1385
1386 final String urlWithoutQuery = StringUtils.substringBefore(urlString, "?");
1387 if (!"blank".equalsIgnoreCase(StringUtils.substringAfter(urlWithoutQuery, UrlUtils.ABOUT_SCHEME))) {
1388 throw new MalformedURLException(url + " is not supported, only about:blank is supported at the moment.");
1389 }
1390 return new StringWebResponse("", url);
1391 }
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401 private WebResponse makeWebResponseForFileUrl(final WebRequest webRequest) throws IOException {
1402 URL cleanUrl = webRequest.getUrl();
1403 if (cleanUrl.getQuery() != null) {
1404
1405 cleanUrl = UrlUtils.getUrlWithNewQuery(cleanUrl, null);
1406 }
1407 if (cleanUrl.getRef() != null) {
1408
1409 cleanUrl = UrlUtils.getUrlWithNewRef(cleanUrl, null);
1410 }
1411
1412 final WebResponse fromCache = getCache().getCachedResponse(webRequest);
1413 if (fromCache != null) {
1414 return new WebResponseFromCache(fromCache, webRequest);
1415 }
1416
1417 String fileUrl = cleanUrl.toExternalForm();
1418 fileUrl = URLDecoder.decode(fileUrl, UTF_8.name());
1419 final File file = new File(fileUrl.substring(5));
1420 if (!file.exists()) {
1421
1422 final List<NameValuePair> compiledHeaders = new ArrayList<>();
1423 compiledHeaders.add(new NameValuePair(HttpHeader.CONTENT_TYPE, MimeType.TEXT_HTML));
1424 final WebResponseData responseData =
1425 new WebResponseData(
1426 org.htmlunit.util.StringUtils
1427 .toByteArray("File: " + file.getAbsolutePath(), UTF_8),
1428 404, "Not Found", compiledHeaders);
1429 return new WebResponse(responseData, webRequest, 0);
1430 }
1431
1432 final String contentType = guessContentType(file);
1433
1434 final DownloadedContent content = new DownloadedContent.OnFile(file, false);
1435 final List<NameValuePair> compiledHeaders = new ArrayList<>();
1436 compiledHeaders.add(new NameValuePair(HttpHeader.CONTENT_TYPE, contentType));
1437 compiledHeaders.add(new NameValuePair(HttpHeader.LAST_MODIFIED,
1438 HttpUtils.formatDate(new Date(file.lastModified()))));
1439 final WebResponseData responseData = new WebResponseData(content, 200, "OK", compiledHeaders);
1440 final WebResponse webResponse = new WebResponse(responseData, webRequest, 0);
1441 getCache().cacheIfPossible(webRequest, webResponse, null);
1442 return webResponse;
1443 }
1444
1445 private WebResponse makeWebResponseForBlobUrl(final WebRequest webRequest) {
1446 final Window window = getCurrentWindow().getScriptableObject();
1447 final Blob fileOrBlob = window.getDocument().resolveBlobUrl(webRequest.getUrl().toString());
1448 if (fileOrBlob == null) {
1449 throw JavaScriptEngine.typeError("Cannot load data from " + webRequest.getUrl());
1450 }
1451
1452 final List<NameValuePair> headers = new ArrayList<>();
1453 final String type = fileOrBlob.getType();
1454 if (!StringUtils.isEmpty(type)) {
1455 headers.add(new NameValuePair(HttpHeader.CONTENT_TYPE, fileOrBlob.getType()));
1456 }
1457 if (fileOrBlob instanceof org.htmlunit.javascript.host.file.File) {
1458 final org.htmlunit.javascript.host.file.File file = (org.htmlunit.javascript.host.file.File) fileOrBlob;
1459 final String fileName = file.getName();
1460 if (!StringUtils.isEmpty(fileName)) {
1461
1462 headers.add(new NameValuePair(HttpHeader.CONTENT_DISPOSITION, "inline; filename=\"" + fileName + "\""));
1463 }
1464 }
1465
1466 final DownloadedContent content = new DownloadedContent.InMemory(fileOrBlob.getBytes());
1467 final WebResponseData responseData = new WebResponseData(content, 200, "OK", headers);
1468 return new WebResponse(responseData, webRequest, 0);
1469 }
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479 public String guessContentType(final File file) {
1480 final String fileName = file.getName();
1481 final String fileNameLC = fileName.toLowerCase(Locale.ROOT);
1482 if (fileNameLC.endsWith(".xhtml")) {
1483
1484 return MimeType.APPLICATION_XHTML;
1485 }
1486
1487
1488 if (fileNameLC.endsWith(".js")) {
1489 return MimeType.TEXT_JAVASCRIPT;
1490 }
1491
1492 if (fileNameLC.endsWith(".css")) {
1493 return MimeType.TEXT_CSS;
1494 }
1495
1496 String contentType = null;
1497 if (!fileNameLC.endsWith(".php")) {
1498 contentType = URLConnection.guessContentTypeFromName(fileName);
1499 }
1500 if (contentType == null) {
1501 try (InputStream inputStream = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
1502 contentType = URLConnection.guessContentTypeFromStream(inputStream);
1503 }
1504 catch (final IOException ignored) {
1505
1506 }
1507 }
1508 if (contentType == null) {
1509 contentType = MimeType.APPLICATION_OCTET_STREAM;
1510 }
1511 return contentType;
1512 }
1513
1514 private WebResponse makeWebResponseForJavaScriptUrl(final WebWindow webWindow, final URL url,
1515 final Charset charset) throws FailingHttpStatusCodeException, IOException {
1516
1517 HtmlPage page = null;
1518 if (webWindow instanceof FrameWindow) {
1519 final FrameWindow frameWindow = (FrameWindow) webWindow;
1520 page = (HtmlPage) frameWindow.getEnclosedPage();
1521 }
1522 else {
1523 final Page currentPage = webWindow.getEnclosedPage();
1524 if (currentPage instanceof HtmlPage) {
1525 page = (HtmlPage) currentPage;
1526 }
1527 }
1528
1529 if (page == null) {
1530 page = getPage(webWindow, WebRequest.newAboutBlankRequest());
1531 }
1532 final ScriptResult r = page.executeJavaScript(url.toExternalForm(), "JavaScript URL", 1);
1533 if (r.getJavaScriptResult() == null || ScriptResult.isUndefined(r)) {
1534
1535 return webWindow.getEnclosedPage().getWebResponse();
1536 }
1537
1538 final String contentString = r.getJavaScriptResult().toString();
1539 final StringWebResponse response = new StringWebResponse(contentString, charset, url);
1540 response.setFromJavascript(true);
1541 return response;
1542 }
1543
1544
1545
1546
1547
1548
1549
1550 public WebResponse loadWebResponse(final WebRequest webRequest) throws IOException {
1551 final String protocol = webRequest.getUrl().getProtocol();
1552 switch (protocol) {
1553 case UrlUtils.ABOUT:
1554 return makeWebResponseForAboutUrl(webRequest);
1555
1556 case "file":
1557 return makeWebResponseForFileUrl(webRequest);
1558
1559 case "data":
1560 return makeWebResponseForDataUrl(webRequest);
1561
1562 case "blob":
1563 return makeWebResponseForBlobUrl(webRequest);
1564
1565 case "http":
1566 case "https":
1567 return loadWebResponseFromWebConnection(webRequest, ALLOWED_REDIRECTIONS_SAME_URL);
1568
1569 default:
1570 throw new IOException("Unsupported protocol '" + protocol + "'");
1571 }
1572 }
1573
1574
1575
1576
1577
1578
1579
1580
1581 private WebResponse loadWebResponseFromWebConnection(final WebRequest webRequest,
1582 final int allowedRedirects) throws IOException {
1583
1584 URL url = webRequest.getUrl();
1585 final HttpMethod method = webRequest.getHttpMethod();
1586 final List<NameValuePair> parameters = webRequest.getRequestParameters();
1587
1588 WebAssert.notNull("url", url);
1589 WebAssert.notNull("method", method);
1590 WebAssert.notNull("parameters", parameters);
1591
1592 url = UrlUtils.encodeUrl(url, webRequest.getCharset());
1593 webRequest.setUrl(url);
1594
1595 if (LOG.isDebugEnabled()) {
1596 LOG.debug("Load response for " + method + " " + url.toExternalForm());
1597 }
1598
1599
1600 if (webRequest.getProxyHost() == null) {
1601 final ProxyConfig proxyConfig = getOptions().getProxyConfig();
1602 if (proxyConfig.getProxyAutoConfigUrl() != null) {
1603 if (!UrlUtils.sameFile(new URL(proxyConfig.getProxyAutoConfigUrl()), url)) {
1604 String content = proxyConfig.getProxyAutoConfigContent();
1605 if (content == null) {
1606 content = getPage(proxyConfig.getProxyAutoConfigUrl())
1607 .getWebResponse().getContentAsString();
1608 proxyConfig.setProxyAutoConfigContent(content);
1609 }
1610 final String allValue = JavaScriptEngine.evaluateProxyAutoConfig(getBrowserVersion(), content, url);
1611 if (LOG.isDebugEnabled()) {
1612 LOG.debug("Proxy Auto-Config: value '" + allValue + "' for URL " + url);
1613 }
1614 String value = allValue.split(";")[0].trim();
1615 if (value.startsWith("PROXY")) {
1616 value = value.substring(6);
1617 final int colonIndex = value.indexOf(':');
1618 webRequest.setSocksProxy(false);
1619 webRequest.setProxyHost(value.substring(0, colonIndex));
1620 webRequest.setProxyPort(Integer.parseInt(value.substring(colonIndex + 1)));
1621 }
1622 else if (value.startsWith("SOCKS")) {
1623 value = value.substring(6);
1624 final int colonIndex = value.indexOf(':');
1625 webRequest.setSocksProxy(true);
1626 webRequest.setProxyHost(value.substring(0, colonIndex));
1627 webRequest.setProxyPort(Integer.parseInt(value.substring(colonIndex + 1)));
1628 }
1629 }
1630 }
1631
1632 else if (!proxyConfig.shouldBypassProxy(webRequest.getUrl().getHost())) {
1633 webRequest.setProxyHost(proxyConfig.getProxyHost());
1634 webRequest.setProxyPort(proxyConfig.getProxyPort());
1635 webRequest.setProxyScheme(proxyConfig.getProxyScheme());
1636 webRequest.setSocksProxy(proxyConfig.isSocksProxy());
1637 }
1638 }
1639
1640
1641 addDefaultHeaders(webRequest);
1642
1643
1644 final WebResponse fromCache = getCache().getCachedResponse(webRequest);
1645 final WebResponse webResponse = getWebResponseOrUseCached(webRequest, fromCache);
1646
1647
1648 final int status = webResponse.getStatusCode();
1649 if (status == HttpStatus.USE_PROXY_305) {
1650 getIncorrectnessListener().notify("Ignoring HTTP status code [305] 'Use Proxy'", this);
1651 }
1652 else if (status >= HttpStatus.MOVED_PERMANENTLY_301
1653 && status <= HttpStatus.PERMANENT_REDIRECT_308
1654 && status != HttpStatus.NOT_MODIFIED_304
1655 && getOptions().isRedirectEnabled()) {
1656
1657 final URL newUrl;
1658 String locationString = null;
1659 try {
1660 locationString = webResponse.getResponseHeaderValue("Location");
1661 if (locationString == null) {
1662 return webResponse;
1663 }
1664 locationString = new String(locationString.getBytes(ISO_8859_1), UTF_8);
1665 newUrl = expandUrl(url, locationString);
1666 }
1667 catch (final MalformedURLException e) {
1668 getIncorrectnessListener().notify("Got a redirect status code [" + status + " "
1669 + webResponse.getStatusMessage()
1670 + "] but the location is not a valid URL [" + locationString
1671 + "]. Skipping redirection processing.", this);
1672 return webResponse;
1673 }
1674
1675 if (LOG.isDebugEnabled()) {
1676 LOG.debug("Got a redirect status code [" + status + "] new location = [" + locationString + "]");
1677 }
1678
1679 if (allowedRedirects == 0) {
1680 throw new FailingHttpStatusCodeException("Too much redirect for "
1681 + webResponse.getWebRequest().getUrl(), webResponse);
1682 }
1683
1684 if (status == HttpStatus.MOVED_PERMANENTLY_301
1685 || status == HttpStatus.FOUND_302
1686 || status == HttpStatus.SEE_OTHER_303) {
1687 final WebRequest wrs = new WebRequest(newUrl, HttpMethod.GET);
1688 wrs.setCharset(webRequest.getCharset());
1689
1690 if (HttpMethod.HEAD == webRequest.getHttpMethod()) {
1691 wrs.setHttpMethod(HttpMethod.HEAD);
1692 }
1693 for (final Map.Entry<String, String> entry : webRequest.getAdditionalHeaders().entrySet()) {
1694 wrs.setAdditionalHeader(entry.getKey(), entry.getValue());
1695 }
1696 return loadWebResponseFromWebConnection(wrs, allowedRedirects - 1);
1697 }
1698 else if (status == HttpStatus.TEMPORARY_REDIRECT_307
1699 || status == HttpStatus.PERMANENT_REDIRECT_308) {
1700
1701
1702
1703 final WebRequest wrs = new WebRequest(newUrl, webRequest.getHttpMethod());
1704 wrs.setCharset(webRequest.getCharset());
1705 if (webRequest.getRequestBody() != null) {
1706 if (HttpMethod.POST == webRequest.getHttpMethod()
1707 || HttpMethod.PUT == webRequest.getHttpMethod()
1708 || HttpMethod.PATCH == webRequest.getHttpMethod()) {
1709 wrs.setRequestBody(webRequest.getRequestBody());
1710 wrs.setEncodingType(webRequest.getEncodingType());
1711 }
1712 }
1713 else {
1714 wrs.setRequestParameters(parameters);
1715 }
1716
1717 for (final Map.Entry<String, String> entry : webRequest.getAdditionalHeaders().entrySet()) {
1718 wrs.setAdditionalHeader(entry.getKey(), entry.getValue());
1719 }
1720
1721 return loadWebResponseFromWebConnection(wrs, allowedRedirects - 1);
1722 }
1723 }
1724
1725 if (fromCache == null) {
1726 getCache().cacheIfPossible(webRequest, webResponse, null);
1727 }
1728 return webResponse;
1729 }
1730
1731
1732
1733
1734
1735
1736
1737 private WebResponse getWebResponseOrUseCached(
1738 final WebRequest webRequest, final WebResponse cached) throws IOException {
1739 if (cached == null) {
1740 return getWebConnection().getResponse(webRequest);
1741 }
1742
1743 if (!HeaderUtils.containsNoCache(cached)) {
1744 return new WebResponseFromCache(cached, webRequest);
1745 }
1746
1747
1748 if (HeaderUtils.containsETag(cached)) {
1749 webRequest.setAdditionalHeader(HttpHeader.IF_NONE_MATCH, cached.getResponseHeaderValue(HttpHeader.ETAG));
1750 }
1751 if (HeaderUtils.containsLastModified(cached)) {
1752 webRequest.setAdditionalHeader(HttpHeader.IF_MODIFIED_SINCE,
1753 cached.getResponseHeaderValue(HttpHeader.LAST_MODIFIED));
1754 }
1755
1756 final WebResponse webResponse = getWebConnection().getResponse(webRequest);
1757
1758 if (webResponse.getStatusCode() >= HttpStatus.INTERNAL_SERVER_ERROR_500) {
1759 return new WebResponseFromCache(cached, webRequest);
1760 }
1761
1762 if (webResponse.getStatusCode() == HttpStatus.NOT_MODIFIED_304) {
1763 final Map<String, NameValuePair> header2NameValuePair = new LinkedHashMap<>();
1764 for (final NameValuePair pair : cached.getResponseHeaders()) {
1765 header2NameValuePair.put(pair.getName(), pair);
1766 }
1767 for (final NameValuePair pair : webResponse.getResponseHeaders()) {
1768 if (preferHeaderFrom304Response(pair.getName())) {
1769 header2NameValuePair.put(pair.getName(), pair);
1770 }
1771 }
1772
1773
1774
1775 final WebResponse updatedCached =
1776 new WebResponseFromCache(cached, new ArrayList<>(header2NameValuePair.values()), webRequest);
1777 getCache().cacheIfPossible(webRequest, updatedCached, null);
1778 return updatedCached;
1779 }
1780
1781 getCache().cacheIfPossible(webRequest, webResponse, null);
1782 return webResponse;
1783 }
1784
1785
1786
1787
1788
1789 private static boolean preferHeaderFrom304Response(final String name) {
1790 final String lcName = name.toLowerCase(Locale.ROOT);
1791 for (final String header : DISCARDING_304_RESPONSE_HEADER_NAMES) {
1792 if (lcName.equals(header)) {
1793 return false;
1794 }
1795 }
1796 for (final String prefix : DISCARDING_304_HEADER_PREFIXES) {
1797 if (lcName.startsWith(prefix)) {
1798 return false;
1799 }
1800 }
1801 return true;
1802 }
1803
1804
1805
1806
1807
1808 private void addDefaultHeaders(final WebRequest wrs) {
1809
1810 requestHeaders_.forEach((name, value) -> {
1811 if (!wrs.isAdditionalHeader(name)) {
1812 wrs.setAdditionalHeader(name, value);
1813 }
1814 });
1815
1816
1817 if (!wrs.isAdditionalHeader(HttpHeader.ACCEPT_LANGUAGE)) {
1818 wrs.setAdditionalHeader(HttpHeader.ACCEPT_LANGUAGE, getBrowserVersion().getAcceptLanguageHeader());
1819 }
1820
1821 if (!wrs.isAdditionalHeader(HttpHeader.SEC_FETCH_DEST)) {
1822 wrs.setAdditionalHeader(HttpHeader.SEC_FETCH_DEST, "document");
1823 }
1824 if (!wrs.isAdditionalHeader(HttpHeader.SEC_FETCH_MODE)) {
1825 wrs.setAdditionalHeader(HttpHeader.SEC_FETCH_MODE, "navigate");
1826 }
1827 if (!wrs.isAdditionalHeader(HttpHeader.SEC_FETCH_SITE)) {
1828 wrs.setAdditionalHeader(HttpHeader.SEC_FETCH_SITE, "same-origin");
1829 }
1830 if (!wrs.isAdditionalHeader(HttpHeader.SEC_FETCH_USER)) {
1831 wrs.setAdditionalHeader(HttpHeader.SEC_FETCH_USER, "?1");
1832 }
1833 if (getBrowserVersion().hasFeature(HTTP_HEADER_PRIORITY)
1834 && !wrs.isAdditionalHeader(HttpHeader.PRIORITY)) {
1835 wrs.setAdditionalHeader(HttpHeader.PRIORITY, "u=0, i");
1836 }
1837
1838 if (getBrowserVersion().hasFeature(HTTP_HEADER_CH_UA)
1839 && !wrs.isAdditionalHeader(HttpHeader.SEC_CH_UA)) {
1840 wrs.setAdditionalHeader(HttpHeader.SEC_CH_UA, getBrowserVersion().getSecClientHintUserAgentHeader());
1841 }
1842 if (getBrowserVersion().hasFeature(HTTP_HEADER_CH_UA)
1843 && !wrs.isAdditionalHeader(HttpHeader.SEC_CH_UA_MOBILE)) {
1844 wrs.setAdditionalHeader(HttpHeader.SEC_CH_UA_MOBILE, "?0");
1845 }
1846 if (getBrowserVersion().hasFeature(HTTP_HEADER_CH_UA)
1847 && !wrs.isAdditionalHeader(HttpHeader.SEC_CH_UA_PLATFORM)) {
1848 wrs.setAdditionalHeader(HttpHeader.SEC_CH_UA_PLATFORM,
1849 getBrowserVersion().getSecClientHintUserAgentPlatformHeader());
1850 }
1851
1852 if (!wrs.isAdditionalHeader(HttpHeader.UPGRADE_INSECURE_REQUESTS)) {
1853 wrs.setAdditionalHeader(HttpHeader.UPGRADE_INSECURE_REQUESTS, "1");
1854 }
1855 }
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867 public List<WebWindow> getWebWindows() {
1868 return Collections.unmodifiableList(new ArrayList<>(windows_));
1869 }
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881 public boolean containsWebWindow(final WebWindow webWindow) {
1882 return windows_.contains(webWindow);
1883 }
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895 public List<TopLevelWindow> getTopLevelWindows() {
1896 return Collections.unmodifiableList(new ArrayList<>(topLevelWindows_));
1897 }
1898
1899
1900
1901
1902
1903
1904 public void setRefreshHandler(final RefreshHandler handler) {
1905 if (handler == null) {
1906 refreshHandler_ = new NiceRefreshHandler(2);
1907 }
1908 else {
1909 refreshHandler_ = handler;
1910 }
1911 }
1912
1913
1914
1915
1916
1917
1918 public RefreshHandler getRefreshHandler() {
1919 return refreshHandler_;
1920 }
1921
1922
1923
1924
1925
1926 public void setScriptPreProcessor(final ScriptPreProcessor scriptPreProcessor) {
1927 scriptPreProcessor_ = scriptPreProcessor;
1928 }
1929
1930
1931
1932
1933
1934 public ScriptPreProcessor getScriptPreProcessor() {
1935 return scriptPreProcessor_;
1936 }
1937
1938
1939
1940
1941
1942 public void setHTMLParserListener(final HTMLParserListener listener) {
1943 htmlParserListener_ = listener;
1944 }
1945
1946
1947
1948
1949
1950 public HTMLParserListener getHTMLParserListener() {
1951 return htmlParserListener_;
1952 }
1953
1954
1955
1956
1957
1958
1959
1960 public CSSErrorHandler getCssErrorHandler() {
1961 return cssErrorHandler_;
1962 }
1963
1964
1965
1966
1967
1968
1969
1970 public void setCssErrorHandler(final CSSErrorHandler cssErrorHandler) {
1971 WebAssert.notNull("cssErrorHandler", cssErrorHandler);
1972 cssErrorHandler_ = cssErrorHandler;
1973 }
1974
1975
1976
1977
1978
1979
1980
1981 public void setJavaScriptTimeout(final long timeout) {
1982 scriptEngine_.setJavaScriptTimeout(timeout);
1983 }
1984
1985
1986
1987
1988
1989
1990
1991 public long getJavaScriptTimeout() {
1992 return scriptEngine_.getJavaScriptTimeout();
1993 }
1994
1995
1996
1997
1998
1999
2000
2001 public IncorrectnessListener getIncorrectnessListener() {
2002 return incorrectnessListener_;
2003 }
2004
2005
2006
2007
2008
2009 public void setIncorrectnessListener(final IncorrectnessListener listener) {
2010 if (listener == null) {
2011 throw new IllegalArgumentException("Null is not a valid IncorrectnessListener");
2012 }
2013 incorrectnessListener_ = listener;
2014 }
2015
2016
2017
2018
2019
2020 public WebConsole getWebConsole() {
2021 if (webConsole_ == null) {
2022 webConsole_ = new WebConsole();
2023 }
2024 return webConsole_;
2025 }
2026
2027
2028
2029
2030
2031 public AjaxController getAjaxController() {
2032 return ajaxController_;
2033 }
2034
2035
2036
2037
2038
2039 public void setAjaxController(final AjaxController newValue) {
2040 if (newValue == null) {
2041 throw new IllegalArgumentException("Null is not a valid AjaxController");
2042 }
2043 ajaxController_ = newValue;
2044 }
2045
2046
2047
2048
2049
2050 public void setAttachmentHandler(final AttachmentHandler handler) {
2051 attachmentHandler_ = handler;
2052 }
2053
2054
2055
2056
2057
2058 public AttachmentHandler getAttachmentHandler() {
2059 return attachmentHandler_;
2060 }
2061
2062
2063
2064
2065
2066 public void setWebStartHandler(final WebStartHandler handler) {
2067 webStartHandler_ = handler;
2068 }
2069
2070
2071
2072
2073
2074 public WebStartHandler getWebStartHandler() {
2075 return webStartHandler_;
2076 }
2077
2078
2079
2080
2081
2082 public ClipboardHandler getClipboardHandler() {
2083 return clipboardHandler_;
2084 }
2085
2086
2087
2088
2089
2090 public void setClipboardHandler(final ClipboardHandler handler) {
2091 clipboardHandler_ = handler;
2092 }
2093
2094
2095
2096
2097
2098
2099 public PrintHandler getPrintHandler() {
2100 return printHandler_;
2101 }
2102
2103
2104
2105
2106
2107
2108
2109
2110 public void setPrintHandler(final PrintHandler handler) {
2111 printHandler_ = handler;
2112 }
2113
2114
2115
2116
2117
2118 public FrameContentHandler getFrameContentHandler() {
2119 return frameContentHandler_;
2120 }
2121
2122
2123
2124
2125
2126 public void setFrameContentHandler(final FrameContentHandler handler) {
2127 frameContentHandler_ = handler;
2128 }
2129
2130
2131
2132
2133
2134 public void setOnbeforeunloadHandler(final OnbeforeunloadHandler onbeforeunloadHandler) {
2135 onbeforeunloadHandler_ = onbeforeunloadHandler;
2136 }
2137
2138
2139
2140
2141
2142 public OnbeforeunloadHandler getOnbeforeunloadHandler() {
2143 return onbeforeunloadHandler_;
2144 }
2145
2146
2147
2148
2149
2150 public Cache getCache() {
2151 return cache_;
2152 }
2153
2154
2155
2156
2157
2158 public void setCache(final Cache cache) {
2159 if (cache == null) {
2160 throw new IllegalArgumentException("cache should not be null!");
2161 }
2162 cache_ = cache;
2163 }
2164
2165
2166
2167
2168 private static final class CurrentWindowTracker implements WebWindowListener, Serializable {
2169 private final WebClient webClient_;
2170 private final boolean ensureOneTopLevelWindow_;
2171
2172 CurrentWindowTracker(final WebClient webClient, final boolean ensureOneTopLevelWindow) {
2173 webClient_ = webClient;
2174 ensureOneTopLevelWindow_ = ensureOneTopLevelWindow;
2175 }
2176
2177
2178
2179
2180 @Override
2181 public void webWindowClosed(final WebWindowEvent event) {
2182 final WebWindow window = event.getWebWindow();
2183 if (window instanceof TopLevelWindow) {
2184 webClient_.topLevelWindows_.remove(window);
2185 if (window == webClient_.getCurrentWindow()) {
2186 if (!webClient_.topLevelWindows_.isEmpty()) {
2187
2188 webClient_.setCurrentWindow(
2189 webClient_.topLevelWindows_.get(webClient_.topLevelWindows_.size() - 1));
2190 }
2191 }
2192 }
2193 else if (window == webClient_.getCurrentWindow()) {
2194
2195 if (webClient_.topLevelWindows_.isEmpty()) {
2196 webClient_.setCurrentWindow(null);
2197 }
2198 else {
2199 webClient_.setCurrentWindow(
2200 webClient_.topLevelWindows_.get(webClient_.topLevelWindows_.size() - 1));
2201 }
2202 }
2203 }
2204
2205
2206
2207
2208 public void afterWebWindowClosedListenersProcessed(final WebWindowEvent event) {
2209 if (!ensureOneTopLevelWindow_) {
2210 return;
2211 }
2212
2213 if (webClient_.topLevelWindows_.isEmpty()) {
2214
2215 final TopLevelWindow newWindow = new TopLevelWindow("", webClient_);
2216 webClient_.setCurrentWindow(newWindow);
2217 }
2218 }
2219
2220
2221
2222
2223 @Override
2224 public void webWindowContentChanged(final WebWindowEvent event) {
2225 final WebWindow window = event.getWebWindow();
2226 boolean use = false;
2227 if (window instanceof DialogWindow) {
2228 use = true;
2229 }
2230 else if (window instanceof TopLevelWindow) {
2231 use = event.getOldPage() == null;
2232 }
2233 else if (window instanceof FrameWindow) {
2234 final FrameWindow fw = (FrameWindow) window;
2235 final String enclosingPageState = fw.getEnclosingPage().getDocumentElement().getReadyState();
2236 final URL frameUrl = fw.getEnclosedPage().getUrl();
2237 if (!DomNode.READY_STATE_COMPLETE.equals(enclosingPageState) || frameUrl == UrlUtils.URL_ABOUT_BLANK) {
2238 return;
2239 }
2240
2241
2242 final BaseFrameElement frameElement = fw.getFrameElement();
2243 if (webClient_.isJavaScriptEnabled() && frameElement.isDisplayed()) {
2244 final ComputedCssStyleDeclaration style = fw.getComputedStyle(frameElement, null);
2245 use = style.getCalculatedWidth(false, false) != 0
2246 && style.getCalculatedHeight(false, false) != 0;
2247 }
2248 }
2249 if (use) {
2250 webClient_.setCurrentWindow(window);
2251 }
2252 }
2253
2254
2255
2256
2257 @Override
2258 public void webWindowOpened(final WebWindowEvent event) {
2259 final WebWindow window = event.getWebWindow();
2260 if (window instanceof TopLevelWindow) {
2261 final TopLevelWindow tlw = (TopLevelWindow) window;
2262 webClient_.topLevelWindows_.add(tlw);
2263 }
2264
2265 }
2266 }
2267
2268
2269
2270
2271
2272
2273
2274
2275 @Override
2276 public void close() {
2277
2278 if (scriptEngine_ != null) {
2279 scriptEngine_.prepareShutdown();
2280 }
2281
2282
2283 currentWindowTracker_ = new CurrentWindowTracker(this, false);
2284
2285
2286
2287 List<WebWindow> windows = new ArrayList<>(windows_);
2288 for (final WebWindow window : windows) {
2289 if (window instanceof TopLevelWindow) {
2290 final TopLevelWindow topLevelWindow = (TopLevelWindow) window;
2291
2292 try {
2293 topLevelWindow.close(true);
2294 }
2295 catch (final Exception e) {
2296 LOG.error("Exception while closing a TopLevelWindow", e);
2297 }
2298 }
2299 else if (window instanceof DialogWindow) {
2300 final DialogWindow dialogWindow = (DialogWindow) window;
2301
2302 try {
2303 dialogWindow.close();
2304 }
2305 catch (final Exception e) {
2306 LOG.error("Exception while closing a DialogWindow", e);
2307 }
2308 }
2309 }
2310
2311
2312
2313 windows = new ArrayList<>(windows_);
2314 for (final WebWindow window : windows) {
2315 if (window instanceof TopLevelWindow) {
2316 final TopLevelWindow topLevelWindow = (TopLevelWindow) window;
2317
2318 try {
2319 topLevelWindow.close(true);
2320 }
2321 catch (final Exception e) {
2322 LOG.error("Exception while closing a TopLevelWindow", e);
2323 }
2324 }
2325 else if (window instanceof DialogWindow) {
2326 final DialogWindow dialogWindow = (DialogWindow) window;
2327
2328 try {
2329 dialogWindow.close();
2330 }
2331 catch (final Exception e) {
2332 LOG.error("Exception while closing a DialogWindow", e);
2333 }
2334 }
2335 }
2336
2337
2338 if (!topLevelWindows_.isEmpty()) {
2339 LOG.error("Sill " + topLevelWindows_.size() + " top level windows are open. Please report this error!");
2340 topLevelWindows_.clear();
2341 }
2342
2343 if (!windows_.isEmpty()) {
2344 LOG.error("Sill " + windows_.size() + " windows are open. Please report this error!");
2345 windows_.clear();
2346 }
2347 currentWindow_ = null;
2348
2349 ThreadDeath toThrow = null;
2350 if (scriptEngine_ != null) {
2351 try {
2352 scriptEngine_.shutdown();
2353 }
2354 catch (final ThreadDeath ex) {
2355
2356 toThrow = ex;
2357 }
2358 catch (final Exception e) {
2359 LOG.error("Exception while shutdown the scriptEngine", e);
2360 }
2361 }
2362 scriptEngine_ = null;
2363
2364 if (webConnection_ != null) {
2365 try {
2366 webConnection_.close();
2367 }
2368 catch (final Exception e) {
2369 LOG.error("Exception while closing the connection", e);
2370 }
2371 }
2372 webConnection_ = null;
2373
2374 synchronized (this) {
2375 if (executor_ != null) {
2376 try {
2377 executor_.shutdownNow();
2378 }
2379 catch (final Exception e) {
2380 LOG.error("Exception while shutdown the executor service", e);
2381 }
2382 }
2383 }
2384 executor_ = null;
2385
2386 cache_.clear();
2387 if (toThrow != null) {
2388 throw toThrow;
2389 }
2390 }
2391
2392
2393
2394
2395
2396
2397
2398
2399 public void reset() {
2400 close();
2401
2402
2403 webConnection_ = new HttpWebConnection(this);
2404 if (javaScriptEngineEnabled_) {
2405 scriptEngine_ = new JavaScriptEngine(this);
2406 }
2407
2408
2409 currentWindowTracker_ = new CurrentWindowTracker(this, true);
2410 currentWindow_ = new TopLevelWindow("", this);
2411 }
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433 public int waitForBackgroundJavaScript(final long timeoutMillis) {
2434 int count = 0;
2435 final long endTime = System.currentTimeMillis() + timeoutMillis;
2436 for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
2437 final JavaScriptJobManager jobManager;
2438 final WeakReference<JavaScriptJobManager> reference;
2439 try {
2440 reference = i.next();
2441 jobManager = reference.get();
2442 if (jobManager == null) {
2443 i.remove();
2444 continue;
2445 }
2446 }
2447 catch (final ConcurrentModificationException e) {
2448 i = jobManagers_.iterator();
2449 count = 0;
2450 continue;
2451 }
2452
2453 final long newTimeout = endTime - System.currentTimeMillis();
2454 count += jobManager.waitForJobs(newTimeout);
2455 }
2456 if (count != getAggregateJobCount()) {
2457 final long newTimeout = endTime - System.currentTimeMillis();
2458 return waitForBackgroundJavaScript(newTimeout);
2459 }
2460 return count;
2461 }
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487 public int waitForBackgroundJavaScriptStartingBefore(final long delayMillis) {
2488 int count = 0;
2489 final long endTime = System.currentTimeMillis() + delayMillis;
2490 for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
2491 final JavaScriptJobManager jobManager;
2492 final WeakReference<JavaScriptJobManager> reference;
2493 try {
2494 reference = i.next();
2495 jobManager = reference.get();
2496 if (jobManager == null) {
2497 i.remove();
2498 continue;
2499 }
2500 }
2501 catch (final ConcurrentModificationException e) {
2502 i = jobManagers_.iterator();
2503 count = 0;
2504 continue;
2505 }
2506 final long newDelay = endTime - System.currentTimeMillis();
2507 count += jobManager.waitForJobsStartingBefore(newDelay);
2508 }
2509 if (count != getAggregateJobCount()) {
2510 final long newDelay = endTime - System.currentTimeMillis();
2511 return waitForBackgroundJavaScriptStartingBefore(newDelay);
2512 }
2513 return count;
2514 }
2515
2516
2517
2518
2519
2520 private int getAggregateJobCount() {
2521 int count = 0;
2522 for (Iterator<WeakReference<JavaScriptJobManager>> i = jobManagers_.iterator(); i.hasNext();) {
2523 final JavaScriptJobManager jobManager;
2524 final WeakReference<JavaScriptJobManager> reference;
2525 try {
2526 reference = i.next();
2527 jobManager = reference.get();
2528 if (jobManager == null) {
2529 i.remove();
2530 continue;
2531 }
2532 }
2533 catch (final ConcurrentModificationException e) {
2534 i = jobManagers_.iterator();
2535 count = 0;
2536 continue;
2537 }
2538 final int jobCount = jobManager.getJobCount();
2539 count += jobCount;
2540 }
2541 return count;
2542 }
2543
2544
2545
2546
2547
2548
2549
2550 private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
2551 in.defaultReadObject();
2552
2553 webConnection_ = new HttpWebConnection(this);
2554 scriptEngine_ = new JavaScriptEngine(this);
2555 jobManagers_ = Collections.synchronizedList(new ArrayList<>());
2556 loadQueue_ = new ArrayList<>();
2557 css3ParserPool_ = new CSS3ParserPool();
2558 }
2559
2560 private static class LoadJob {
2561 private final WebWindow requestingWindow_;
2562 private final String target_;
2563 private final WebResponse response_;
2564 private final WeakReference<Page> originalPage_;
2565 private final WebRequest request_;
2566 private final String forceAttachmentWithFilename_;
2567
2568
2569
2570 LoadJob(final WebRequest request, final WebResponse response,
2571 final WebWindow requestingWindow, final String target, final String forceAttachmentWithFilename) {
2572 request_ = request;
2573 requestingWindow_ = requestingWindow;
2574 target_ = target;
2575 response_ = response;
2576 originalPage_ = new WeakReference<>(requestingWindow.getEnclosedPage());
2577 forceAttachmentWithFilename_ = forceAttachmentWithFilename;
2578 }
2579
2580 public boolean isOutdated() {
2581 if (target_ != null && !target_.isEmpty()) {
2582 return false;
2583 }
2584
2585 if (requestingWindow_.isClosed()) {
2586 return true;
2587 }
2588
2589 if (requestingWindow_.getEnclosedPage() != originalPage_.get()) {
2590 return true;
2591 }
2592
2593 return false;
2594 }
2595 }
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611 public void download(final WebWindow requestingWindow, final String target,
2612 final WebRequest request, final boolean checkHash,
2613 final String forceAttachmentWithFilename, final String description) {
2614
2615 final WebWindow targetWindow = resolveWindow(requestingWindow, target);
2616 final URL url = request.getUrl();
2617
2618 if (targetWindow != null && HttpMethod.POST != request.getHttpMethod()) {
2619 final Page page = targetWindow.getEnclosedPage();
2620 if (page != null) {
2621 if (page.isHtmlPage() && !((HtmlPage) page).isOnbeforeunloadAccepted()) {
2622 return;
2623 }
2624
2625 if (checkHash) {
2626 final URL current = page.getUrl();
2627 final boolean justHashJump =
2628 HttpMethod.GET == request.getHttpMethod()
2629 && UrlUtils.sameFile(url, current)
2630 && null != url.getRef();
2631
2632 if (justHashJump) {
2633 processOnlyHashChange(targetWindow, url);
2634 return;
2635 }
2636 }
2637 }
2638 }
2639
2640 synchronized (loadQueue_) {
2641
2642 for (final LoadJob otherLoadJob : loadQueue_) {
2643 if (otherLoadJob.response_ == null) {
2644 continue;
2645 }
2646 final WebRequest otherRequest = otherLoadJob.request_;
2647 final URL otherUrl = otherRequest.getUrl();
2648
2649 if (url.getPath().equals(otherUrl.getPath())
2650 && url.toString().equals(otherUrl.toString())
2651 && request.getRequestParameters().equals(otherRequest.getRequestParameters())
2652 && Objects.equals(request.getRequestBody(), otherRequest.getRequestBody())) {
2653 return;
2654 }
2655 }
2656 }
2657
2658 final LoadJob loadJob;
2659 try {
2660 WebResponse response;
2661 try {
2662 response = loadWebResponse(request);
2663 }
2664 catch (final NoHttpResponseException e) {
2665 LOG.error("NoHttpResponseException while downloading; generating a NoHttpResponse", e);
2666 response = new WebResponse(RESPONSE_DATA_NO_HTTP_RESPONSE, request, 0);
2667 }
2668 loadJob = new LoadJob(request, response, requestingWindow, target, forceAttachmentWithFilename);
2669 }
2670 catch (final IOException e) {
2671 throw new RuntimeException(e);
2672 }
2673
2674 synchronized (loadQueue_) {
2675 loadQueue_.add(loadJob);
2676 }
2677 }
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687 public void loadDownloadedResponses() throws FailingHttpStatusCodeException, IOException {
2688 final List<LoadJob> queue;
2689
2690
2691
2692 synchronized (loadQueue_) {
2693 if (loadQueue_.isEmpty()) {
2694 return;
2695 }
2696 queue = new ArrayList<>(loadQueue_);
2697 loadQueue_.clear();
2698 }
2699
2700 final HashSet<WebWindow> updatedWindows = new HashSet<>();
2701 for (int i = queue.size() - 1; i >= 0; --i) {
2702 final LoadJob loadJob = queue.get(i);
2703 if (loadJob.isOutdated()) {
2704 if (LOG.isInfoEnabled()) {
2705 LOG.info("No usage of download: " + loadJob);
2706 }
2707 continue;
2708 }
2709
2710 final WebWindow window = resolveWindow(loadJob.requestingWindow_, loadJob.target_);
2711 if (updatedWindows.contains(window)) {
2712 if (LOG.isInfoEnabled()) {
2713 LOG.info("No usage of download: " + loadJob);
2714 }
2715 continue;
2716 }
2717
2718 final WebWindow win = openTargetWindow(loadJob.requestingWindow_, loadJob.target_, TARGET_SELF);
2719 final Page pageBeforeLoad = win.getEnclosedPage();
2720 loadWebResponseInto(loadJob.response_, win, loadJob.forceAttachmentWithFilename_);
2721
2722
2723 if (scriptEngine_ != null) {
2724 scriptEngine_.registerWindowAndMaybeStartEventLoop(win);
2725 }
2726
2727 if (pageBeforeLoad != win.getEnclosedPage()) {
2728 updatedWindows.add(win);
2729 }
2730
2731
2732 throwFailingHttpStatusCodeExceptionIfNecessary(loadJob.response_);
2733 }
2734 }
2735
2736 private static void processOnlyHashChange(final WebWindow window, final URL urlWithOnlyHashChange) {
2737 final Page page = window.getEnclosedPage();
2738 final String oldURL = page.getUrl().toExternalForm();
2739
2740
2741 final WebRequest req = page.getWebResponse().getWebRequest();
2742 req.setUrl(urlWithOnlyHashChange);
2743
2744
2745 final Window jsWindow = window.getScriptableObject();
2746 if (null != jsWindow) {
2747 final Location location = jsWindow.getLocation();
2748 location.setHash(oldURL, urlWithOnlyHashChange.getRef());
2749 }
2750
2751
2752 window.getHistory().addPage(page);
2753 }
2754
2755
2756
2757
2758
2759 public WebClientOptions getOptions() {
2760 return options_;
2761 }
2762
2763
2764
2765
2766
2767
2768 public StorageHolder getStorageHolder() {
2769 return storageHolder_;
2770 }
2771
2772
2773
2774
2775
2776
2777
2778 public synchronized Set<Cookie> getCookies(final URL url) {
2779 final CookieManager cookieManager = getCookieManager();
2780
2781 if (!cookieManager.isCookiesEnabled()) {
2782 return Collections.emptySet();
2783 }
2784
2785 final URL normalizedUrl = HttpClientConverter.replaceForCookieIfNecessary(url);
2786
2787 final String host = normalizedUrl.getHost();
2788
2789
2790 if (host.isEmpty()) {
2791 return Collections.emptySet();
2792 }
2793
2794
2795 cookieManager.clearExpired(new Date());
2796
2797 final Set<Cookie> matchingCookies = new LinkedHashSet<>();
2798 HttpClientConverter.addMatching(cookieManager.getCookies(), normalizedUrl,
2799 getBrowserVersion(), matchingCookies);
2800 return Collections.unmodifiableSet(matchingCookies);
2801 }
2802
2803
2804
2805
2806
2807
2808
2809 public void addCookie(final String cookieString, final URL pageUrl, final Object origin) {
2810 final CookieManager cookieManager = getCookieManager();
2811 if (!cookieManager.isCookiesEnabled()) {
2812 if (LOG.isDebugEnabled()) {
2813 LOG.debug("Skipped adding cookie: '" + cookieString
2814 + "' because cookies are not enabled for the CookieManager.");
2815 }
2816 return;
2817 }
2818
2819 try {
2820 final List<Cookie> cookies = HttpClientConverter.parseCookie(cookieString, pageUrl, getBrowserVersion());
2821
2822 for (final Cookie cookie : cookies) {
2823 cookieManager.addCookie(cookie);
2824
2825 if (LOG.isDebugEnabled()) {
2826 LOG.debug("Added cookie: '" + cookieString + "'");
2827 }
2828 }
2829 }
2830 catch (final MalformedCookieException e) {
2831 if (LOG.isDebugEnabled()) {
2832 LOG.warn("Adding cookie '" + cookieString + "' failed.", e);
2833 }
2834 getIncorrectnessListener().notify("Adding cookie '" + cookieString
2835 + "' failed; reason: '" + e.getMessage() + "'.", origin);
2836 }
2837 }
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847 public boolean isJavaScriptEnabled() {
2848 return javaScriptEngineEnabled_ && getOptions().isJavaScriptEnabled();
2849 }
2850
2851
2852
2853
2854
2855
2856
2857 public boolean isJavaScriptEngineEnabled() {
2858 return javaScriptEngineEnabled_;
2859 }
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869 public HtmlPage loadHtmlCodeIntoCurrentWindow(final String htmlCode) throws IOException {
2870 final HTMLParser htmlParser = getPageCreator().getHtmlParser();
2871 final WebWindow webWindow = getCurrentWindow();
2872
2873 final StringWebResponse webResponse =
2874 new StringWebResponse(htmlCode, new URL("https://www.htmlunit.org/dummy.html"));
2875 final HtmlPage page = new HtmlPage(webResponse, webWindow);
2876 webWindow.setEnclosedPage(page);
2877
2878 htmlParser.parse(this, webResponse, page, false, false);
2879 return page;
2880 }
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890 public XHtmlPage loadXHtmlCodeIntoCurrentWindow(final String xhtmlCode) throws IOException {
2891 final HTMLParser htmlParser = getPageCreator().getHtmlParser();
2892 final WebWindow webWindow = getCurrentWindow();
2893
2894 final StringWebResponse webResponse =
2895 new StringWebResponse(xhtmlCode, new URL("https://www.htmlunit.org/dummy.html"));
2896 final XHtmlPage page = new XHtmlPage(webResponse, webWindow);
2897 webWindow.setEnclosedPage(page);
2898
2899 htmlParser.parse(this, webResponse, page, true, false);
2900 return page;
2901 }
2902
2903
2904
2905
2906
2907
2908
2909 public WebSocketAdapter buildWebSocketAdapter(final WebSocketListener webSocketListener) {
2910 return webSocketAdapterFactory_.buildWebSocketAdapter(this, webSocketListener);
2911 }
2912
2913
2914
2915
2916
2917
2918 public void setWebSocketAdapter(final WebSocketAdapterFactory factory) {
2919 webSocketAdapterFactory_ = factory;
2920 }
2921
2922
2923
2924
2925
2926
2927
2928 public PooledCSS3Parser getCSS3Parser() {
2929 return this.css3ParserPool_.get();
2930 }
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948 static class CSS3ParserPool {
2949
2950
2951
2952
2953 private final ConcurrentLinkedDeque<PooledCSS3Parser> parsers_ = new ConcurrentLinkedDeque<>();
2954
2955
2956
2957
2958
2959
2960
2961
2962 public PooledCSS3Parser get() {
2963
2964 final PooledCSS3Parser parser = parsers_.pollLast();
2965
2966
2967 return parser != null ? parser.markInUse(this) : new PooledCSS3Parser(this);
2968 }
2969
2970
2971
2972
2973
2974
2975
2976
2977 protected void recycle(final PooledCSS3Parser parser) {
2978 parsers_.addLast(parser);
2979 }
2980 }
2981
2982
2983
2984
2985
2986
2987
2988
2989 public static class PooledCSS3Parser extends CSS3Parser implements AutoCloseable {
2990
2991
2992
2993
2994 private CSS3ParserPool pool_;
2995
2996
2997
2998
2999
3000
3001 protected PooledCSS3Parser(final CSS3ParserPool pool) {
3002 super();
3003 this.pool_ = pool;
3004 }
3005
3006
3007
3008
3009
3010
3011
3012 protected PooledCSS3Parser markInUse(final CSS3ParserPool pool) {
3013
3014 if (this.pool_ == null) {
3015 this.pool_ = pool;
3016 }
3017 else {
3018 throw new IllegalStateException("This PooledParser was not returned to the pool properly");
3019 }
3020
3021 return this;
3022 }
3023
3024
3025
3026
3027
3028
3029
3030
3031 @Override
3032 public void close() {
3033 if (this.pool_ != null) {
3034 final CSS3ParserPool oldPool = this.pool_;
3035
3036
3037 this.pool_ = null;
3038
3039
3040 oldPool.recycle(this);
3041 }
3042 else {
3043 throw new IllegalStateException("This PooledParser was returned already");
3044 }
3045 }
3046 }
3047 }