1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package org.htmlunit;
16
17 import java.io.IOException;
18 import java.io.ObjectInputStream;
19 import java.io.ObjectOutputStream;
20 import java.io.Serializable;
21 import java.net.IDN;
22 import java.net.MalformedURLException;
23 import java.net.URL;
24 import java.nio.charset.Charset;
25 import java.nio.charset.StandardCharsets;
26 import java.util.ArrayList;
27 import java.util.Collections;
28 import java.util.EnumSet;
29 import java.util.HashMap;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Set;
33 import java.util.regex.Pattern;
34
35 import org.apache.http.auth.Credentials;
36 import org.htmlunit.http.HttpUtils;
37 import org.htmlunit.httpclient.HtmlUnitUsernamePasswordCredentials;
38 import org.htmlunit.util.NameValuePair;
39 import org.htmlunit.util.UrlUtils;
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56 @SuppressWarnings("PMD.TooManyFields")
57 public class WebRequest implements Serializable {
58
59
60
61
62 public enum HttpHint {
63
64 IncludeCharsetInContentTypeHeader,
65
66
67 BlockCookies
68 }
69
70 private static final Pattern DOT_PATTERN = Pattern.compile("/\\./");
71 private static final Pattern DOT_DOT_PATTERN = Pattern.compile("/(?!\\.\\.)[^/]*/\\.\\./");
72 private static final Pattern REMOVE_DOTS_PATTERN = Pattern.compile("^/(\\.\\.?/)*");
73
74 private String url_;
75 private String proxyHost_;
76 private int proxyPort_;
77 private String proxyScheme_;
78 private boolean isSocksProxy_;
79 private HttpMethod httpMethod_ = HttpMethod.GET;
80 private FormEncodingType encodingType_ = FormEncodingType.URL_ENCODED;
81 private Map<String, String> additionalHeaders_ = new HashMap<>();
82 private Credentials urlCredentials_;
83 private Credentials credentials_;
84 private int timeout_;
85 private transient Set<HttpHint> httpHints_;
86
87 private transient Charset charset_ = StandardCharsets.ISO_8859_1;
88
89
90 private transient Charset defaultResponseContentCharset_ = StandardCharsets.ISO_8859_1;
91
92
93
94
95
96 private List<NameValuePair> requestParameters_ = Collections.emptyList();
97 private String requestBody_;
98
99
100
101
102
103
104
105
106
107
108 public WebRequest(final URL url, final String acceptHeader, final String acceptEncodingHeader) {
109 setUrl(url);
110 if (acceptHeader != null) {
111 setAdditionalHeader(HttpHeader.ACCEPT, acceptHeader);
112 }
113 if (acceptEncodingHeader != null) {
114 setAdditionalHeader(HttpHeader.ACCEPT_ENCODING, acceptEncodingHeader);
115 }
116 timeout_ = -1;
117 }
118
119
120
121
122
123
124
125
126
127
128 public WebRequest(final URL url, final Charset charset, final URL refererUrl) {
129 setUrl(url);
130 setCharset(charset);
131 setRefererHeader(refererUrl);
132 }
133
134
135
136
137
138
139 public static WebRequest newAboutBlankRequest() {
140 return new WebRequest(UrlUtils.URL_ABOUT_BLANK, "*/*", "gzip, deflate");
141 }
142
143
144
145
146
147
148
149
150 public WebRequest(final URL url) {
151 this(url, "*/*", "gzip, deflate");
152 }
153
154
155
156
157
158
159
160
161
162
163 public WebRequest(final URL url, final HttpMethod submitMethod) {
164 this(url);
165 setHttpMethod(submitMethod);
166 }
167
168
169
170
171
172
173
174
175 public URL getUrl() {
176 return UrlUtils.toUrlSafe(url_);
177 }
178
179
180
181
182
183
184
185
186
187 public void setUrl(URL url) {
188 if (url == null) {
189 url_ = null;
190 return;
191 }
192
193 final String path = url.getPath();
194 if (path.isEmpty()) {
195 if (!url.getFile().isEmpty() || url.getProtocol().startsWith("http")) {
196 url = buildUrlWithNewPath(url, "/");
197 }
198 }
199 else if (path.contains("/.")) {
200 url = buildUrlWithNewPath(url, removeDots(path));
201 }
202
203 try {
204 final String idn = IDN.toASCII(url.getHost());
205 if (!idn.equals(url.getHost())) {
206 url = UrlUtils.getUrlWithNewHost(url, idn);
207 }
208 }
209 catch (final Exception e) {
210 throw new IllegalArgumentException(
211 "Cannot convert the hostname of URL: '" + url.toExternalForm() + "' to ASCII.", e);
212 }
213
214 try {
215 url_ = UrlUtils.removeRedundantPort(url).toExternalForm();
216 }
217 catch (final MalformedURLException e) {
218 throw new RuntimeException("Cannot strip default port of URL: " + url.toExternalForm(), e);
219 }
220
221
222 final String userInfo = url.getUserInfo();
223 if (userInfo != null) {
224 final int splitPos = userInfo.indexOf(':');
225 if (splitPos == -1) {
226 urlCredentials_ = new HtmlUnitUsernamePasswordCredentials(userInfo, new char[0]);
227 }
228 else {
229 final String username = userInfo.substring(0, splitPos);
230 final String password = userInfo.substring(splitPos + 1);
231 urlCredentials_ = new HtmlUnitUsernamePasswordCredentials(username, password.toCharArray());
232 }
233 }
234 }
235
236
237
238
239
240
241
242 private static String removeDots(final String path) {
243 String newPath = path;
244
245
246 newPath = REMOVE_DOTS_PATTERN.matcher(newPath).replaceAll("/");
247 if ("/..".equals(newPath)) {
248 newPath = "/";
249 }
250
251
252 while (DOT_PATTERN.matcher(newPath).find()) {
253 newPath = DOT_PATTERN.matcher(newPath).replaceAll("/");
254 }
255
256
257
258 while (DOT_DOT_PATTERN.matcher(newPath).find()) {
259 newPath = DOT_DOT_PATTERN.matcher(newPath).replaceAll("/");
260 }
261
262 return newPath;
263 }
264
265 private static URL buildUrlWithNewPath(URL url, final String newPath) {
266 try {
267 url = UrlUtils.getUrlWithNewPath(url, newPath);
268 }
269 catch (final Exception e) {
270 throw new RuntimeException("Cannot change path of URL: " + url.toExternalForm(), e);
271 }
272 return url;
273 }
274
275
276
277
278
279
280
281
282 public String getProxyHost() {
283 return proxyHost_;
284 }
285
286
287
288
289
290
291
292
293 public void setProxyHost(final String proxyHost) {
294 proxyHost_ = proxyHost;
295 }
296
297
298
299
300
301
302
303
304 public int getProxyPort() {
305 return proxyPort_;
306 }
307
308
309
310
311
312
313
314
315 public void setProxyPort(final int proxyPort) {
316 proxyPort_ = proxyPort;
317 }
318
319
320
321
322
323
324
325
326 public String getProxyScheme() {
327 return proxyScheme_;
328 }
329
330
331
332
333
334
335
336
337
338 public void setProxyScheme(final String proxyScheme) {
339 proxyScheme_ = proxyScheme;
340 }
341
342
343
344
345
346
347
348
349
350 public boolean isSocksProxy() {
351 return isSocksProxy_;
352 }
353
354
355
356
357
358
359
360
361
362 public void setSocksProxy(final boolean isSocksProxy) {
363 isSocksProxy_ = isSocksProxy;
364 }
365
366
367
368
369
370
371
372 public int getTimeout() {
373 return timeout_;
374 }
375
376
377
378
379
380
381
382
383
384 public void setTimeout(final int timeout) {
385 timeout_ = timeout;
386 }
387
388
389
390
391
392
393
394
395
396 public FormEncodingType getEncodingType() {
397 return encodingType_;
398 }
399
400
401
402
403
404
405
406
407
408 public void setEncodingType(final FormEncodingType encodingType) {
409 encodingType_ = encodingType;
410 }
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428 public List<NameValuePair> getParameters() {
429
430
431
432
433
434
435
436 final List<NameValuePair> allParameters = new ArrayList<>(
437 HttpUtils.parseUrlQuery(getUrl().getQuery(), getCharset()));
438
439
440 final HttpMethod httpMethod = getHttpMethod();
441 if (httpMethod == HttpMethod.POST
442 || httpMethod == HttpMethod.PUT
443 || httpMethod == HttpMethod.PATCH
444 || httpMethod == HttpMethod.DELETE
445 || httpMethod == HttpMethod.OPTIONS) {
446 if (FormEncodingType.URL_ENCODED == getEncodingType()
447 && httpMethod != HttpMethod.OPTIONS) {
448
449
450 if (getRequestBody() == null) {
451 allParameters.addAll(getRequestParameters());
452 }
453 else {
454 allParameters.addAll(HttpUtils.parseUrlQuery(getRequestBody(), getCharset()));
455 }
456 }
457 else if (FormEncodingType.MULTIPART == getEncodingType()) {
458 if (httpMethod == HttpMethod.POST) {
459 allParameters.addAll(getRequestParameters());
460 }
461 else {
462
463
464 allParameters.addAll(0, getRequestParameters());
465 }
466 }
467 }
468
469 return normalize(allParameters);
470 }
471
472 private static List<NameValuePair> normalize(final List<NameValuePair> pairs) {
473 if (pairs == null || pairs.isEmpty()) {
474 return pairs;
475 }
476
477 final List<NameValuePair> resultingPairs = new ArrayList<>();
478 for (final NameValuePair pair : pairs) {
479 resultingPairs.add(pair.normalized());
480 }
481
482 return resultingPairs;
483 }
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499 public List<NameValuePair> getRequestParameters() {
500 return requestParameters_;
501 }
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518 public void setRequestParameters(final List<NameValuePair> requestParameters) throws RuntimeException {
519 if (requestBody_ != null) {
520 final String msg = "Trying to set the request parameters, but the request body has already been specified;"
521 + "the two are mutually exclusive!";
522 throw new RuntimeException(msg);
523 }
524 requestParameters_ = requestParameters;
525 }
526
527
528
529
530
531
532
533
534
535
536
537
538
539 public String getRequestBody() {
540 return requestBody_;
541 }
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558 public void setRequestBody(final String requestBody) throws RuntimeException {
559 if (requestParameters_ != null && !requestParameters_.isEmpty()) {
560 final String msg = "Trying to set the request body, but the request parameters have already been specified;"
561 + "the two are mutually exclusive!";
562 throw new RuntimeException(msg);
563 }
564 if (httpMethod_ != HttpMethod.POST
565 && httpMethod_ != HttpMethod.PUT
566 && httpMethod_ != HttpMethod.PATCH
567 && httpMethod_ != HttpMethod.DELETE
568 && httpMethod_ != HttpMethod.OPTIONS) {
569 final String msg = "The request body may only be set for POST, PUT, PATCH, DELETE or OPTIONS requests!";
570 throw new RuntimeException(msg);
571 }
572 requestBody_ = requestBody;
573 }
574
575
576
577
578
579
580
581
582
583 public HttpMethod getHttpMethod() {
584 return httpMethod_;
585 }
586
587
588
589
590
591
592
593
594
595 public void setHttpMethod(final HttpMethod submitMethod) {
596 httpMethod_ = submitMethod;
597 }
598
599
600
601
602
603
604
605
606
607 public Map<String, String> getAdditionalHeaders() {
608 return additionalHeaders_;
609 }
610
611
612
613
614
615
616
617
618
619 public void setAdditionalHeaders(final Map<String, String> additionalHeaders) {
620 additionalHeaders_ = additionalHeaders;
621 }
622
623
624
625
626
627
628
629
630
631
632
633
634 public boolean isAdditionalHeader(final String name) {
635 for (final String key : additionalHeaders_.keySet()) {
636 if (name.equalsIgnoreCase(key)) {
637 return true;
638 }
639 }
640 return false;
641 }
642
643
644
645
646
647
648
649
650
651
652 public String getAdditionalHeader(final String name) {
653 String newKey = name;
654 for (final String key : additionalHeaders_.keySet()) {
655 if (name.equalsIgnoreCase(key)) {
656 newKey = key;
657 break;
658 }
659 }
660 return additionalHeaders_.get(newKey);
661 }
662
663
664
665
666
667
668
669
670
671 public void setRefererHeader(final URL url) {
672 if (url == null || !url.getProtocol().startsWith("http")) {
673 return;
674 }
675
676 try {
677 setAdditionalHeader(HttpHeader.REFERER, UrlUtils.getUrlWithoutRef(url).toExternalForm());
678 }
679 catch (final MalformedURLException ignored) {
680
681 }
682 }
683
684
685
686
687
688
689
690
691
692
693 public void setAdditionalHeader(final String name, final String value) {
694 String newKey = name;
695 for (final String key : additionalHeaders_.keySet()) {
696 if (name.equalsIgnoreCase(key)) {
697 newKey = key;
698 break;
699 }
700 }
701 additionalHeaders_.put(newKey, value);
702 }
703
704
705
706
707
708
709
710
711
712 public void removeAdditionalHeader(String name) {
713 for (final String key : additionalHeaders_.keySet()) {
714 if (name.equalsIgnoreCase(key)) {
715 name = key;
716 break;
717 }
718 }
719 additionalHeaders_.remove(name);
720 }
721
722
723
724
725
726
727
728
729
730 public Credentials getUrlCredentials() {
731 return urlCredentials_;
732 }
733
734
735
736
737
738
739
740
741
742 public Credentials getCredentials() {
743 return credentials_;
744 }
745
746
747
748
749
750
751
752
753
754 public void setCredentials(final Credentials credentials) {
755 credentials_ = credentials;
756 }
757
758
759
760
761
762
763
764
765
766 public Charset getCharset() {
767 return charset_;
768 }
769
770
771
772
773
774
775
776
777
778
779 public void setCharset(final Charset charset) {
780 charset_ = charset;
781 }
782
783
784
785
786
787
788
789
790
791 public Charset getDefaultResponseContentCharset() {
792 return defaultResponseContentCharset_;
793 }
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808 public void setDefaultResponseContentCharset(final Charset defaultResponseContentCharset) {
809 WebAssert.notNull("defaultResponseContentCharset", defaultResponseContentCharset);
810 defaultResponseContentCharset_ = defaultResponseContentCharset;
811 }
812
813
814
815
816
817
818
819
820 public boolean hasHint(final HttpHint hint) {
821 if (httpHints_ == null) {
822 return false;
823 }
824 return httpHints_.contains(hint);
825 }
826
827
828
829
830
831
832
833
834
835 public void addHint(final HttpHint hint) {
836 if (httpHints_ == null) {
837 httpHints_ = EnumSet.noneOf(HttpHint.class);
838 }
839 httpHints_.add(hint);
840 }
841
842
843
844
845
846
847
848
849
850 @Override
851 public String toString() {
852 final StringBuilder builder = new StringBuilder(100)
853 .append(getClass().getSimpleName())
854 .append("[<url=\"")
855 .append(url_)
856 .append("\", ").append(httpMethod_)
857 .append(", ").append(encodingType_)
858 .append(", ").append(requestParameters_)
859 .append(", ").append(additionalHeaders_)
860 .append(", ").append(credentials_)
861 .append(">]");
862 return builder.toString();
863 }
864
865 private void writeObject(final ObjectOutputStream oos) throws IOException {
866 oos.defaultWriteObject();
867 oos.writeObject(charset_ == null ? null : charset_.name());
868 oos.writeObject(defaultResponseContentCharset_ == null ? null : defaultResponseContentCharset_.name());
869 }
870
871 private void readObject(final ObjectInputStream ois) throws ClassNotFoundException, IOException {
872 ois.defaultReadObject();
873 final String charsetName = (String) ois.readObject();
874 if (charsetName != null) {
875 charset_ = Charset.forName(charsetName);
876 }
877 final String defaultResponseContentCharset = (String) ois.readObject();
878 if (defaultResponseContentCharset != null) {
879 defaultResponseContentCharset_ = Charset.forName(defaultResponseContentCharset);
880 }
881 }
882 }