1 /*
2 * Copyright (c) 2002-2026 Gargoyle Software Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 * https://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 package org.htmlunit;
16
17 import static java.nio.charset.StandardCharsets.UTF_16BE;
18 import static java.nio.charset.StandardCharsets.UTF_16LE;
19 import static java.nio.charset.StandardCharsets.UTF_8;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.Serializable;
24 import java.net.URL;
25 import java.nio.charset.Charset;
26 import java.util.List;
27
28 import org.apache.commons.io.ByteOrderMark;
29 import org.apache.commons.io.IOUtils;
30 import org.apache.commons.io.input.BOMInputStream;
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.htmlunit.http.HttpStatus;
34 import org.htmlunit.util.EncodingSniffer;
35 import org.htmlunit.util.MimeType;
36 import org.htmlunit.util.NameValuePair;
37 import org.htmlunit.util.StringUtils;
38
39 /**
40 * A response from a web server.
41 *
42 * @author Mike Bowler
43 * @author Brad Clarke
44 * @author Noboru Sinohara
45 * @author Marc Guillemot
46 * @author Ahmed Ashour
47 * @author Ronald Brill
48 * @author Lai Quang Duong
49 */
50 public class WebResponse implements Serializable {
51
52 private static final Log LOG = LogFactory.getLog(WebResponse.class);
53 private static final ByteOrderMark[] BOM_HEADERS = {
54 ByteOrderMark.UTF_8,
55 ByteOrderMark.UTF_16LE,
56 ByteOrderMark.UTF_16BE};
57
58 private final long loadTime_;
59 private final WebResponseData responseData_;
60 private final WebRequest request_;
61 private boolean wasContentCharsetTentative_;
62 private boolean wasBlocked_;
63 private String blockReason_;
64
65 /**
66 * Constructs a web response.
67 *
68 * @param responseData the response data
69 * @param url Where this response came from
70 * @param requestMethod the method used to get this response
71 * @param loadTime How long the response took to be sent
72 */
73 public WebResponse(final WebResponseData responseData, final URL url,
74 final HttpMethod requestMethod, final long loadTime) {
75 this(responseData, new WebRequest(url, requestMethod), loadTime);
76 }
77
78 /**
79 * Constructs a web response.
80 *
81 * @param responseData the response data
82 * @param request the request used to get this response
83 * @param loadTime How long the response took to be sent
84 */
85 public WebResponse(final WebResponseData responseData,
86 final WebRequest request, final long loadTime) {
87 responseData_ = responseData;
88 request_ = request;
89 loadTime_ = loadTime;
90 }
91
92 /**
93 * Returns the request used to load this response.
94 * @return the request used to load this response
95 */
96 public WebRequest getWebRequest() {
97 return request_;
98 }
99
100 /**
101 * Returns the response headers.
102 * @return the response headers as a list of {@link NameValuePair}s
103 */
104 public List<NameValuePair> getResponseHeaders() {
105 return responseData_.getResponseHeaders();
106 }
107
108 /**
109 * Returns the value of the specified response header.
110 * @param headerName the name of the header whose value is to be returned
111 * @return the header value, {@code null} if no response header exists with this name
112 */
113 public String getResponseHeaderValue(final String headerName) {
114 for (final NameValuePair pair : responseData_.getResponseHeaders()) {
115 if (pair.getName().equalsIgnoreCase(headerName)) {
116 return pair.getValue();
117 }
118 }
119 return null;
120 }
121
122 /**
123 * Returns the HTTP status code.
124 * @return the status code that was returned by the server
125 */
126 public int getStatusCode() {
127 return responseData_.getStatusCode();
128 }
129
130 /**
131 * Returns the HTTP status message.
132 * @return the status message that was returned from the server
133 */
134 public String getStatusMessage() {
135 return responseData_.getStatusMessage();
136 }
137
138 /**
139 * Returns the response content type.
140 * @return the content type returned from the server, e.g. "text/html"
141 */
142 public String getContentType() {
143 final String contentTypeHeader = getResponseHeaderValue(HttpHeader.CONTENT_TYPE_LC);
144 if (contentTypeHeader == null) {
145 // Not technically legal but some servers don't return a content-type
146 return "";
147 }
148 final int index = contentTypeHeader.indexOf(';');
149 if (index == -1) {
150 return contentTypeHeader;
151 }
152 return contentTypeHeader.substring(0, index);
153 }
154
155 /**
156 * Returns the content charset specified explicitly in the {@code Content-Type} header
157 * or {@code null} if none was specified.
158 * @return the content charset specified header or {@code null} if none was specified
159 */
160 public Charset getHeaderContentCharset() {
161 final String contentType = getResponseHeaderValue(HttpHeader.CONTENT_TYPE_LC);
162 if (contentType == null) {
163 return null;
164 }
165
166 final int index = contentType.indexOf(';');
167 if (index == -1 || index == 0) {
168 return null;
169 }
170 if (StringUtils.isBlank(contentType.substring(0, index))) {
171 return null;
172 }
173
174 return EncodingSniffer.extractEncodingFromContentType(contentType);
175 }
176
177 /**
178 * Returns the content charset for this response, even if no charset was specified explicitly.
179 * <p>
180 * This method always returns a valid charset. This method first checks the {@code Content-Type}
181 * header or in the content BOM for viable charset. If not found, it attempts to determine the
182 * charset based on the type of the content. As a last resort, this method returns the
183 * value of {@link org.htmlunit.WebRequest#getDefaultResponseContentCharset()} which is
184 * {@link java.nio.charset.StandardCharsets#UTF_8} by default.
185 * </p>
186 * @return the content charset for this response
187 */
188 public Charset getContentCharset() {
189 wasContentCharsetTentative_ = false;
190
191 try (InputStream is = getContentAsStreamWithBomIfApplicable()) {
192 if (is instanceof BOMInputStream stream) {
193 final String bomCharsetName = stream.getBOMCharsetName();
194 if (bomCharsetName != null) {
195 return Charset.forName(bomCharsetName);
196 }
197 }
198
199 Charset charset = getHeaderContentCharset();
200 if (charset != null) {
201 return charset;
202 }
203
204 final String contentType = getContentType();
205 switch (DefaultPageCreator.determinePageType(contentType)) {
206 case HTML:
207 charset = EncodingSniffer.sniffEncodingFromMetaTag(is);
208 wasContentCharsetTentative_ = true;
209 break;
210 case XML:
211 charset = EncodingSniffer.sniffEncodingFromXmlDeclaration(is);
212 if (charset == null) {
213 charset = UTF_8;
214 }
215 break;
216 default:
217 if (MimeType.TEXT_CSS.equals(contentType)) {
218 charset = EncodingSniffer.sniffEncodingFromCssDeclaration(is);
219 }
220 break;
221 }
222
223 if (charset != null) {
224 return charset;
225 }
226 }
227 catch (final IOException e) {
228 LOG.warn("Error trying to sniff encoding.", e);
229 wasContentCharsetTentative_ = true;
230 }
231 return getWebRequest().getDefaultResponseContentCharset();
232 }
233
234 /**
235 * Returns whether the charset of the previous call to {@link #getContentCharset()} was "tentative".
236 * <p>
237 * A charset is classed as "tentative" if its detection is prone to false positive/negatives.
238 * </p>
239 * <p>
240 * For example, HTML meta-tag sniffing can be fooled by text that looks-like-a-meta-tag inside
241 * JavaScript code (false positive) or if the meta-tag is after the first 1024 bytes (false negative).
242 * </p>
243 * @return {@code true} if the charset of the previous call to {@link #getContentCharset()} was
244 * "tentative".
245 * @see <a href="https://html.spec.whatwg.org/multipage/parsing.html#concept-encoding-confidence">
246 * https://html.spec.whatwg.org/multipage/parsing.html#concept-encoding-confidence</a>
247 */
248 public boolean wasContentCharsetTentative() {
249 return wasContentCharsetTentative_;
250 }
251
252 /**
253 * Returns the response content as a string, using the charset/encoding specified in the server response.
254 * @return the response content as a string, using the charset/encoding specified in the server response
255 * or null if the content retrieval was failing
256 */
257 public String getContentAsString() {
258 return getContentAsString(getContentCharset());
259 }
260
261 /**
262 * Returns the response content as a string, using the specified charset,
263 * rather than the charset/encoding specified in the server response.
264 * If there is a bom header the charset parameter will be overwritten by the bom.
265 * @param encoding the charset/encoding to use to convert the response content into a string
266 * @return the response content as a string or null if the content retrieval was failing
267 */
268 public String getContentAsString(final Charset encoding) {
269 if (responseData_ != null) {
270 try (InputStream in = responseData_.getInputStreamWithBomIfApplicable(BOM_HEADERS)) {
271 if (in instanceof BOMInputStream bomIn) {
272 // there seems to be a bug in BOMInputStream
273 // we have to call this before hasBOM(ByteOrderMark)
274 if (bomIn.hasBOM()) {
275 if (bomIn.hasBOM(ByteOrderMark.UTF_8)) {
276 return IOUtils.toString(bomIn, UTF_8);
277 }
278 if (bomIn.hasBOM(ByteOrderMark.UTF_16BE)) {
279 return IOUtils.toString(bomIn, UTF_16BE);
280 }
281 if (bomIn.hasBOM(ByteOrderMark.UTF_16LE)) {
282 return IOUtils.toString(bomIn, UTF_16LE);
283 }
284 }
285 return IOUtils.toString(bomIn, encoding);
286 }
287
288 return IOUtils.toString(in, encoding);
289 }
290 catch (final IOException e) {
291 LOG.warn(e.getMessage(), e);
292 }
293 }
294 return null;
295 }
296
297 /**
298 * Returns length of the content data.
299 * @return the length
300 */
301 public long getContentLength() {
302 if (responseData_ == null) {
303 return 0;
304 }
305 return responseData_.getContentLength();
306 }
307
308 /**
309 * Returns the response content as an input stream.
310 * @return the response content as an input stream
311 * @throws IOException in case of IOProblems
312 */
313 public InputStream getContentAsStream() throws IOException {
314 return responseData_.getInputStream();
315 }
316
317 /**
318 * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
319 *
320 * @return the associated InputStream wrapped with a bom input stream if applicable
321 * @throws IOException in case of IO problems
322 */
323 public InputStream getContentAsStreamWithBomIfApplicable() throws IOException {
324 if (responseData_ != null) {
325 return responseData_.getInputStreamWithBomIfApplicable(BOM_HEADERS);
326 }
327 return null;
328 }
329
330 /**
331 * Returns the time it took to load this web response, in milliseconds.
332 * @return the time it took to load this web response, in milliseconds
333 */
334 public long getLoadTime() {
335 return loadTime_;
336 }
337
338 /**
339 * Clean up the response data.
340 */
341 public void cleanUp() {
342 if (responseData_ != null) {
343 responseData_.cleanUp();
344 }
345 }
346
347 /**
348 * Returns whether the response has a successful HTTP status code.
349 *
350 * @return {@code true} if the status code is in the 2xx range
351 */
352 public boolean isSuccess() {
353 final int statusCode = getStatusCode();
354 return statusCode >= HttpStatus.OK_200 && statusCode < HttpStatus.MULTIPLE_CHOICES_300;
355 }
356
357 /**
358 * Returns whether the response has a successful HTTP status code.
359 *
360 * @return {@code true} if the status code is in the 2xx range or 305
361 */
362 public boolean isSuccessOrUseProxy() {
363 final int statusCode = getStatusCode();
364 return (statusCode >= HttpStatus.OK_200 && statusCode < HttpStatus.MULTIPLE_CHOICES_300)
365 || statusCode == HttpStatus.USE_PROXY_305;
366 }
367
368 /**
369 * Returns whether the response has a successful HTTP status code.
370 *
371 * @return {@code true} if the status code is in the 2xx range or 305
372 */
373 public boolean isSuccessOrUseProxyOrNotModified() {
374 final int statusCode = getStatusCode();
375 return (statusCode >= HttpStatus.OK_200 && statusCode < HttpStatus.MULTIPLE_CHOICES_300)
376 || statusCode == HttpStatus.USE_PROXY_305
377 || statusCode == HttpStatus.NOT_MODIFIED_304;
378 }
379
380 /**
381 * Returns whether the request was blocked.
382 *
383 * @return {@code true} if the request was blocked
384 */
385 public boolean wasBlocked() {
386 return wasBlocked_;
387 }
388
389 /**
390 * Returns the reason for blocking or null.
391 *
392 * @return the reason for blocking or null
393 */
394 public String getBlockReason() {
395 return blockReason_;
396 }
397
398 /**
399 * Sets the wasBlocked state to true.
400 *
401 * @param blockReason the reason
402 */
403 public void markAsBlocked(final String blockReason) {
404 wasBlocked_ = true;
405 blockReason_ = blockReason;
406 }
407 }