View Javadoc
1   /*
2    * Copyright (c) 2002-2026 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit.util;
16  
17  import static java.nio.charset.StandardCharsets.ISO_8859_1;
18  
19  import java.io.BufferedWriter;
20  import java.io.EOFException;
21  import java.io.File;
22  import java.io.IOException;
23  import java.io.InputStream;
24  import java.io.OutputStream;
25  import java.net.URL;
26  import java.nio.charset.StandardCharsets;
27  import java.nio.file.Files;
28  import java.nio.file.StandardOpenOption;
29  import java.util.ArrayList;
30  import java.util.List;
31  import java.util.regex.Pattern;
32  
33  import org.apache.commons.io.FileUtils;
34  import org.apache.commons.io.IOUtils;
35  import org.apache.commons.lang3.StringUtils;
36  import org.apache.commons.logging.Log;
37  import org.apache.commons.logging.LogFactory;
38  import org.htmlunit.FormEncodingType;
39  import org.htmlunit.HttpMethod;
40  import org.htmlunit.WebConnection;
41  import org.htmlunit.WebRequest;
42  import org.htmlunit.WebResponse;
43  import org.htmlunit.WebResponseData;
44  import org.htmlunit.javascript.JavaScriptEngine;
45  
46  /**
47   * Wrapper around a "real" {@link WebConnection} that uses the wrapped connection
48   * to perform the actual work and saves all received responses
49   * in the temp directory with an overview page.<br>
50   * <br>
51   * This may be useful during development to understand what is being browsed.<br>
52   * <br>
53   * Example:
54   * <pre>
55   * final WebClient client = new WebClient();
56   * final WebConnection connection = new DebuggingWebConnection(client.getWebConnection(), "myTest");
57   * client.setWebConnection(connection);
58   * </pre>
59   * In this example an overview page will be generated under the name {@code myTest/index.html} in the temp directory
60   * and all received responses will be saved into the {@code myTest} folder.<br>
61   * <br>
62   * <em>This class is only intended as a debugging aid during development.</em>
63   *
64   * @author Marc Guillemot
65   * @author Ahmed Ashour
66   * @author Ronald Brill
67   */
68  public class DebuggingWebConnection extends WebConnectionWrapper {
69      private static final Log LOG = LogFactory.getLog(DebuggingWebConnection.class);
70  
71      private static final Pattern ESCAPE_QUOTE_PATTERN = Pattern.compile("'");
72  
73      private int counter_;
74      private final WebConnection wrappedWebConnection_;
75      private final File javaScriptFile_;
76      private final File reportFolder_;
77      private boolean uncompressJavaScript_ = true;
78  
79      /**
80       * Wraps a web connection to have a report generated of the received responses.
81       *
82       * @param webConnection the web connection that does the real work
83       * @param dirName the name of the directory to create in the temp folder to save received responses;
84       *        if this folder already exists, it will be deleted first
85       * @throws IOException if there is a problem writing the files
86       */
87      public DebuggingWebConnection(final WebConnection webConnection,
88              final String dirName) throws IOException {
89  
90          super(webConnection);
91  
92          wrappedWebConnection_ = webConnection;
93          final File tmpDir = new File(System.getProperty("java.io.tmpdir"));
94          reportFolder_ = new File(tmpDir, dirName);
95          if (reportFolder_.exists()) {
96              FileUtils.forceDelete(reportFolder_);
97          }
98          FileUtils.forceMkdir(reportFolder_);
99          javaScriptFile_ = new File(reportFolder_, "hu.js");
100         createOverview();
101     }
102 
103     /**
104      * Calls the wrapped web connection and saves the received response.
105      * {@inheritDoc}
106      */
107     @Override
108     public WebResponse getResponse(final WebRequest request) throws IOException {
109         WebResponse response = wrappedWebConnection_.getResponse(request);
110         if (isUncompressJavaScript() && isJavaScript(response.getContentType())) {
111             response = uncompressJavaScript(response);
112         }
113         saveResponse(response, request);
114         return response;
115     }
116 
117     /**
118      * Tries to uncompress the JavaScript code in the provided response.
119      *
120      * @param response the response to uncompress
121      * @return a new response with uncompressed JavaScript code, or the original response if uncompression fails
122      */
123     protected WebResponse uncompressJavaScript(final WebResponse response) {
124         final WebRequest request = response.getWebRequest();
125         final String scriptName = request.getUrl().toString();
126         final String scriptSource = response.getContentAsString();
127 
128         // skip if it is already formatted? => TODO
129 
130         try {
131             final String decompileScript = JavaScriptEngine.uncompressJavaScript(scriptSource, scriptName);
132 
133             final List<NameValuePair> responseHeaders = new ArrayList<>(response.getResponseHeaders());
134             for (int i = responseHeaders.size() - 1; i >= 0; i--) {
135                 if ("content-encoding".equalsIgnoreCase(responseHeaders.get(i).getName())) {
136                     responseHeaders.remove(i);
137                 }
138             }
139             final WebResponseData wrd = new WebResponseData(decompileScript.getBytes(), response.getStatusCode(),
140                 response.getStatusMessage(), responseHeaders);
141             return new WebResponse(wrd, response.getWebRequest().getUrl(),
142                 response.getWebRequest().getHttpMethod(), response.getLoadTime());
143         }
144         catch (final Exception e) {
145             LOG.warn("Failed to decompress JavaScript response. Delivering as it.", e);
146         }
147 
148         return response;
149     }
150 
151     /**
152      * Adds a mark that will be visible in the HTML result page generated by this class.
153      *
154      * @param mark the text of the mark
155      * @throws IOException if a problem occurs writing the file
156      */
157     public void addMark(final String mark) throws IOException {
158         final String escapedMark = mark == null ? null : mark.replace("\"", "\\\"");
159         appendToJSFile("tab[tab.length] = \"" + escapedMark + "\";\n");
160         if (LOG.isInfoEnabled()) {
161             LOG.info("--- " + escapedMark + " ---");
162         }
163     }
164 
165     /**
166      * Saves the response content in the temp directory and adds it to the summary page.
167      *
168      * @param response the response to save
169      * @param request the request used to obtain the response
170      * @throws IOException if a problem occurs writing the file
171      */
172     protected void saveResponse(final WebResponse response, final WebRequest request)
173         throws IOException {
174         counter_++;
175         final String extension = chooseExtension(response.getContentType());
176         final File file = createFile(request.getUrl(), extension);
177         int length = 0;
178         try (InputStream input = response.getContentAsStream()) {
179             try (OutputStream fos = Files.newOutputStream(file.toPath())) {
180                 length = IOUtils.copy(input, fos);
181             }
182             catch (final EOFException ignored) {
183                 // ignore
184             }
185         }
186 
187         final URL url = response.getWebRequest().getUrl();
188         if (LOG.isInfoEnabled()) {
189             LOG.info("Created file " + file.getAbsolutePath() + " for response " + counter_ + ": " + url);
190         }
191 
192         final StringBuilder bduiler = new StringBuilder();
193         bduiler.append("tab[tab.length] = {code: ").append(response.getStatusCode())
194                 .append(", fileName: '").append(file.getName()).append("', ")
195                 .append("contentType: '").append(response.getContentType())
196                 .append("', method: '").append(request.getHttpMethod().name()).append("', ");
197         if (request.getHttpMethod() == HttpMethod.POST && request.getEncodingType() == FormEncodingType.URL_ENCODED) {
198             bduiler.append("postParameters: ").append(nameValueListToJsMap(request.getRequestParameters()))
199                 .append(", ");
200         }
201         bduiler.append("url: '").append(escapeJSString(url.toString()))
202                 .append("', loadTime: ").append(response.getLoadTime())
203                 .append(", responseSize: ").append(length)
204                 .append(", responseHeaders: ").append(nameValueListToJsMap(response.getResponseHeaders()))
205                 .append("};\n");
206         appendToJSFile(bduiler.toString());
207     }
208 
209     static String escapeJSString(final String string) {
210         return ESCAPE_QUOTE_PATTERN.matcher(string).replaceAll("\\\\'");
211     }
212 
213     static String chooseExtension(final String contentType) {
214         if (isJavaScript(contentType)) {
215             return ".js";
216         }
217         else if (MimeType.TEXT_HTML.equals(contentType)) {
218             return ".html";
219         }
220         else if (MimeType.TEXT_CSS.equals(contentType)) {
221             return ".css";
222         }
223         else if (MimeType.TEXT_XML.equals(contentType)) {
224             return ".xml";
225         }
226         else if (MimeType.IMAGE_GIF.equals(contentType)) {
227             return ".gif";
228         }
229         return ".txt";
230     }
231 
232     /**
233      * Returns whether the response contains JavaScript content.
234      *
235      * @param contentType the response's content type
236      * @return {@code false} if the content type is not recognized as JavaScript
237      */
238     static boolean isJavaScript(final String contentType) {
239         return contentType.contains("javascript") || contentType.contains("ecmascript")
240             || (contentType.startsWith("text/") && contentType.endsWith("js"));
241     }
242 
243     /**
244      * Returns whether JavaScript responses should be uncompressed before saving.
245      *
246      * @return {@code true} if JavaScript responses should be uncompressed; default is {@code true}
247      */
248     public boolean isUncompressJavaScript() {
249         return uncompressJavaScript_;
250     }
251 
252     /**
253      * Sets whether responses recognized as JavaScript should be uncompressed.
254      * Uncompressing is useful for debugging when the original script is minified to a single line,
255      * as it produces more readable output and more useful error messages.
256      *
257      * @param decompress {@code true} if JavaScript responses should be uncompressed
258      */
259     public void setUncompressJavaScript(final boolean decompress) {
260         uncompressJavaScript_ = decompress;
261     }
262 
263     private void appendToJSFile(final String str) throws IOException {
264         try (BufferedWriter jsFileWriter = Files.newBufferedWriter(javaScriptFile_.toPath(),
265                                                     StandardCharsets.UTF_8, StandardOpenOption.APPEND)) {
266             jsFileWriter.write(str);
267         }
268     }
269 
270     /**
271      * Computes the best file name to save the response for the given URL.
272      *
273      * @param url the requested URL
274      * @param extension the preferred file extension
275      * @return the file to create
276      * @throws IOException if a problem occurs creating the file
277      */
278     private File createFile(final URL url, final String extension) throws IOException {
279         String name = url.getPath().replaceFirst("/$", "").replaceAll(".*/", "");
280         name = org.htmlunit.util.StringUtils.substringBefore(name, "?"); // remove query
281         name = org.htmlunit.util.StringUtils.substringBefore(name, ";"); // remove additional info
282         name = StringUtils.substring(name, 0, 30); // avoid exceptions due to too long file names
283         name = org.htmlunit.util.StringUtils.sanitizeForFileName(name);
284         if (!name.endsWith(extension)) {
285             name += extension;
286         }
287         int counter = 0;
288         while (true) {
289             final String fileName;
290             if (counter != 0) {
291                 fileName = StringUtils.substringBeforeLast(name, ".")
292                     + "_" + counter + "." + StringUtils.substringAfterLast(name, ".");
293             }
294             else {
295                 fileName = name;
296             }
297             final File f = new File(reportFolder_, fileName);
298             if (f.createNewFile()) {
299                 return f;
300             }
301             counter++;
302         }
303     }
304 
305     /**
306      * Produces a JavaScript map literal string from a list of {@link NameValuePair}s,
307      * e.g. {@code {'key1': 'value1', 'key 2': 'value2'}}.
308      *
309      * @param headers a list of {@link NameValuePair}s
310      * @return the JavaScript map literal string
311      */
312     static String nameValueListToJsMap(final List<NameValuePair> headers) {
313         if (headers == null || headers.isEmpty()) {
314             return "{}";
315         }
316         final StringBuilder bduiler = new StringBuilder("{");
317         for (final NameValuePair header : headers) {
318             bduiler.append('\'').append(header.getName()).append("': '")
319                     .append(escapeJSString(header.getValue())).append("', ");
320         }
321         bduiler.delete(bduiler.length() - 2, bduiler.length());
322         bduiler.append('}');
323         return bduiler.toString();
324     }
325 
326     /**
327      * Creates the summary file and the JavaScript file that will be updated for each received response.
328      *
329      * @throws IOException if a problem occurs writing the files
330      */
331     private void createOverview() throws IOException {
332         FileUtils.writeStringToFile(javaScriptFile_, "var tab = [];\n", ISO_8859_1);
333 
334         final URL indexResource = DebuggingWebConnection.class.getResource("DebuggingWebConnection.index.html");
335         if (indexResource == null) {
336             throw new RuntimeException("Missing dependency DebuggingWebConnection.index.html");
337         }
338         final File summary = new File(reportFolder_, "index.html");
339         FileUtils.copyURLToFile(indexResource, summary);
340 
341         if (LOG.isInfoEnabled()) {
342             LOG.info("Summary will be in " + summary.getAbsolutePath());
343         }
344     }
345 
346     File getReportFolder() {
347         return reportFolder_;
348     }
349 }