1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package org.htmlunit.util;
16
17 import static java.nio.charset.StandardCharsets.US_ASCII;
18 import static java.nio.charset.StandardCharsets.UTF_16BE;
19 import static java.nio.charset.StandardCharsets.UTF_16LE;
20 import static java.nio.charset.StandardCharsets.UTF_8;
21
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.nio.charset.Charset;
25 import java.nio.charset.IllegalCharsetNameException;
26 import java.nio.charset.UnsupportedCharsetException;
27 import java.util.Arrays;
28 import java.util.List;
29 import java.util.Locale;
30
31 import org.apache.commons.io.ByteOrderMark;
32 import org.apache.commons.io.IOUtils;
33 import org.apache.commons.lang3.ArrayUtils;
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.htmlunit.HttpHeader;
37 import org.htmlunit.cyberneko.xerces.util.StandardEncodingTranslator;
38
39
40
41
42
43
44
45
46
47
48
49 public final class EncodingSniffer {
50
51
52 private static final Log LOG = LogFactory.getLog(EncodingSniffer.class);
53
54
55 private static final byte[][] COMMENT_START = {
56 new byte[] {'<'},
57 new byte[] {'!'},
58 new byte[] {'-'},
59 new byte[] {'-'}
60 };
61
62
63 private static final byte[][] META_START = {
64 new byte[] {'<'},
65 new byte[] {'m', 'M'},
66 new byte[] {'e', 'E'},
67 new byte[] {'t', 'T'},
68 new byte[] {'a', 'A'},
69 new byte[] {0x09, 0x0A, 0x0C, 0x0D, 0x20, 0x2F}
70 };
71
72
73 private static final byte[][] OTHER_START = {
74 new byte[] {'<'},
75 new byte[] {'!', '/', '?'}
76 };
77
78
79 private static final byte[][] CHARSET_START = {
80 new byte[] {'c', 'C'},
81 new byte[] {'h', 'H'},
82 new byte[] {'a', 'A'},
83 new byte[] {'r', 'R'},
84 new byte[] {'s', 'S'},
85 new byte[] {'e', 'E'},
86 new byte[] {'t', 'T'}
87 };
88
89 private static final byte[] WHITESPACE = {0x09, 0x0A, 0x0C, 0x0D, 0x20, 0x3E};
90 private static final byte[] COMMENT_END = {'-', '-', '>'};
91
92 private static final byte[] XML_DECLARATION_PREFIX = "<?xml ".getBytes(US_ASCII);
93
94 private static final byte[] CSS_CHARSET_DECLARATION_PREFIX = "@charset \"".getBytes(US_ASCII);
95
96
97
98
99 private static final int SIZE_OF_HTML_CONTENT_SNIFFED = 1024;
100
101
102
103
104
105 private static final int SIZE_OF_XML_CONTENT_SNIFFED = 512;
106
107 private static final int SIZE_OF_CSS_CONTENT_SNIFFED = 1024;
108
109
110
111
112 private EncodingSniffer() {
113
114 }
115
116
117
118
119
120
121
122
123
124 static boolean contentTypeEndsWith(final List<NameValuePair> headers, final String... contentTypeEndings) {
125 for (final NameValuePair pair : headers) {
126 final String name = pair.getName();
127 if (HttpHeader.CONTENT_TYPE_LC.equalsIgnoreCase(name)) {
128 String value = pair.getValue();
129 final int i = value.indexOf(';');
130 if (i != -1) {
131 value = value.substring(0, i);
132 }
133 value = value.trim().toLowerCase(Locale.ROOT);
134 for (final String ending : contentTypeEndings) {
135 if (value.endsWith(ending.toLowerCase(Locale.ROOT))) {
136 return true;
137 }
138 }
139 return false;
140 }
141 }
142 return false;
143 }
144
145
146
147
148
149
150
151
152
153 static Charset sniffEncodingFromUnicodeBom(final byte[] bytes) {
154 if (bytes == null) {
155 return null;
156 }
157
158 Charset encoding = null;
159 if (startsWith(bytes, ByteOrderMark.UTF_8)) {
160 encoding = UTF_8;
161 }
162 else if (startsWith(bytes, ByteOrderMark.UTF_16BE)) {
163 encoding = UTF_16BE;
164 }
165 else if (startsWith(bytes, ByteOrderMark.UTF_16LE)) {
166 encoding = UTF_16LE;
167 }
168
169 if (encoding != null && LOG.isDebugEnabled()) {
170 LOG.debug("Encoding found in Unicode Byte Order Mark: '" + encoding + "'.");
171 }
172 return encoding;
173 }
174
175
176
177
178
179
180
181
182 private static boolean startsWith(final byte[] bytes, final ByteOrderMark bom) {
183 final byte[] bomBytes = bom.getBytes();
184 final byte[] firstBytes = Arrays.copyOfRange(bytes, 0, Math.min(bytes.length, bomBytes.length));
185 return Arrays.equals(firstBytes, bomBytes);
186 }
187
188
189
190
191
192
193
194
195 public static Charset sniffEncodingFromMetaTag(final InputStream is) throws IOException {
196 final byte[] bytes = read(is, SIZE_OF_HTML_CONTENT_SNIFFED);
197 for (int i = 0; i < bytes.length; i++) {
198 if (matches(bytes, i, COMMENT_START)) {
199 i = indexOfSubArray(bytes, COMMENT_END, i);
200 if (i == -1) {
201 break;
202 }
203 i += 2;
204 }
205 else if (matches(bytes, i, META_START)) {
206 i += META_START.length;
207 for (Attribute att = getAttribute(bytes, i); att != null; att = getAttribute(bytes, i)) {
208 i = att.getUpdatedIndex();
209 final String name = att.getName().toLowerCase(Locale.ROOT);
210 if ("charset".equals(name) || "content".equals(name)) {
211 final String value = att.getValue().toLowerCase(Locale.ROOT);
212 Charset charset = null;
213 if ("charset".equals(name)) {
214 charset = toCharset(value);
215
216 if (charset == null && "x-user-defined".equals(value)) {
217 charset = Charset.forName("windows-1252");
218 }
219 }
220 else if ("content".equals(name)) {
221 charset = extractEncodingFromContentType(value);
222
223 if (charset == null && value != null && value.contains("x-user-defined")) {
224 charset = Charset.forName("windows-1252");
225 }
226 if (charset == null) {
227 continue;
228 }
229 }
230 if (UTF_16BE == charset || UTF_16LE == charset) {
231 charset = UTF_8;
232 }
233 if (charset != null) {
234 if (LOG.isDebugEnabled()) {
235 LOG.debug("Encoding found in meta tag: '" + charset + "'.");
236 }
237 return charset;
238 }
239 }
240 }
241 }
242 else if (i + 1 < bytes.length && bytes[i] == '<' && Character.isLetter(bytes[i + 1])) {
243 i = skipToAnyOf(bytes, i, WHITESPACE);
244 if (i == -1) {
245 break;
246 }
247 Attribute att = getAttribute(bytes, i);
248 while (att != null) {
249 i = att.getUpdatedIndex();
250 att = getAttribute(bytes, i);
251 }
252 }
253 else if (i + 2 < bytes.length && bytes[i] == '<' && bytes[i + 1] == '/' && Character.isLetter(bytes[i + 2])) {
254 i = skipToAnyOf(bytes, i, new byte[] {0x09, 0x0A, 0x0C, 0x0D, 0x20, 0x3E});
255 if (i == -1) {
256 break;
257 }
258 Attribute attribute = getAttribute(bytes, i);
259 while (attribute != null) {
260 i = attribute.getUpdatedIndex();
261 attribute = getAttribute(bytes, i);
262 }
263 }
264 else if (matches(bytes, i, OTHER_START)) {
265 i = skipToAnyOf(bytes, i, new byte[] {0x3E});
266 if (i == -1) {
267 break;
268 }
269 }
270 }
271 return null;
272 }
273
274
275
276
277
278
279
280
281
282
283 static Attribute getAttribute(final byte[] bytes, final int startFrom) {
284 if (startFrom >= bytes.length) {
285 return null;
286 }
287
288 int pos = startFrom;
289 while (bytes[pos] == 0x09 || bytes[pos] == 0x0A || bytes[pos] == 0x0C || bytes[pos] == 0x0D || bytes[pos] == 0x20 || bytes[pos] == 0x2F) {
290 pos++;
291 if (pos >= bytes.length) {
292 return null;
293 }
294 }
295 if (bytes[pos] == '>') {
296 return null;
297 }
298 final StringBuilder name = new StringBuilder();
299 final StringBuilder value = new StringBuilder();
300 for ( ;; pos++) {
301 if (pos >= bytes.length) {
302 return new Attribute(name.toString(), value.toString(), pos);
303 }
304 if (bytes[pos] == '=' && name.length() != 0) {
305 pos++;
306 break;
307 }
308 if (bytes[pos] == 0x09 || bytes[pos] == 0x0A || bytes[pos] == 0x0C || bytes[pos] == 0x0D || bytes[pos] == 0x20) {
309 while (bytes[pos] == 0x09 || bytes[pos] == 0x0A || bytes[pos] == 0x0C || bytes[pos] == 0x0D || bytes[pos] == 0x20) {
310 pos++;
311 if (pos >= bytes.length) {
312 return new Attribute(name.toString(), value.toString(), pos);
313 }
314 }
315 if (bytes[pos] != '=') {
316 return new Attribute(name.toString(), value.toString(), pos);
317 }
318 pos++;
319 break;
320 }
321 if (bytes[pos] == '/' || bytes[pos] == '>') {
322 return new Attribute(name.toString(), value.toString(), pos);
323 }
324 name.append((char) bytes[pos]);
325 }
326 if (pos >= bytes.length) {
327 return new Attribute(name.toString(), value.toString(), pos);
328 }
329 while (bytes[pos] == 0x09 || bytes[pos] == 0x0A || bytes[pos] == 0x0C || bytes[pos] == 0x0D || bytes[pos] == 0x20) {
330 pos++;
331 if (pos >= bytes.length) {
332 return new Attribute(name.toString(), value.toString(), pos);
333 }
334 }
335 if (bytes[pos] == '"' || bytes[pos] == '\'') {
336 final byte b = bytes[pos];
337 for (pos++; pos < bytes.length; pos++) {
338 if (bytes[pos] == b) {
339 pos++;
340 return new Attribute(name.toString(), value.toString(), pos);
341 }
342 else if (bytes[pos] >= 'A' && bytes[pos] <= 'Z') {
343 final byte b2 = (byte) (bytes[pos] + 0x20);
344 value.append((char) b2);
345 }
346 else {
347 value.append((char) bytes[pos]);
348 }
349 }
350 return new Attribute(name.toString(), value.toString(), pos);
351 }
352 else if (bytes[pos] == '>') {
353 return new Attribute(name.toString(), value.toString(), pos);
354 }
355 else if (bytes[pos] >= 'A' && bytes[pos] <= 'Z') {
356 final byte b = (byte) (bytes[pos] + 0x20);
357 value.append((char) b);
358 pos++;
359 }
360 else {
361 value.append((char) bytes[pos]);
362 pos++;
363 }
364 for ( ; pos < bytes.length; pos++) {
365 if (bytes[pos] == 0x09 || bytes[pos] == 0x0A || bytes[pos] == 0x0C || bytes[pos] == 0x0D || bytes[pos] == 0x20 || bytes[pos] == 0x3E) {
366 return new Attribute(name.toString(), value.toString(), pos);
367 }
368 else if (bytes[pos] >= 'A' && bytes[pos] <= 'Z') {
369 final byte b = (byte) (bytes[pos] + 0x20);
370 value.append((char) b);
371 }
372 else {
373 value.append((char) bytes[pos]);
374 }
375 }
376 return new Attribute(name.toString(), value.toString(), pos);
377 }
378
379
380
381
382
383
384
385
386
387 public static Charset extractEncodingFromContentType(final String s) {
388 if (s == null) {
389 return null;
390 }
391 final byte[] bytes = s.getBytes(US_ASCII);
392 int i;
393 for (i = 0; i < bytes.length; i++) {
394 if (matches(bytes, i, CHARSET_START)) {
395 i += CHARSET_START.length;
396 break;
397 }
398 }
399 if (i == bytes.length) {
400 return null;
401 }
402 while (bytes[i] == 0x09 || bytes[i] == 0x0A || bytes[i] == 0x0C || bytes[i] == 0x0D || bytes[i] == 0x20) {
403 i++;
404 if (i == bytes.length) {
405 return null;
406 }
407 }
408 if (bytes[i] != '=') {
409 return null;
410 }
411 do {
412 i++;
413 if (i == bytes.length) {
414 return null;
415 }
416 }
417 while (bytes[i] == 0x09 || bytes[i] == 0x0A || bytes[i] == 0x0C || bytes[i] == 0x0D || bytes[i] == 0x20);
418
419 if (bytes[i] == '"') {
420 if (bytes.length <= i + 1) {
421 return null;
422 }
423 final int index = ArrayUtils.indexOf(bytes, (byte) '"', i + 1);
424 if (index == -1) {
425 return null;
426 }
427 final String charsetName = new String(ArrayUtils.subarray(bytes, i + 1, index), US_ASCII);
428 return toCharset(charsetName);
429 }
430 if (bytes[i] == '\'') {
431 if (bytes.length <= i + 1) {
432 return null;
433 }
434 final int index = ArrayUtils.indexOf(bytes, (byte) '\'', i + 1);
435 if (index == -1) {
436 return null;
437 }
438 final String charsetName = new String(ArrayUtils.subarray(bytes, i + 1, index), US_ASCII);
439 return toCharset(charsetName);
440 }
441 int end = skipToAnyOf(bytes, i, new byte[] {0x09, 0x0A, 0x0C, 0x0D, 0x20, 0x3B});
442 if (end == -1) {
443 end = bytes.length;
444 }
445 final String charsetName = new String(ArrayUtils.subarray(bytes, i, end), US_ASCII);
446 return toCharset(charsetName);
447 }
448
449
450
451
452
453
454
455
456
457 public static Charset sniffEncodingFromXmlDeclaration(final InputStream is) throws IOException {
458 final byte[] bytes = read(is, SIZE_OF_XML_CONTENT_SNIFFED);
459 Charset encoding = null;
460 if (bytes.length > 5
461 && XML_DECLARATION_PREFIX[0] == bytes[0]
462 && XML_DECLARATION_PREFIX[1] == bytes[1]
463 && XML_DECLARATION_PREFIX[2] == bytes[2]
464 && XML_DECLARATION_PREFIX[3] == bytes[3]
465 && XML_DECLARATION_PREFIX[4] == bytes[4]
466 && XML_DECLARATION_PREFIX[5] == bytes[5]) {
467 final int index = ArrayUtils.indexOf(bytes, (byte) '?', 2);
468 if (index + 1 < bytes.length && bytes[index + 1] == '>') {
469 final String declaration = new String(bytes, 0, index + 2, US_ASCII);
470 int start = declaration.indexOf("encoding");
471 if (start != -1) {
472 start += 8;
473 final char delimiter;
474 outer:
475 while (true) {
476 switch (declaration.charAt(start)) {
477 case '"':
478 case '\'':
479 delimiter = declaration.charAt(start);
480 start = start + 1;
481 break outer;
482
483 default:
484 start++;
485 }
486 }
487 final int end = declaration.indexOf(delimiter, start);
488 encoding = toCharset(declaration.substring(start, end));
489 }
490 }
491 }
492 if (encoding != null && LOG.isDebugEnabled()) {
493 LOG.debug("Encoding found in XML declaration: '" + encoding + "'.");
494 }
495 return encoding;
496 }
497
498
499
500
501
502
503
504
505
506
507 public static Charset sniffEncodingFromCssDeclaration(final InputStream is) throws IOException {
508 final byte[] bytes = read(is, SIZE_OF_CSS_CONTENT_SNIFFED);
509 if (bytes.length < CSS_CHARSET_DECLARATION_PREFIX.length) {
510 return null;
511 }
512 for (int i = 0; i < CSS_CHARSET_DECLARATION_PREFIX.length; i++) {
513 if (bytes[i] != CSS_CHARSET_DECLARATION_PREFIX[i]) {
514 return null;
515 }
516 }
517
518 Charset encoding = null;
519 final int index = ArrayUtils.indexOf(bytes, (byte) '"', CSS_CHARSET_DECLARATION_PREFIX.length);
520 if (index + 1 < bytes.length && bytes[index + 1] == ';') {
521 encoding = toCharset(new String(bytes, CSS_CHARSET_DECLARATION_PREFIX.length, index - CSS_CHARSET_DECLARATION_PREFIX.length, US_ASCII));
522
523 if (encoding == UTF_16BE || encoding == UTF_16LE) {
524 encoding = UTF_8;
525 }
526 }
527 return encoding;
528 }
529
530
531
532
533
534
535
536
537 public static Charset toCharset(final String charsetName) {
538 final String nameFromLabel = translateEncodingLabel(charsetName);
539 if (nameFromLabel == null) {
540 return null;
541 }
542 try {
543 return Charset.forName(nameFromLabel);
544 }
545 catch (final IllegalCharsetNameException | UnsupportedCharsetException e) {
546 return null;
547 }
548 }
549
550
551
552
553
554
555
556
557
558
559 static boolean matches(final byte[] bytes, final int i, final byte[][] sought) {
560 if (i + sought.length > bytes.length) {
561 return false;
562 }
563 for (int x = 0; x < sought.length; x++) {
564 final byte[] possibilities = sought[x];
565 boolean match = false;
566 for (final byte possibility : possibilities) {
567 if (bytes[i + x] == possibility) {
568 match = true;
569 break;
570 }
571 }
572 if (!match) {
573 return false;
574 }
575 }
576 return true;
577 }
578
579
580
581
582
583
584
585
586
587
588 static int skipToAnyOf(final byte[] bytes, final int startFrom, final byte[] targets) {
589 int i = startFrom;
590 for ( ; i < bytes.length; i++) {
591 if (org.htmlunit.util.ArrayUtils.contains(targets, bytes[i])) {
592 break;
593 }
594 }
595 if (i == bytes.length) {
596 i = -1;
597 }
598 return i;
599 }
600
601
602
603
604
605
606
607
608
609
610 static int indexOfSubArray(final byte[] array, final byte[] subarray, final int startIndex) {
611 for (int i = startIndex; i < array.length; i++) {
612 boolean found = true;
613 if (i + subarray.length > array.length) {
614 break;
615 }
616 for (int j = 0; j < subarray.length; j++) {
617 final byte a = array[i + j];
618 final byte b = subarray[j];
619 if (a != b) {
620 found = false;
621 break;
622 }
623 }
624 if (found) {
625 return i;
626 }
627 }
628 return -1;
629 }
630
631
632
633
634
635
636
637
638
639
640
641 static byte[] read(final InputStream content, final int size) throws IOException {
642 byte[] bytes = new byte[size];
643
644
645 final int count = IOUtils.read(content, bytes);
646 if (count < size) {
647 final byte[] smaller = new byte[count];
648 System.arraycopy(bytes, 0, smaller, 0, count);
649 bytes = smaller;
650 }
651 return bytes;
652 }
653
654
655
656
657
658
659
660
661
662
663
664
665 static byte[] readAndPrepend(final InputStream content, final int size, final byte[] prefix) throws IOException {
666 final int prefixLength = prefix.length;
667 final byte[] joined = new byte[prefixLength + size];
668
669
670
671 final int count = IOUtils.read(content, joined, prefixLength, joined.length - prefixLength);
672 if (count < size) {
673 final byte[] smaller = new byte[prefixLength + count];
674 System.arraycopy(prefix, 0, smaller, 0, prefix.length);
675 System.arraycopy(joined, prefixLength, smaller, prefixLength, count);
676 return smaller;
677 }
678
679 System.arraycopy(prefix, 0, joined, 0, prefix.length);
680 return joined;
681 }
682
683 static class Attribute {
684 private final String name_;
685 private final String value_;
686 private final int updatedIndex_;
687 Attribute(final String name, final String value, final int updatedIndex) {
688 name_ = name;
689 value_ = value;
690 updatedIndex_ = updatedIndex;
691 }
692 String getName() {
693 return name_;
694 }
695 String getValue() {
696 return value_;
697 }
698 int getUpdatedIndex() {
699 return updatedIndex_;
700 }
701 }
702
703
704
705
706
707
708
709
710 public static String translateEncodingLabel(final String encodingLabel) {
711 if (StringUtils.isEmptyOrNull(encodingLabel)) {
712 return null;
713 }
714
715 final String encLC = encodingLabel.toLowerCase(Locale.ROOT);
716 final String enc = StandardEncodingTranslator.INSTANCE.encodingNameFromLabel(encodingLabel);
717 if (encLC.equals(enc)) {
718 return encodingLabel;
719 }
720 return enc;
721 }
722 }