1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package org.htmlunit.html.serializer;
16
17 import static org.htmlunit.css.CssStyleSheet.BLOCK;
18 import static org.htmlunit.html.DomElement.ATTRIBUTE_NOT_DEFINED;
19
20 import java.util.IdentityHashMap;
21 import java.util.Iterator;
22 import java.util.List;
23 import java.util.Map;
24
25 import org.htmlunit.Page;
26 import org.htmlunit.SgmlPage;
27 import org.htmlunit.WebWindow;
28 import org.htmlunit.html.DomComment;
29 import org.htmlunit.html.DomElement;
30 import org.htmlunit.html.DomNode;
31 import org.htmlunit.html.DomText;
32 import org.htmlunit.html.HtmlBody;
33 import org.htmlunit.html.HtmlBreak;
34 import org.htmlunit.html.HtmlCheckBoxInput;
35 import org.htmlunit.html.HtmlDetails;
36 import org.htmlunit.html.HtmlElement;
37 import org.htmlunit.html.HtmlElement.DisplayStyle;
38 import org.htmlunit.html.HtmlHiddenInput;
39 import org.htmlunit.html.HtmlInlineFrame;
40 import org.htmlunit.html.HtmlInput;
41 import org.htmlunit.html.HtmlListItem;
42 import org.htmlunit.html.HtmlNoFrames;
43 import org.htmlunit.html.HtmlNoScript;
44 import org.htmlunit.html.HtmlNumberInput;
45 import org.htmlunit.html.HtmlOption;
46 import org.htmlunit.html.HtmlOrderedList;
47 import org.htmlunit.html.HtmlPreformattedText;
48 import org.htmlunit.html.HtmlRadioButtonInput;
49 import org.htmlunit.html.HtmlResetInput;
50 import org.htmlunit.html.HtmlScript;
51 import org.htmlunit.html.HtmlSelect;
52 import org.htmlunit.html.HtmlStyle;
53 import org.htmlunit.html.HtmlSubmitInput;
54 import org.htmlunit.html.HtmlSummary;
55 import org.htmlunit.html.HtmlTable;
56 import org.htmlunit.html.HtmlTableCell;
57 import org.htmlunit.html.HtmlTableFooter;
58 import org.htmlunit.html.HtmlTableHeader;
59 import org.htmlunit.html.HtmlTableRow;
60 import org.htmlunit.html.HtmlTextArea;
61 import org.htmlunit.html.HtmlTitle;
62 import org.htmlunit.html.HtmlUnorderedList;
63 import org.htmlunit.html.TableRowGroup;
64 import org.htmlunit.html.serializer.HtmlSerializerNormalizedText.HtmlSerializerTextBuilder.Mode;
65
66
67
68
69
70
71
72
73
74
75
76
77 public class HtmlSerializerNormalizedText {
78
79 private boolean ignoreMaskedElements_ = true;
80
81
82
83
84
85 private final Map<DomNode, Boolean> visibilityCache_ = new IdentityHashMap<>();
86
87
88
89
90
91
92 public String asText(final DomNode node) {
93 visibilityCache_.clear();
94 final HtmlSerializerTextBuilder builder = new HtmlSerializerTextBuilder();
95 appendNode(builder, node);
96 return builder.getText();
97 }
98
99
100
101
102
103
104
105 protected void appendChildren(final HtmlSerializerTextBuilder builder, final DomNode node) {
106 for (final DomNode child : node.getChildren()) {
107 appendNode(builder, child);
108 }
109 }
110
111
112
113
114
115
116
117
118 protected void appendNode(final HtmlSerializerTextBuilder builder, final DomNode node) {
119 if (node instanceof DomText text1) {
120 appendText(builder, text1);
121 }
122 else if (node instanceof DomComment) {
123
124 }
125 else if (node instanceof HtmlBreak break1) {
126 appendBreak(builder, break1);
127 }
128 else if (node instanceof HtmlHiddenInput) {
129
130 }
131 else if (node instanceof HtmlScript) {
132
133 }
134 else if (node instanceof HtmlStyle) {
135
136 }
137 else if (node instanceof HtmlNoFrames) {
138
139 }
140 else if (node instanceof HtmlTextArea area) {
141 appendTextArea(builder, area);
142 }
143 else if (node instanceof HtmlTitle title) {
144 appendTitle(builder, title);
145 }
146 else if (node instanceof HtmlTableRow row) {
147 appendTableRow(builder, row);
148 }
149 else if (node instanceof HtmlSelect select) {
150 appendSelect(builder, select);
151 }
152 else if (node instanceof HtmlSubmitInput input5) {
153 appendSubmitInput(builder, input5);
154 }
155 else if (node instanceof HtmlResetInput input4) {
156 appendResetInput(builder, input4);
157 }
158 else if (node instanceof HtmlCheckBoxInput input3) {
159 appendCheckBoxInput(builder, input3);
160 }
161 else if (node instanceof HtmlRadioButtonInput input2) {
162 appendRadioButtonInput(builder, input2);
163 }
164 else if (node instanceof HtmlNumberInput input1) {
165 appendNumberInput(builder, input1);
166 }
167 else if (node instanceof HtmlInput input) {
168 appendInput(builder, input);
169 }
170 else if (node instanceof HtmlTable table) {
171 appendTable(builder, table);
172 }
173 else if (node instanceof HtmlOrderedList list1) {
174 appendOrderedList(builder, list1);
175 }
176 else if (node instanceof HtmlUnorderedList list) {
177 appendUnorderedList(builder, list);
178 }
179 else if (node instanceof HtmlPreformattedText text) {
180 appendPreformattedText(builder, text);
181 }
182 else if (node instanceof HtmlInlineFrame frame) {
183 appendInlineFrame(builder, frame);
184 }
185 else if (node instanceof HtmlDetails details) {
186 appendDetails(builder, details);
187 }
188 else if (node instanceof HtmlNoScript && node.getPage().getWebClient().isJavaScriptEnabled()) {
189
190 }
191 else {
192 appendDomNode(builder, node);
193 }
194 }
195
196
197
198
199
200
201
202 protected void appendDomNode(final HtmlSerializerTextBuilder builder, final DomNode domNode) {
203 boolean block = false;
204 if (!(domNode instanceof HtmlBody)) {
205 final SgmlPage page = domNode.getPage();
206 final WebWindow window = page.getEnclosingWindow();
207 if (window.getWebClient().getOptions().isCssEnabled()) {
208 if (domNode instanceof DomElement element) {
209 final String display = window.getComputedStyle(element, null).getDisplay();
210 block = BLOCK.equals(display);
211 }
212 }
213 else if (domNode instanceof HtmlElement element) {
214 block = DisplayStyle.BLOCK == element.getDefaultStyleDisplay();
215 }
216 }
217
218 if (block) {
219 builder.appendBlockSeparator();
220 }
221 appendChildren(builder, domNode);
222 if (block) {
223 builder.appendBlockSeparator();
224 }
225 }
226
227
228
229
230
231
232
233 protected void appendSubmitInput(final HtmlSerializerTextBuilder builder, final HtmlSubmitInput htmlSubmitInput) {
234 String text = htmlSubmitInput.getValueAttribute();
235 if (ATTRIBUTE_NOT_DEFINED == text) {
236 text = HtmlSubmitInput.DEFAULT_VALUE;
237 }
238
239 builder.append(text, Mode.NORMALIZE);
240 }
241
242
243
244
245
246
247
248 protected void appendInput(final HtmlSerializerTextBuilder builder, final HtmlInput htmlInput) {
249 builder.append(" ", Mode.NORMALIZE);
250 builder.append(htmlInput.getRawValue(), Mode.NORMALIZE);
251 builder.append(" ", Mode.NORMALIZE);
252 }
253
254
255
256
257
258
259
260 protected void appendNumberInput(final HtmlSerializerTextBuilder builder, final HtmlNumberInput htmlNumberInput) {
261 builder.append(" ", Mode.NORMALIZE);
262
263 String val = htmlNumberInput.getRawValue();
264 final int lastPos = val.length() - 1;
265 if (lastPos >= 0 && val.charAt(lastPos) == '.') {
266 val = val.substring(0, lastPos);
267 }
268 builder.append(val, Mode.NORMALIZE);
269
270 builder.append(" ", Mode.NORMALIZE);
271 }
272
273
274
275
276
277
278
279 protected void appendResetInput(final HtmlSerializerTextBuilder builder, final HtmlResetInput htmlResetInput) {
280 String text = htmlResetInput.getValueAttribute();
281 if (ATTRIBUTE_NOT_DEFINED == text) {
282 text = HtmlResetInput.DEFAULT_VALUE;
283 }
284
285 builder.append(text, Mode.NORMALIZE);
286 }
287
288
289
290
291
292
293
294 protected void appendUnorderedList(final HtmlSerializerTextBuilder builder,
295 final HtmlUnorderedList htmlUnorderedList) {
296 builder.appendBlockSeparator();
297 boolean first = true;
298 for (final DomNode item : htmlUnorderedList.getChildren()) {
299 if (!first) {
300 builder.appendBlockSeparator();
301 }
302 first = false;
303 appendNode(builder, item);
304 }
305 builder.appendBlockSeparator();
306 }
307
308
309
310
311
312
313
314 protected void appendDetails(final HtmlSerializerTextBuilder builder,
315 final HtmlDetails htmlDetails) {
316 if (htmlDetails.isOpen()) {
317 appendChildren(builder, htmlDetails);
318 return;
319 }
320
321 for (final DomNode child : htmlDetails.getChildren()) {
322 if (child instanceof HtmlSummary) {
323
324
325 appendNode(builder, child);
326 }
327 }
328 }
329
330
331
332
333
334
335
336 protected void appendTitle(final HtmlSerializerTextBuilder builder, final HtmlTitle htmlTitle) {
337
338
339
340
341 final DomNode child = htmlTitle.getFirstChild();
342 if (child instanceof DomText text) {
343 builder.append(text.getData(), Mode.NORMALIZE);
344 builder.appendBlockSeparator();
345 }
346 }
347
348
349
350
351
352
353
354 protected void appendTableRow(final HtmlSerializerTextBuilder builder, final HtmlTableRow htmlTableRow) {
355 boolean first = true;
356 for (final HtmlTableCell cell : htmlTableRow.getCells()) {
357 if (first) {
358 first = false;
359 }
360 else {
361 builder.appendTab();
362 }
363 appendChildren(builder, cell);
364 }
365 }
366
367
368
369
370
371
372
373 protected void appendTextArea(final HtmlSerializerTextBuilder builder, final HtmlTextArea htmlTextArea) {
374 if (isVisible(htmlTextArea)) {
375 builder.append(htmlTextArea.getText(), Mode.PRESERVE_BLANK_NEWLINE);
376 }
377 }
378
379
380
381
382
383
384
385 protected void appendTable(final HtmlSerializerTextBuilder builder, final HtmlTable htmlTable) {
386 builder.appendBlockSeparator();
387 final String caption = htmlTable.getCaptionText();
388 if (caption != null) {
389 builder.append(caption, Mode.NORMALIZE);
390 builder.appendBlockSeparator();
391 }
392
393 boolean first = true;
394
395
396 final HtmlTableHeader tableHeader = htmlTable.getHeader();
397 if (tableHeader != null) {
398 first = appendTableRows(builder, tableHeader.getRows(), true, null, null);
399 }
400 final HtmlTableFooter tableFooter = htmlTable.getFooter();
401
402 final List<HtmlTableRow> tableRows = htmlTable.getRows();
403 first = appendTableRows(builder, tableRows, first, tableHeader, tableFooter);
404
405 if (tableFooter != null) {
406 first = appendTableRows(builder, tableFooter.getRows(), first, null, null);
407 }
408 else if (tableRows.isEmpty()) {
409 final DomNode firstChild = htmlTable.getFirstChild();
410 if (firstChild != null) {
411 appendNode(builder, firstChild);
412 }
413 }
414
415 builder.appendBlockSeparator();
416 }
417
418
419
420
421
422
423
424
425
426
427
428 protected boolean appendTableRows(final HtmlSerializerTextBuilder builder,
429 final List<HtmlTableRow> rows, boolean first, final TableRowGroup skipParent1,
430 final TableRowGroup skipParent2) {
431 for (final HtmlTableRow row : rows) {
432 if (row.getParentNode() == skipParent1 || row.getParentNode() == skipParent2) {
433 continue;
434 }
435 if (!first) {
436 builder.appendBlockSeparator();
437 }
438 first = false;
439 appendTableRow(builder, row);
440 }
441 return first;
442 }
443
444
445
446
447
448
449
450 protected void appendSelect(final HtmlSerializerTextBuilder builder, final HtmlSelect htmlSelect) {
451 final List<HtmlOption> options = htmlSelect.getSelectedOptions();
452
453 for (final Iterator<HtmlOption> i = options.iterator(); i.hasNext();) {
454 final HtmlOption currentOption = i.next();
455 appendChildren(builder, currentOption);
456 if (i.hasNext()) {
457 builder.appendBlockSeparator();
458 }
459 }
460 }
461
462
463
464
465
466
467
468 protected void appendOrderedList(final HtmlSerializerTextBuilder builder, final HtmlOrderedList htmlOrderedList) {
469 builder.appendBlockSeparator();
470 boolean first = true;
471 int i = 1;
472
473 final String start = htmlOrderedList.getStartAttribute();
474 if (ATTRIBUTE_NOT_DEFINED != start) {
475 try {
476 i = (int) Math.round(Double.parseDouble(start));
477 }
478 catch (final Exception e) {
479
480 }
481 }
482
483 for (final DomNode item : htmlOrderedList.getChildren()) {
484 if (!first) {
485 builder.appendBlockSeparator();
486 }
487 first = false;
488 if (item instanceof HtmlListItem) {
489 builder.append(Integer.toString(i++), Mode.NORMALIZE);
490 builder.append(". ", Mode.NORMALIZE);
491 appendChildren(builder, item);
492 }
493 else {
494 appendNode(builder, item);
495 }
496 }
497 builder.appendBlockSeparator();
498 }
499
500
501
502
503
504
505
506 protected void appendPreformattedText(final HtmlSerializerTextBuilder builder,
507 final HtmlPreformattedText htmlPreformattedText) {
508 if (isVisible(htmlPreformattedText)) {
509 builder.appendBlockSeparator();
510 builder.append(htmlPreformattedText.getTextContent(), Mode.PRESERVE_BLANK_TAB_NEWLINE);
511 builder.appendBlockSeparator();
512 }
513 }
514
515
516
517
518
519
520
521 protected void appendInlineFrame(final HtmlSerializerTextBuilder builder,
522 final HtmlInlineFrame htmlInlineFrame) {
523 if (isVisible(htmlInlineFrame)) {
524 builder.appendBlockSeparator();
525 final Page page = htmlInlineFrame.getEnclosedPage();
526 if (page instanceof SgmlPage sgmlPage) {
527 builder.append(sgmlPage.asNormalizedText(), Mode.NORMALIZE);
528 }
529 builder.appendBlockSeparator();
530 }
531 }
532
533
534
535
536
537
538
539 protected void appendText(final HtmlSerializerTextBuilder builder, final DomText domText) {
540 final DomNode parent = domText.getParentNode();
541 if (parent == null || parent instanceof HtmlTitle || isVisible(parent)) {
542 builder.append(domText.getData(), Mode.NORMALIZE);
543 }
544 }
545
546
547
548
549
550
551
552 protected void appendBreak(final HtmlSerializerTextBuilder builder, final HtmlBreak htmlBreak) {
553 builder.appendNewLine();
554 }
555
556
557
558
559
560
561
562 protected void appendCheckBoxInput(final HtmlSerializerTextBuilder builder,
563 final HtmlCheckBoxInput htmlCheckBoxInput) {
564 appendCheckedState(builder, htmlCheckBoxInput.isChecked());
565 }
566
567
568
569
570
571
572
573 protected void appendRadioButtonInput(final HtmlSerializerTextBuilder builder,
574 final HtmlRadioButtonInput htmlRadioButtonInput) {
575 appendCheckedState(builder, htmlRadioButtonInput.isChecked());
576 }
577
578
579
580
581
582
583
584 private static void appendCheckedState(final HtmlSerializerTextBuilder builder, final boolean checked) {
585 builder.append(checked ? "checked " : "unchecked ", Mode.NORMALIZE);
586 }
587
588
589
590
591
592
593
594
595
596
597 private boolean isVisible(final DomNode node) {
598 if (!ignoreMaskedElements_) {
599 return true;
600 }
601 return visibilityCache_.computeIfAbsent(node, DomNode::isDisplayed);
602 }
603
604
605
606
607
608
609 public void setIgnoreMaskedElements(final boolean ignore) {
610 ignoreMaskedElements_ = ignore;
611 }
612
613
614
615
616 protected static class HtmlSerializerTextBuilder {
617
618
619 protected enum Mode {
620
621 NORMALIZE,
622
623
624 PRESERVE_BLANK_TAB_NEWLINE,
625
626
627 PRESERVE_BLANK_NEWLINE
628 }
629
630 private enum State {
631 DEFAULT,
632 EMPTY,
633 TRIM,
634 BLANK_AT_END,
635 BLANK_AT_END_AFTER_NEWLINE,
636 NEWLINE_AT_END,
637 BLOCK_SEPARATOR_AT_END
638 }
639
640 private static final String LINE_SEPARATOR = "\n";
641 private static final int LINE_SEPARATOR_LENGTH = LINE_SEPARATOR.length();
642
643 private State state_;
644 private final StringBuilder builder_;
645 private int trimRightPos_;
646
647
648
649
650 public HtmlSerializerTextBuilder() {
651 builder_ = new StringBuilder();
652 state_ = State.EMPTY;
653 trimRightPos_ = builder_.length();
654 }
655
656
657
658
659
660
661
662 public void append(final String content, final Mode mode) {
663 if (content == null) {
664 return;
665 }
666 final int length = content.length();
667 if (length == 0) {
668 return;
669 }
670
671
672
673
674 boolean crFound = false;
675 for (int i = 0; i < length; i++) {
676 final char c = content.charAt(i);
677
678 if (mode == Mode.NORMALIZE) {
679 if (isSpace(c)) {
680 switch (state_) {
681 case EMPTY:
682 case TRIM:
683 case BLANK_AT_END:
684 case BLANK_AT_END_AFTER_NEWLINE:
685 case BLOCK_SEPARATOR_AT_END:
686 break;
687 case NEWLINE_AT_END:
688 builder_.append(' ');
689 state_ = State.BLANK_AT_END_AFTER_NEWLINE;
690 break;
691 default:
692 builder_.append(' ');
693 state_ = State.BLANK_AT_END;
694 break;
695 }
696 }
697 else if (c == (char) 160) {
698 builder_.append(' ');
699 state_ = State.DEFAULT;
700 trimRightPos_ = builder_.length();
701 }
702 else {
703 builder_.append(c);
704 state_ = State.DEFAULT;
705 trimRightPos_ = builder_.length();
706 }
707 continue;
708 }
709
710
711 if (c == '\n') {
712 appendNewLine();
713 crFound = false;
714 }
715 else {
716 if (crFound) {
717 appendNewLine();
718 }
719 crFound = c == '\r';
720
721 if (c == '\t') {
722 if (mode == Mode.PRESERVE_BLANK_TAB_NEWLINE) {
723 appendTab();
724 }
725 else if (state_ != State.BLOCK_SEPARATOR_AT_END) {
726 builder_.append(' ');
727 }
728 }
729 else if (c == (char) 160) {
730 appendBlank();
731 }
732 else if (c == ' ') {
733 appendBlank();
734 }
735 else {
736 builder_.append(c);
737 }
738 trimRightPos_ = builder_.length();
739 }
740 }
741
742 if (crFound) {
743 appendNewLine();
744 }
745
746 if (mode != Mode.NORMALIZE) {
747
748 state_ = State.TRIM;
749 }
750 }
751
752
753
754
755 public void appendBlockSeparator() {
756 switch (state_) {
757 case EMPTY:
758 break;
759 case BLANK_AT_END:
760 builder_.setLength(trimRightPos_);
761 if (builder_.length() == 0) {
762 state_ = State.EMPTY;
763 }
764 else {
765 builder_.append(LINE_SEPARATOR);
766 state_ = State.BLOCK_SEPARATOR_AT_END;
767 }
768 break;
769 case BLANK_AT_END_AFTER_NEWLINE:
770 builder_.setLength(trimRightPos_ - LINE_SEPARATOR_LENGTH);
771 trimRightPos_ = trimRightPos_ - LINE_SEPARATOR_LENGTH;
772 if (builder_.length() == 0) {
773 state_ = State.EMPTY;
774 }
775 else {
776 builder_.append(LINE_SEPARATOR);
777 state_ = State.BLOCK_SEPARATOR_AT_END;
778 }
779 break;
780 case BLOCK_SEPARATOR_AT_END:
781 break;
782 case NEWLINE_AT_END:
783 builder_.setLength(builder_.length() - LINE_SEPARATOR_LENGTH);
784 trimRightPos_ = trimRightPos_ - LINE_SEPARATOR_LENGTH;
785 if (builder_.length() == 0) {
786 state_ = State.EMPTY;
787 }
788 else {
789 builder_.append(LINE_SEPARATOR);
790 state_ = State.BLOCK_SEPARATOR_AT_END;
791 }
792 break;
793 default:
794 builder_.append(LINE_SEPARATOR);
795 state_ = State.BLOCK_SEPARATOR_AT_END;
796 break;
797 }
798 }
799
800
801
802
803 public void appendNewLine() {
804 builder_.append(LINE_SEPARATOR);
805 state_ = State.NEWLINE_AT_END;
806 trimRightPos_ = builder_.length();
807 }
808
809
810
811
812 public void appendTab() {
813 builder_.append('\t');
814 trimRightPos_ = builder_.length();
815 }
816
817
818
819
820 private void appendBlank() {
821 builder_.append(' ');
822 trimRightPos_ = builder_.length();
823 }
824
825
826
827
828
829
830 public String getText() {
831 return builder_.substring(0, trimRightPos_);
832 }
833
834 private static boolean isSpace(final char ch) {
835 return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\f' || ch == '\r';
836 }
837 }
838 }