1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package org.htmlunit.html.serializer;
16
17 import static org.htmlunit.css.CssStyleSheet.BLOCK;
18 import static org.htmlunit.html.DomElement.ATTRIBUTE_NOT_DEFINED;
19
20 import java.util.Iterator;
21 import java.util.List;
22
23 import org.htmlunit.Page;
24 import org.htmlunit.SgmlPage;
25 import org.htmlunit.WebWindow;
26 import org.htmlunit.html.DomComment;
27 import org.htmlunit.html.DomElement;
28 import org.htmlunit.html.DomNode;
29 import org.htmlunit.html.DomText;
30 import org.htmlunit.html.HtmlBody;
31 import org.htmlunit.html.HtmlBreak;
32 import org.htmlunit.html.HtmlCheckBoxInput;
33 import org.htmlunit.html.HtmlDetails;
34 import org.htmlunit.html.HtmlElement;
35 import org.htmlunit.html.HtmlElement.DisplayStyle;
36 import org.htmlunit.html.HtmlHiddenInput;
37 import org.htmlunit.html.HtmlInlineFrame;
38 import org.htmlunit.html.HtmlInput;
39 import org.htmlunit.html.HtmlListItem;
40 import org.htmlunit.html.HtmlNoFrames;
41 import org.htmlunit.html.HtmlNoScript;
42 import org.htmlunit.html.HtmlNumberInput;
43 import org.htmlunit.html.HtmlOption;
44 import org.htmlunit.html.HtmlOrderedList;
45 import org.htmlunit.html.HtmlPreformattedText;
46 import org.htmlunit.html.HtmlRadioButtonInput;
47 import org.htmlunit.html.HtmlResetInput;
48 import org.htmlunit.html.HtmlScript;
49 import org.htmlunit.html.HtmlSelect;
50 import org.htmlunit.html.HtmlStyle;
51 import org.htmlunit.html.HtmlSubmitInput;
52 import org.htmlunit.html.HtmlSummary;
53 import org.htmlunit.html.HtmlTable;
54 import org.htmlunit.html.HtmlTableCell;
55 import org.htmlunit.html.HtmlTableFooter;
56 import org.htmlunit.html.HtmlTableHeader;
57 import org.htmlunit.html.HtmlTableRow;
58 import org.htmlunit.html.HtmlTextArea;
59 import org.htmlunit.html.HtmlTitle;
60 import org.htmlunit.html.HtmlUnorderedList;
61 import org.htmlunit.html.TableRowGroup;
62 import org.htmlunit.html.serializer.HtmlSerializerNormalizedText.HtmlSerializerTextBuilder.Mode;
63
64
65
66
67
68
69
70
71
72
73
74
75 public class HtmlSerializerNormalizedText {
76
77 private boolean ignoreMaskedElements_ = true;
78
79
80
81
82
83
84 public String asText(final DomNode node) {
85 final HtmlSerializerTextBuilder builder = new HtmlSerializerTextBuilder();
86 appendNode(builder, node);
87 return builder.getText();
88 }
89
90
91
92
93
94
95
96 protected void appendChildren(final HtmlSerializerTextBuilder builder, final DomNode node) {
97 for (final DomNode child : node.getChildren()) {
98 appendNode(builder, child);
99 }
100 }
101
102
103
104
105
106
107
108
109 protected void appendNode(final HtmlSerializerTextBuilder builder, final DomNode node) {
110 if (node instanceof DomText text1) {
111 appendText(builder, text1);
112 }
113 else if (node instanceof DomComment) {
114
115 }
116 else if (node instanceof HtmlBreak break1) {
117 appendBreak(builder, break1);
118 }
119 else if (node instanceof HtmlHiddenInput) {
120
121 }
122 else if (node instanceof HtmlScript) {
123
124 }
125 else if (node instanceof HtmlStyle) {
126
127 }
128 else if (node instanceof HtmlNoFrames) {
129
130 }
131 else if (node instanceof HtmlTextArea area) {
132 appendTextArea(builder, area);
133 }
134 else if (node instanceof HtmlTitle title) {
135 appendTitle(builder, title);
136 }
137 else if (node instanceof HtmlTableRow row) {
138 appendTableRow(builder, row);
139 }
140 else if (node instanceof HtmlSelect select) {
141 appendSelect(builder, select);
142 }
143 else if (node instanceof HtmlSubmitInput input5) {
144 appendSubmitInput(builder, input5);
145 }
146 else if (node instanceof HtmlResetInput input4) {
147 appendResetInput(builder, input4);
148 }
149 else if (node instanceof HtmlCheckBoxInput input3) {
150 appendCheckBoxInput(builder, input3);
151 }
152 else if (node instanceof HtmlRadioButtonInput input2) {
153 appendRadioButtonInput(builder, input2);
154 }
155 else if (node instanceof HtmlNumberInput input1) {
156 appendNumberInput(builder, input1);
157 }
158 else if (node instanceof HtmlInput input) {
159 appendInput(builder, input);
160 }
161 else if (node instanceof HtmlTable table) {
162 appendTable(builder, table);
163 }
164 else if (node instanceof HtmlOrderedList list1) {
165 appendOrderedList(builder, list1);
166 }
167 else if (node instanceof HtmlUnorderedList list) {
168 appendUnorderedList(builder, list);
169 }
170 else if (node instanceof HtmlPreformattedText text) {
171 appendPreformattedText(builder, text);
172 }
173 else if (node instanceof HtmlInlineFrame frame) {
174 appendInlineFrame(builder, frame);
175 }
176 else if (node instanceof HtmlDetails details) {
177 appendDetails(builder, details);
178 }
179 else if (node instanceof HtmlNoScript && node.getPage().getWebClient().isJavaScriptEnabled()) {
180
181 }
182 else {
183 appendDomNode(builder, node);
184 }
185 }
186
187
188
189
190
191
192
193 protected void appendDomNode(final HtmlSerializerTextBuilder builder, final DomNode domNode) {
194 boolean block = false;
195 if (!(domNode instanceof HtmlBody)) {
196 final SgmlPage page = domNode.getPage();
197 final WebWindow window = page.getEnclosingWindow();
198 if (window.getWebClient().getOptions().isCssEnabled()) {
199 if (domNode instanceof DomElement element) {
200 final String display = window.getComputedStyle(element, null).getDisplay();
201 block = BLOCK.equals(display);
202 }
203 }
204 else if (domNode instanceof HtmlElement element) {
205 block = DisplayStyle.BLOCK == element.getDefaultStyleDisplay();
206 }
207 }
208
209 if (block) {
210 builder.appendBlockSeparator();
211 }
212 appendChildren(builder, domNode);
213 if (block) {
214 builder.appendBlockSeparator();
215 }
216 }
217
218
219
220
221
222
223
224 protected void appendSubmitInput(final HtmlSerializerTextBuilder builder, final HtmlSubmitInput htmlSubmitInput) {
225 String text = htmlSubmitInput.getValueAttribute();
226 if (ATTRIBUTE_NOT_DEFINED == text) {
227 text = HtmlSubmitInput.DEFAULT_VALUE;
228 }
229
230 builder.append(text, Mode.NORMALIZE);
231 }
232
233
234
235
236
237
238
239 protected void appendInput(final HtmlSerializerTextBuilder builder, final HtmlInput htmlInput) {
240 builder.append(" ", Mode.NORMALIZE);
241 builder.append(htmlInput.getRawValue(), Mode.NORMALIZE);
242 builder.append(" ", Mode.NORMALIZE);
243 }
244
245
246
247
248
249
250
251 protected void appendNumberInput(final HtmlSerializerTextBuilder builder, final HtmlNumberInput htmlNumberInput) {
252 builder.append(" ", Mode.NORMALIZE);
253
254 String val = htmlNumberInput.getRawValue();
255 final int lastPos = val.length() - 1;
256 if (lastPos >= 0 && val.charAt(lastPos) == '.') {
257 val = val.substring(0, lastPos);
258 }
259 builder.append(val, Mode.NORMALIZE);
260
261 builder.append(" ", Mode.NORMALIZE);
262 }
263
264
265
266
267
268
269
270 protected void appendResetInput(final HtmlSerializerTextBuilder builder, final HtmlResetInput htmlResetInput) {
271 String text = htmlResetInput.getValueAttribute();
272 if (ATTRIBUTE_NOT_DEFINED == text) {
273 text = HtmlResetInput.DEFAULT_VALUE;
274 }
275
276 builder.append(text, Mode.NORMALIZE);
277 }
278
279
280
281
282
283
284 protected void appendUnorderedList(final HtmlSerializerTextBuilder builder,
285 final HtmlUnorderedList htmlUnorderedList) {
286 builder.appendBlockSeparator();
287 boolean first = true;
288 for (final DomNode item : htmlUnorderedList.getChildren()) {
289 if (!first) {
290 builder.appendBlockSeparator();
291 }
292 first = false;
293 appendNode(builder, item);
294 }
295 builder.appendBlockSeparator();
296 }
297
298
299
300
301
302
303 protected void appendDetails(final HtmlSerializerTextBuilder builder,
304 final HtmlDetails htmlDetails) {
305 if (htmlDetails.isOpen()) {
306 appendChildren(builder, htmlDetails);
307 return;
308 }
309
310 for (final DomNode child : htmlDetails.getChildren()) {
311 if (child instanceof HtmlSummary) {
312 appendNode(builder, child);
313 }
314 }
315 }
316
317
318
319
320
321
322 protected void appendTitle(final HtmlSerializerTextBuilder builder, final HtmlTitle htmlTitle) {
323
324
325
326
327 final DomNode child = htmlTitle.getFirstChild();
328 if (child instanceof DomText text) {
329 builder.append(text.getData(), Mode.NORMALIZE);
330 builder.appendBlockSeparator();
331 }
332 }
333
334
335
336
337
338
339
340 protected void appendTableRow(final HtmlSerializerTextBuilder builder, final HtmlTableRow htmlTableRow) {
341 boolean first = true;
342 for (final HtmlTableCell cell : htmlTableRow.getCells()) {
343 if (first) {
344 first = false;
345 }
346 else {
347 builder.appendTab();
348 }
349 appendChildren(builder, cell);
350 }
351 }
352
353
354
355
356
357
358
359 protected void appendTextArea(final HtmlSerializerTextBuilder builder, final HtmlTextArea htmlTextArea) {
360 if (isVisible(htmlTextArea)) {
361 builder.append(htmlTextArea.getText(), Mode.PRESERVE_BLANK_NEWLINE);
362 }
363 }
364
365
366
367
368
369
370
371 protected void appendTable(final HtmlSerializerTextBuilder builder, final HtmlTable htmlTable) {
372 builder.appendBlockSeparator();
373 final String caption = htmlTable.getCaptionText();
374 if (caption != null) {
375 builder.append(caption, Mode.NORMALIZE);
376 builder.appendBlockSeparator();
377 }
378
379 boolean first = true;
380
381
382 final HtmlTableHeader tableHeader = htmlTable.getHeader();
383 if (tableHeader != null) {
384 first = appendTableRows(builder, tableHeader.getRows(), true, null, null);
385 }
386 final HtmlTableFooter tableFooter = htmlTable.getFooter();
387
388 final List<HtmlTableRow> tableRows = htmlTable.getRows();
389 first = appendTableRows(builder, tableRows, first, tableHeader, tableFooter);
390
391 if (tableFooter != null) {
392 first = appendTableRows(builder, tableFooter.getRows(), first, null, null);
393 }
394 else if (tableRows.isEmpty()) {
395 final DomNode firstChild = htmlTable.getFirstChild();
396 if (firstChild != null) {
397 appendNode(builder, firstChild);
398 }
399 }
400
401 builder.appendBlockSeparator();
402 }
403
404
405
406
407
408
409
410
411
412
413
414 protected boolean appendTableRows(final HtmlSerializerTextBuilder builder,
415 final List<HtmlTableRow> rows, boolean first, final TableRowGroup skipParent1,
416 final TableRowGroup skipParent2) {
417 for (final HtmlTableRow row : rows) {
418 if (row.getParentNode() == skipParent1 || row.getParentNode() == skipParent2) {
419 continue;
420 }
421 if (!first) {
422 builder.appendBlockSeparator();
423 }
424 first = false;
425 appendTableRow(builder, row);
426 }
427 return first;
428 }
429
430
431
432
433
434
435
436 protected void appendSelect(final HtmlSerializerTextBuilder builder, final HtmlSelect htmlSelect) {
437 final List<HtmlOption> options = htmlSelect.getSelectedOptions();
438
439 for (final Iterator<HtmlOption> i = options.iterator(); i.hasNext();) {
440 final HtmlOption currentOption = i.next();
441 appendChildren(builder, currentOption);
442 if (i.hasNext()) {
443 builder.appendBlockSeparator();
444 }
445 }
446 }
447
448
449
450
451
452
453
454 protected void appendOrderedList(final HtmlSerializerTextBuilder builder, final HtmlOrderedList htmlOrderedList) {
455 builder.appendBlockSeparator();
456 boolean first = true;
457 int i = 1;
458
459 final String start = htmlOrderedList.getStartAttribute();
460 if (ATTRIBUTE_NOT_DEFINED != start) {
461 try {
462 i = (int) Math.round(Double.parseDouble(start));
463 }
464 catch (final Exception e) {
465
466 }
467 }
468
469 for (final DomNode item : htmlOrderedList.getChildren()) {
470 if (!first) {
471 builder.appendBlockSeparator();
472 }
473 first = false;
474 if (item instanceof HtmlListItem) {
475 builder.append(Integer.toString(i++), Mode.NORMALIZE);
476 builder.append(". ", Mode.NORMALIZE);
477 appendChildren(builder, item);
478 }
479 else {
480 appendNode(builder, item);
481 }
482 }
483 builder.appendBlockSeparator();
484 }
485
486
487
488
489
490
491
492 protected void appendPreformattedText(final HtmlSerializerTextBuilder builder,
493 final HtmlPreformattedText htmlPreformattedText) {
494 if (isVisible(htmlPreformattedText)) {
495 builder.appendBlockSeparator();
496 builder.append(htmlPreformattedText.getTextContent(), Mode.PRESERVE_BLANK_TAB_NEWLINE);
497 builder.appendBlockSeparator();
498 }
499 }
500
501
502
503
504
505
506
507 protected void appendInlineFrame(final HtmlSerializerTextBuilder builder,
508 final HtmlInlineFrame htmlInlineFrame) {
509 if (isVisible(htmlInlineFrame)) {
510 builder.appendBlockSeparator();
511 final Page page = htmlInlineFrame.getEnclosedPage();
512 if (page instanceof SgmlPage sgmlPage) {
513 builder.append(sgmlPage.asNormalizedText(), Mode.NORMALIZE);
514 }
515 builder.appendBlockSeparator();
516 }
517 }
518
519
520
521
522
523
524
525 protected void appendText(final HtmlSerializerTextBuilder builder, final DomText domText) {
526 final DomNode parent = domText.getParentNode();
527 if (parent == null || parent instanceof HtmlTitle || isVisible(parent)) {
528 builder.append(domText.getData(), Mode.NORMALIZE);
529 }
530 }
531
532
533
534
535
536
537
538 protected void appendBreak(final HtmlSerializerTextBuilder builder, final HtmlBreak htmlBreak) {
539 builder.appendNewLine();
540 }
541
542
543
544
545
546
547
548 protected void appendCheckBoxInput(final HtmlSerializerTextBuilder builder,
549 final HtmlCheckBoxInput htmlCheckBoxInput) {
550 if (htmlCheckBoxInput.isChecked()) {
551 builder.append("checked ", Mode.NORMALIZE);
552 }
553 else {
554 builder.append("unchecked ", Mode.NORMALIZE);
555 }
556 }
557
558
559
560
561
562
563
564 protected void appendRadioButtonInput(final HtmlSerializerTextBuilder builder,
565 final HtmlRadioButtonInput htmlRadioButtonInput) {
566 if (htmlRadioButtonInput.isChecked()) {
567 builder.append("checked ", Mode.NORMALIZE);
568 }
569 else {
570 builder.append("unchecked ", Mode.NORMALIZE);
571 }
572 }
573
574 private boolean isVisible(final DomNode node) {
575 return !ignoreMaskedElements_ || node.isDisplayed();
576 }
577
578
579
580
581
582
583 public void setIgnoreMaskedElements(final boolean ignore) {
584 ignoreMaskedElements_ = ignore;
585 }
586
587
588
589
590 protected static class HtmlSerializerTextBuilder {
591
592
593 protected enum Mode {
594
595 NORMALIZE,
596
597
598 PRESERVE_BLANK_TAB_NEWLINE,
599
600
601 PRESERVE_BLANK_NEWLINE
602 }
603
604 private enum State {
605 DEFAULT,
606 EMPTY,
607 TRIM,
608 BLANK_AT_END,
609 BLANK_AT_END_AFTER_NEWLINE,
610 NEWLINE_AT_END,
611 BLOCK_SEPARATOR_AT_END
612 }
613
614 private static final String LINE_SEPARATOR = "\n";
615 private static final int LINE_SEPARATOR_LENGTH = LINE_SEPARATOR.length();
616
617 private State state_;
618 private final StringBuilder builder_;
619 private int trimRightPos_;
620
621
622
623
624 public HtmlSerializerTextBuilder() {
625 builder_ = new StringBuilder();
626 state_ = State.EMPTY;
627 trimRightPos_ = builder_.length();
628 }
629
630
631
632
633
634
635
636 public void append(final String content, final Mode mode) {
637 if (content == null) {
638 return;
639 }
640 final int length = content.length();
641 if (length == 0) {
642 return;
643 }
644
645
646
647
648 boolean crFound = false;
649 final int contentLength = content.length();
650 for (int i = 0; i < contentLength; i++) {
651 final char c = content.charAt(i);
652
653 if (mode == Mode.NORMALIZE) {
654 if (isSpace(c)) {
655 switch (state_) {
656 case EMPTY:
657 case TRIM:
658 case BLANK_AT_END:
659 case BLANK_AT_END_AFTER_NEWLINE:
660 case BLOCK_SEPARATOR_AT_END:
661 break;
662 case NEWLINE_AT_END:
663 builder_.append(' ');
664 state_ = State.BLANK_AT_END_AFTER_NEWLINE;
665 break;
666 default:
667 builder_.append(' ');
668 state_ = State.BLANK_AT_END;
669 break;
670 }
671 }
672 else if (c == (char) 160) {
673 builder_.append(' ');
674 state_ = State.DEFAULT;
675 trimRightPos_ = builder_.length();
676 }
677 else {
678 builder_.append(c);
679 state_ = State.DEFAULT;
680 trimRightPos_ = builder_.length();
681 }
682 continue;
683 }
684
685
686 if (c == '\n') {
687 appendNewLine();
688 crFound = false;
689 }
690 else {
691 if (crFound) {
692 appendNewLine();
693 }
694 crFound = c == '\r';
695
696 if (c == '\t') {
697 if (mode == Mode.PRESERVE_BLANK_TAB_NEWLINE) {
698 appendTab();
699 }
700 else if (state_ != State.BLOCK_SEPARATOR_AT_END) {
701 builder_.append(' ');
702 }
703 }
704 else if (c == (char) 160) {
705 appendBlank();
706 }
707 else if (c == ' ') {
708 appendBlank();
709 }
710 else {
711 builder_.append(c);
712 }
713 trimRightPos_ = builder_.length();
714 }
715 }
716
717 if (crFound) {
718 appendNewLine();
719 }
720
721 if (mode != Mode.NORMALIZE) {
722
723 state_ = State.TRIM;
724 }
725 }
726
727
728
729
730 public void appendBlockSeparator() {
731 switch (state_) {
732 case EMPTY:
733 break;
734 case BLANK_AT_END:
735 builder_.setLength(trimRightPos_);
736 if (builder_.length() == 0) {
737 state_ = State.EMPTY;
738 }
739 else {
740 builder_.append(LINE_SEPARATOR);
741 state_ = State.BLOCK_SEPARATOR_AT_END;
742 }
743 break;
744 case BLANK_AT_END_AFTER_NEWLINE:
745 builder_.setLength(trimRightPos_ - LINE_SEPARATOR_LENGTH);
746 trimRightPos_ = trimRightPos_ - LINE_SEPARATOR_LENGTH;
747 if (builder_.length() == 0) {
748 state_ = State.EMPTY;
749 }
750 else {
751 builder_.append(LINE_SEPARATOR);
752 state_ = State.BLOCK_SEPARATOR_AT_END;
753 }
754 break;
755 case BLOCK_SEPARATOR_AT_END:
756 break;
757 case NEWLINE_AT_END:
758 builder_.setLength(builder_.length() - LINE_SEPARATOR_LENGTH);
759 trimRightPos_ = trimRightPos_ - LINE_SEPARATOR_LENGTH;
760 if (builder_.length() == 0) {
761 state_ = State.EMPTY;
762 }
763 else {
764 builder_.append(LINE_SEPARATOR);
765 state_ = State.BLOCK_SEPARATOR_AT_END;
766 }
767 break;
768 default:
769 builder_.append(LINE_SEPARATOR);
770 state_ = State.BLOCK_SEPARATOR_AT_END;
771 break;
772 }
773 }
774
775
776
777
778 public void appendNewLine() {
779 builder_.append(LINE_SEPARATOR);
780 state_ = State.NEWLINE_AT_END;
781 trimRightPos_ = builder_.length();
782 }
783
784
785
786
787 public void appendTab() {
788 builder_.append('\t');
789 trimRightPos_ = builder_.length();
790 }
791
792
793
794
795 private void appendBlank() {
796 builder_.append(' ');
797 trimRightPos_ = builder_.length();
798 }
799
800
801
802
803
804
805 public String getText() {
806 return builder_.substring(0, trimRightPos_);
807 }
808
809 private static boolean isSpace(final char ch) {
810 return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\f' || ch == '\r';
811 }
812 }
813 }