chunks = trimmedLines.stream()
.flatMap(line -> Arrays.stream(line.split(" "))
.map(String::trim))
.collect(toList());
// Filter empty chunks and join with newlines
return chunks.stream()
.filter(chunk -> !chunk.isEmpty())
.collect(joining("\n"));
}
/**
* Fetches HTML content from a URL
*/
private static String fetchHtml(String urlString) throws IOException {
try (InputStream inputStream = new URL(urlString).openStream()) {
StringBuilder content = new StringBuilder();
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(inputStream))) {
String line;
while ((line = reader.readLine()) != null) {
content.append(line).append("\n");
}
}
return content.toString();
}
}
/**
* Removes specified HTML elements and their content
*/
private static String removeElements(String html, String... elementNames) {
String result = html;
for (String element : elementNames) {
// Pattern to match ... and self-closing tags
String pattern = "<" + element + "\\s*[^>]*>.*?|<" + element + "\\s*[^>]*/?>";
result = Pattern.compile(pattern, Pattern.DOTALL).matcher(result).replaceAll("");
}
return result;
}
/**
* Removes all HTML tags from content
*/
private static String removeHtmlTags(String html) {
// Replace
and tags with newlines for better text formatting
String withLineBreaks = html.replaceAll("|]*>", "\n");
// Remove remaining HTML tags
String noTags = withLineBreaks.replaceAll("<[^>]*>", "");
// Decode HTML entities (simplified for common entities)
return decodeHtmlEntities(noTags);
}
/**
* Simple HTML entity decoder for common entities
*/
private static String decodeHtmlEntities(String text) {
return text
.replaceAll(" ", " ")
.replaceAll("&", "&")
.replaceAll("<", "<")
.replaceAll(">", ">")
.replaceAll(""", "\"")
.replaceAll("'", "'")
.replaceAll("…", "...")
.replaceAll("—", "—");
}
}
```