improve literal text parsing - webdump - HTML to plain-text converter for webpages
HTML git clone git://git.codemadness.org/webdump
DIR Log
DIR Files
DIR Refs
DIR README
DIR LICENSE
---
DIR commit ea4304af9bf0a29e671632dad484376d7a709ac7
DIR parent 7491447d1bb2676212a7c4926023513ecb5e26e6
HTML Author: Hiltjo Posthuma <hiltjo@codemadness.org>
Date: Sun, 5 Jul 2026 21:22:14 +0200
improve literal text parsing
- Fix parsing repeated < characters in titles.
This last character < would be added to the pushback buffer but not handled
as a data entity.
Test-case added to webdump_tests:
<title><<<<< <u>Hello</u> >>>>> This is the end:</title>My friend</title>
- Make the code a bit more reusable and readable (ahem).
Don't hardcode the end tag character '>'.
Diffstat:
M webdump.c | 118 +++++++++++++++++--------------
1 file changed, 66 insertions(+), 52 deletions(-)
---
DIR diff --git a/webdump.c b/webdump.c
@@ -423,84 +423,103 @@ string_append(String *s, const char *data, size_t len)
}
static char *ignorestate;
-static char endtagmatch[16], endtagname[16];
+static char endtagmatch[16];
static unsigned char pushbuf[32], *pushoff, *pushend; /* pushback buffer */
-static unsigned char *pushtagend;
+static unsigned char *pushtagend; /* end position of tag in pushback buffer */
static int (*getnext)(void);
static void
-getnext_literal_init(void)
+getnext_literal_reset(void)
{
pushoff = pushbuf;
pushend = pushbuf;
pushtagend = NULL;
}
-/* Collect data as literal text, used for certain HTML tags such as <script> or <title>.
- To the parser return a space for all data (so it is parsed as text data)
- until some case-insensitive string occurs (closing tag).
+static void
+getnext_literal_end(void)
+{
+ parser.getnext = getnext; /* restore original getnext callback */
+}
+
+/* Collect data as literal text, used for certain HTML tags such as <script> or
+ <title> until some case-insensitive string occurs (closing tag).
This code is very ugly and I'm not proud of it. Mmmmm spaghetti. */
-static inline int
+static int
getnext_literal(void)
{
- int c, len;
+ char *p;
+ int c, fullmatch;
- /* there is data in pushback buffer */
+ /* there is data in the pushback buffer */
if (pushend != pushbuf) {
-pushback:
if (pushoff >= pushend) {
- /* reached the offset where the tag ended, fake the callback now */
+ /* reached the offset where the tag ended */
if (pushtagend) {
- parser.getnext = getnext; /* restore */
- getnext_literal_init(); /* reset */
- return '>'; /* end tag */
+ c = *pushtagend;
+ getnext_literal_end();
+ return c; /* end tag */
}
- getnext_literal_init(); /* reset */
+ getnext_literal_reset(); /* reset pushback buffer */
} else {
return (int)*(pushoff++);
}
}
- if ((c = getnext()) == EOF)
- return EOF;
+ for (; *ignorestate; ignorestate++) {
+ if ((c = getnext()) == EOF)
+ return EOF;
+ if (TOLOWER((unsigned char)c) != TOLOWER((unsigned char)*ignorestate))
+ break;
+ }
- if (TOLOWER((unsigned char)c) == TOLOWER((unsigned char)*ignorestate)) {
- ignorestate++;
- if (*ignorestate == '\0') {
- /* copy complete tag: buffer not checked, it always fits. */
- memcpy(pushbuf, endtagmatch, strlen(endtagmatch));
- pushoff = pushbuf;
- pushend = pushbuf + strlen(endtagmatch) - 1; /* except NUL byte */
- pushtagend = pushend; /* offset of where the tag ended */
- }
- goto pushback;
- } else {
- /* incomplete tag, fake incomplete leading "<" as data entity "<". */
- if (ignorestate != endtagmatch) {
- pushoff = pushbuf;
- memcpy(pushend, "<", 4); /* buffer not checked, it always fits. */
- pushend = pushbuf + 4;
-
- len = ignorestate - endtagmatch - 1;
- if (len > 0) {
- /* buffer not checked, it always fits. */
- memcpy(pushend, endtagmatch + 1, len); /* copy except leading "<" */
- pushend += len;
+ /* process a partial or full match on the tag:
+ partial: handle "<" as a data entity "<", full: handle as end tag */
+ if (ignorestate != endtagmatch) {
+ fullmatch = (*ignorestate == '\0');
+ pushoff = pushbuf;
+ pushend = pushbuf;
+ /* buffer not checked, it should alway fit. */
+ if (fullmatch) {
+ for (p = endtagmatch; *p && p < ignorestate; p++)
+ *(pushend++) = *p;
+ pushtagend = pushend; /* offset where the tag ended */
+ } else {
+ for (p = endtagmatch; *p && p < ignorestate; p++) {
+ if (!fullmatch && *p == '<') {
+ memcpy(pushend, "<", 4);
+ pushend += 4;
+ } else {
+ *(pushend++) = *p;
+ }
+ }
+ /* append current character to end of pushback buffer */
+ if (c == '<') {
+ memcpy(pushend, "<", 4);
+ pushend += 4;
+ } else {
+ *(pushend++) = c;
}
- ignorestate = endtagmatch; /* no full match: reset to beginning */
}
- }
+ ignorestate = endtagmatch; /* reset state for matching to beginning */
- if (pushend != pushbuf) {
- /* append current character to end of pushback buffer */
- *pushend = c;
- pushend++;
- goto pushback;
+ return (int)*(pushoff++);
}
return c;
}
+static void
+getnext_literal_start(const char *t)
+{
+ snprintf(endtagmatch, sizeof(endtagmatch), "</%s>", t);
+ ignorestate = endtagmatch;
+ getnext_literal_reset(); /* reset pushback buffer */
+
+ getnext = parser.getnext; /* for restore */
+ parser.getnext = getnext_literal;
+}
+
static char *
estrdup(const char *s)
{
@@ -2349,12 +2368,7 @@ xmltagstartparsed(XMLParser *p, const char *t, size_t tl, int isshort)
case TagTitle:
case TagTemplate:
case TagTextarea:
- snprintf(endtagmatch, sizeof(endtagmatch), "</%s>", t);
- snprintf(endtagname, sizeof(endtagname), "%s", t);
- ignorestate = endtagmatch;
- getnext = p->getnext; /* for restore */
- getnext_literal_init();
- p->getnext = getnext_literal;
+ getnext_literal_start(t);
break;
default:
break;