Lightweight HTML/XML extraction: links, forms, and sitemap URLs. More...

#include "nilorea/n_html.h"
#include "nilorea/n_common.h"
#include "nilorea/n_list.h"
#include "nilorea/n_log.h"
#include "nilorea/n_str.h"
#include <ctype.h>
#include <stdlib.h>
#include <string.h>

Include dependency graph for n_html.c:

Go to the source code of this file.

Data Structures
struct	N_HTML_ATTR_
	attribute parsed off an HTML start tag (name lowercased, value raw) More...

struct	N_HTML_TAG_
	a single tag picked out by the scanner More...

Functions
static void	_n_form_field_free (void *p)

static void	_n_html_attr_free (void *p)

static const char *	_n_html_attr_lookup (const N_HTML_TAG_ tag, const char name)

static int	_n_html_block_kind (const char *name)

static int	_n_html_ci_eq (const char a, const char b)

static const char *	_n_html_find (const char hay, size_t haylen, const char needle, size_t nlen)

static const char *	_n_html_find_ci (const char hay, size_t haylen, const char needle, size_t nlen)

static void	_n_html_flush (char buf, size_t cap, size_t o, int pending_nl, int pending_sp, int *line_content)

static void	_n_html_form_free (void *p)

static int	_n_html_is_name_char (int c)

static int	_n_html_js_url_like (const char *s, size_t n)

static char	_n_html_lc (char c)

static void	_n_html_parse_tag (const char html, size_t len, size_t pos, LIST *out)

static void	_n_html_push_js_url (LIST out, const char tok, size_t len)

static void	_n_html_push_link (LIST out, const char url)

static void	_n_html_put_cp (char buf, size_t cap, size_t o, unsigned long cp)

static void	_n_html_put_str (char buf, size_t cap, size_t o, const char *s)

static LIST *	_n_html_scan_tags (const char *html, size_t len)

static size_t	_n_html_skip_rawtext (const char html, size_t len, size_t start, const char close_tag)

static int	_n_html_starts_ci (const char s, size_t n, const char pre, size_t plen)

static char *	_n_html_strdup_upper (const char *s)

static char *	_n_html_strndup (const char *s, size_t n)

static char *	_n_html_strndup_lower (const char *s, size_t n)

static void	_n_html_tag_free (void *p)

LIST *	n_html_extract_forms (const char *html, size_t len)
	extract forms as a LIST of N_HTML_FORM*; free with n_html_forms_free

LIST *	n_html_extract_js_urls (const char *js, size_t len)
	extract URL/path tokens from JavaScript source (quoted string literals that look like an http(s) URL, a protocol-relative "//host" URL, or a "/" / "./" / "../" path; a "${...}" template is truncated to its static prefix), deduped, as a LIST of N_STR*; free with n_html_links_free.

LIST *	n_html_extract_links (const char *html, size_t len)
	extract link URLs (a/href, form/action, img/src, script/src, link/href, iframe/src) as a LIST of N_STR*; free with n_html_links_free

LIST *	n_html_extract_scripts (const char *html, size_t len)
	extract the inline <script> bodies (those without a src attribute) from an HTML document as a LIST of N_STR*; free with n_html_links_free.

void	n_html_forms_free (LIST **forms)
	free a list returned by n_html_extract_forms

void	n_html_links_free (LIST **links)
	free a list returned by n_html_extract_links or n_sitemap_extract_urls

N_STR *	n_html_to_text (const char *html, size_t len)
	render HTML to readable plain text: drop tags, skip script/style, decode common entities, collapse whitespace, and line-break block elements; returns a new N_STR (free with free_nstr) or NULL

LIST *	n_sitemap_extract_urls (const char *xml, size_t len)
	extract <loc> URLs from a sitemap.xml as a LIST of N_STR*; free with n_html_links_free

Variables
struct {

const char *	name

const char *	rep

}	_n_html_entities []

Detailed Description

Lightweight HTML/XML extraction: links, forms, and sitemap URLs.

Author: Castagnier Mickael

Version: 1.0

Definition in file n_html.c.

Data Structure Documentation

◆ N_HTML_ATTR_

struct N_HTML_ATTR_

attribute parsed off an HTML start tag (name lowercased, value raw)

Definition at line 39 of file n_html.c.

Collaboration diagram for N_HTML_ATTR_:

Data Fields
char *	name	lowercased attribute name
char *	value	raw value, "" when the attribute has no '='

◆ N_HTML_TAG_

struct N_HTML_TAG_

a single tag picked out by the scanner

Definition at line 45 of file n_html.c.

Collaboration diagram for N_HTML_TAG_:

Data Fields
LIST *	attrs	attributes (only populated for open tags)
int	closing	1 if the tag started with '/'
char *	name	lowercased tag name
int	self_closing	1 if the tag ended with '/>'

Function Documentation

◆ _n_form_field_free()

static void _n_form_field_free ( void * p )

static

Definition at line 71 of file n_html.c.

References Free, FreeNoLog, N_FORM_FIELD::name, N_FORM_FIELD::type, and N_FORM_FIELD::value.

Referenced by n_html_extract_forms().

Here is the caller graph for this function:

◆ _n_html_attr_free()

static void _n_html_attr_free ( void * p )

static

Definition at line 53 of file n_html.c.

References Free, FreeNoLog, N_HTML_ATTR_::name, and N_HTML_ATTR_::value.

Referenced by _n_html_parse_tag().

Here is the caller graph for this function:

◆ _n_html_attr_lookup()

static const char * _n_html_attr_lookup	(	const N_HTML_TAG_ *	tag,
		const char *	name
	)

static

Definition at line 149 of file n_html.c.

References _n_html_ci_eq(), N_HTML_TAG_::attrs, list_foreach, N_HTML_ATTR_::name, and N_HTML_ATTR_::value.

Referenced by n_html_extract_forms(), and n_html_extract_links().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _n_html_block_kind()

static int _n_html_block_kind ( const char * name )

static

Definition at line 597 of file n_html.c.

Referenced by n_html_to_text().

Here is the caller graph for this function:

◆ _n_html_ci_eq()

static int _n_html_ci_eq	(	const char *	a,
		const char *	b
	)

static

Definition at line 130 of file n_html.c.

Referenced by _n_html_attr_lookup().

Here is the caller graph for this function:

◆ _n_html_find()

static const char * _n_html_find	(	const char *	hay,
		size_t	haylen,
		const char *	needle,
		size_t	nlen
	)

static

Definition at line 820 of file n_html.c.

Referenced by _n_html_push_js_url().

Here is the caller graph for this function:

◆ _n_html_find_ci()

static const char * _n_html_find_ci	(	const char *	hay,
		size_t	haylen,
		const char *	needle,
		size_t	nlen
	)

static

Definition at line 831 of file n_html.c.

Referenced by n_html_extract_scripts().

Here is the caller graph for this function:

◆ _n_html_flush()

static void _n_html_flush	(	char *	buf,
		size_t	cap,
		size_t *	o,
		int *	pending_nl,
		int *	pending_sp,
		int *	line_content
	)

static

Definition at line 611 of file n_html.c.

Referenced by n_html_to_text().

Here is the caller graph for this function:

◆ _n_html_form_free()

static void _n_html_form_free ( void * p )

static

Definition at line 81 of file n_html.c.

References N_HTML_FORM::action, N_HTML_FORM::enctype, N_HTML_FORM::fields, Free, FreeNoLog, list_destroy(), and N_HTML_FORM::method.

Referenced by n_html_extract_forms().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _n_html_is_name_char()

static int _n_html_is_name_char ( int c )

static

Definition at line 144 of file n_html.c.

Referenced by _n_html_parse_tag().

Here is the caller graph for this function:

◆ _n_html_js_url_like()

static int _n_html_js_url_like	(	const char *	s,
		size_t	n
	)

static

Definition at line 862 of file n_html.c.

References _n_html_starts_ci().

Referenced by _n_html_push_js_url().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _n_html_lc()

static char _n_html_lc ( char c )

static

Definition at line 520 of file n_html.c.

Referenced by n_html_to_text().

Here is the caller graph for this function:

◆ _n_html_parse_tag()

static void _n_html_parse_tag	(	const char *	html,
		size_t	len,
		size_t *	pos,
		LIST *	out
	)

static

Definition at line 162 of file n_html.c.

References _n_html_attr_free(), _n_html_is_name_char(), _n_html_strndup(), _n_html_strndup_lower(), _n_html_tag_free(), N_HTML_TAG_::attrs, N_HTML_TAG_::closing, Free, FreeNoLog, list_push(), Malloc, MAX_LIST_ITEMS, N_HTML_ATTR_::name, N_HTML_TAG_::name, new_generic_list(), N_HTML_TAG_::self_closing, and N_HTML_ATTR_::value.

Referenced by _n_html_scan_tags().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _n_html_push_js_url()

static void _n_html_push_js_url	(	LIST *	out,
		const char *	tok,
		size_t	len
	)

static

Definition at line 887 of file n_html.c.

References _n_html_find(), _n_html_js_url_like(), _n_html_push_link(), N_STR::data, and list_foreach.

Referenced by n_html_extract_js_urls().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _n_html_push_link()

static void _n_html_push_link	(	LIST *	out,
		const char *	url
	)

static

Definition at line 354 of file n_html.c.

References char_to_nstr(), free_nstr_ptr(), and list_push().

Referenced by _n_html_push_js_url(), and n_html_extract_links().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _n_html_put_cp()

static void _n_html_put_cp	(	char *	buf,
		size_t	cap,
		size_t *	o,
		unsigned long	cp
	)

static

Definition at line 526 of file n_html.c.

Referenced by n_html_to_text().

Here is the caller graph for this function:

◆ _n_html_put_str()

static void _n_html_put_str	(	char *	buf,
		size_t	cap,
		size_t *	o,
		const char *	s
	)

static

Definition at line 552 of file n_html.c.

Referenced by n_html_to_text().

Here is the caller graph for this function:

◆ _n_html_scan_tags()

static LIST * _n_html_scan_tags	(	const char *	html,
		size_t	len
	)

static

Definition at line 290 of file n_html.c.

References _n_html_parse_tag(), _n_html_skip_rawtext(), N_HTML_TAG_::closing, LIST::end, MAX_LIST_ITEMS, N_HTML_TAG_::name, new_generic_list(), LIST_NODE::ptr, and N_HTML_TAG_::self_closing.

Referenced by n_html_extract_forms(), and n_html_extract_links().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _n_html_skip_rawtext()

static size_t _n_html_skip_rawtext	(	const char *	html,
		size_t	len,
		size_t	start,
		const char *	close_tag
	)

static

Definition at line 267 of file n_html.c.

Referenced by _n_html_scan_tags().

Here is the caller graph for this function:

◆ _n_html_starts_ci()

static int _n_html_starts_ci	(	const char *	s,
		size_t	n,
		const char *	pre,
		size_t	plen
	)

static

Definition at line 846 of file n_html.c.

Referenced by _n_html_js_url_like().

Here is the caller graph for this function:

◆ _n_html_strdup_upper()

static char * _n_html_strdup_upper ( const char * s )

static

Definition at line 115 of file n_html.c.

References _n_html_strndup(), and Malloc.

Referenced by n_html_extract_forms().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _n_html_strndup()

static char * _n_html_strndup	(	const char *	s,
		size_t	n
	)

static

Definition at line 105 of file n_html.c.

References Malloc.

Referenced by _n_html_parse_tag(), _n_html_strdup_upper(), n_html_extract_forms(), and n_sitemap_extract_urls().

Here is the caller graph for this function:

◆ _n_html_strndup_lower()

static char * _n_html_strndup_lower	(	const char *	s,
		size_t	n
	)

static

Definition at line 92 of file n_html.c.

References Malloc.

Referenced by _n_html_parse_tag(), and n_html_extract_forms().

Here is the caller graph for this function:

◆ _n_html_tag_free()

static void _n_html_tag_free ( void * p )

static

Definition at line 62 of file n_html.c.

References N_HTML_TAG_::attrs, Free, FreeNoLog, list_destroy(), and N_HTML_TAG_::name.

Referenced by _n_html_parse_tag().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ n_html_extract_forms()

LIST * n_html_extract_forms	(	const char *	html,
		size_t	len
	)

extract forms as a LIST of N_HTML_FORM*; free with n_html_forms_free

Examples: ex_html.c.

Definition at line 392 of file n_html.c.

References _n_form_field_free(), _n_html_attr_lookup(), _n_html_form_free(), _n_html_scan_tags(), _n_html_strdup_upper(), _n_html_strndup(), _n_html_strndup_lower(), N_HTML_FORM::action, N_HTML_TAG_::closing, N_HTML_FORM::enctype, N_HTML_FORM::fields, list_destroy(), list_foreach, list_push(), Malloc, MAX_LIST_ITEMS, N_HTML_FORM::method, N_FORM_FIELD::name, N_HTML_TAG_::name, new_generic_list(), N_FORM_FIELD::required, N_FORM_FIELD::type, and N_FORM_FIELD::value.

Referenced by test_empty_and_null(), test_enctype(), and test_forms().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ n_html_extract_js_urls()

LIST * n_html_extract_js_urls	(	const char *	js,
		size_t	len
	)

extract URL/path tokens from JavaScript source (quoted string literals that look like an http(s) URL, a protocol-relative "//host" URL, or a "/" / "./" / "../" path; a "${...}" template is truncated to its static prefix), deduped, as a LIST of N_STR*; free with n_html_links_free.

Recovers fetch()/XHR/route endpoints a static link scan misses.

Examples: ex_html.c.

Definition at line 915 of file n_html.c.

References _n_html_push_js_url(), MAX_LIST_ITEMS, and new_generic_list().

Referenced by test_js_urls(), and test_scripts().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ n_html_extract_links()

LIST * n_html_extract_links	(	const char *	html,
		size_t	len
	)

extract link URLs (a/href, form/action, img/src, script/src, link/href, iframe/src) as a LIST of N_STR*; free with n_html_links_free

Examples: ex_html.c.

Definition at line 360 of file n_html.c.

References _n_html_attr_lookup(), _n_html_push_link(), _n_html_scan_tags(), N_HTML_TAG_::closing, list_destroy(), list_foreach, MAX_LIST_ITEMS, N_HTML_TAG_::name, and new_generic_list().

Referenced by test_empty_and_null(), and test_links().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ n_html_extract_scripts()

LIST * n_html_extract_scripts	(	const char *	html,
		size_t	len
	)

extract the inline <script> bodies (those without a src attribute) from an HTML document as a LIST of N_STR*; free with n_html_links_free.

Feed each to n_html_extract_js_urls to mine inline endpoints.

Examples: ex_html.c.

Definition at line 941 of file n_html.c.

References _n_html_find_ci(), char_to_nstr_ex(), free_nstr_ptr(), list_push(), MAX_LIST_ITEMS, and new_generic_list().

Referenced by test_scripts().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ n_html_forms_free()

void n_html_forms_free ( LIST ** forms )

free a list returned by n_html_extract_forms

Examples: ex_html.c.

Definition at line 984 of file n_html.c.

References list_destroy().

Referenced by test_empty_and_null(), test_enctype(), and test_forms().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ n_html_links_free()

void n_html_links_free ( LIST ** links )

free a list returned by n_html_extract_links or n_sitemap_extract_urls

Examples: ex_html.c.

Definition at line 979 of file n_html.c.

References list_destroy().

Referenced by test_empty_and_null(), test_js_urls(), test_links(), test_scripts(), and test_sitemap().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ n_html_to_text()

N_STR * n_html_to_text	(	const char *	html,
		size_t	len
	)

render HTML to readable plain text: drop tags, skip script/style, decode common entities, collapse whitespace, and line-break block elements; returns a new N_STR (free with free_nstr) or NULL

Examples: ex_html.c.

Definition at line 627 of file n_html.c.

References _n_html_block_kind(), _n_html_entities, _n_html_flush(), _n_html_lc(), _n_html_put_cp(), _n_html_put_str(), char_to_nstr_ex(), Free, LOG_ERR, Malloc, and n_log.

Referenced by test_to_text().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ n_sitemap_extract_urls()

LIST * n_sitemap_extract_urls	(	const char *	xml,
		size_t	len
	)

extract <loc> URLs from a sitemap.xml as a LIST of N_STR*; free with n_html_links_free

Examples: ex_html.c.

Definition at line 456 of file n_html.c.

References _n_html_strndup(), char_to_nstr(), Free, free_nstr_ptr(), list_push(), MAX_LIST_ITEMS, and new_generic_list().

Referenced by test_empty_and_null(), and test_sitemap().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

◆ [struct]

const struct { ... } _n_html_entities[]

Initial value:

= {
    {"amp", "&"},
    {"lt", "<"},
    {"gt", ">"},
    {"quot", "\""},
    {"apos", "'"},
    {"nbsp", " "},
    {"copy", "(c)"},
    {"reg", "(R)"},
    {"trade", "(tm)"},
    {"mdash", "-"},
    {"ndash", "-"},
    {"minus", "-"},
    {"shy", ""},
    {"hellip", "..."},
    {"lsquo", "'"},
    {"rsquo", "'"},
    {"sbquo", "'"},
    {"ldquo", "\""},
    {"rdquo", "\""},
    {"bdquo", "\""},
    {"middot", "."},
    {"bull", "*"},
    {"deg", " deg "},
    {"laquo", "<<"},
    {"raquo", ">>"},
    {"times", "x"},
    {"divide", "/"},
    {"frac12", "1/2"},
    {"frac14", "1/4"},
    {"frac34", "3/4"},
    {"plusmn", "+/-"}}

Referenced by n_html_to_text().

Data Structures

Functions

Variables

Detailed Description

Data Structure Documentation

◆ N_HTML_ATTR_

◆ N_HTML_TAG_

Function Documentation

◆ _n_form_field_free()

◆ _n_html_attr_free()

◆ _n_html_attr_lookup()

◆ _n_html_block_kind()

◆ _n_html_ci_eq()

◆ _n_html_find()

◆ _n_html_find_ci()

◆ _n_html_flush()

◆ _n_html_form_free()

◆ _n_html_is_name_char()

◆ _n_html_js_url_like()

◆ _n_html_lc()

◆ _n_html_parse_tag()

◆ _n_html_push_js_url()

◆ _n_html_push_link()

◆ _n_html_put_cp()

◆ _n_html_put_str()

◆ _n_html_scan_tags()

◆ _n_html_skip_rawtext()

◆ _n_html_starts_ci()

◆ _n_html_strdup_upper()

◆ _n_html_strndup()

◆ _n_html_strndup_lower()

◆ _n_html_tag_free()

◆ n_html_extract_forms()

◆ n_html_extract_js_urls()

◆ n_html_extract_links()

◆ n_html_extract_scripts()

◆ n_html_forms_free()

◆ n_html_links_free()

◆ n_html_to_text()

◆ n_sitemap_extract_urls()

Variable Documentation

◆ [struct]