← Commit list

cb0636980b

Daniel Stenberg

2026-05-12

SUCCESS Duration: 215s

Subject

tool_urlglob: add named globs

Binary

No binary (build failed)

Architecture & Compiler Flags

Architecture: x86_64-linux-gnu Compiler: gcc (Ubuntu 11.4.0-1ubuntu1~22.04.3) 11.4.0

—

Build Log

(log yok)

Commit Diff GitHub →

docs/cmdline-opts/_GLOBBING.md +13 -0 docs/cmdline-opts/output.md +11 -0 src/tool_operate.c +6 -1 src/tool_urlglob.c +135 -53 src/tool_urlglob.h +4 -0 tests/data/Makefile.am +1 -0 tests/data/test2408 +92 -0 tests/data/test2409 +92 -0 tests/data/test2410 +37 -0 tests/data/test2411 +37 -0 tests/data/test75 +1 -1 tests/data/test759 +1 -1 tests/data/test761 +2 -2

docs/cmdline-opts/_GLOBBING.md+13 -0

@@ -39,4 +39,17 @@ probably have to put the full URL within double quotes to avoid the shell from
	interfering with it. This also goes for other characters treated special, like
	for example '&', '?' and '*'.

+	The separate globbing components can be referenced in the --output option to
+	allow pieces to be reused in the target filename.
+
+	Starting in curl 8.21.0, the separate globbing parts can be named and
+	referenced by their names. The case sensitive alphanumeric name is set
+	enclosed within angle brackets after the opening character. Examples:
+
+	https://fun.example/{<number>one,two,three}.jpg
+
+	ftp://ftp.example.com/file[<range>1-100].txt
+
+	Setting the same glob name twice is an error.
+
	Switch off globbing with --globoff.

docs/cmdline-opts/output.md+11 -0

@@ -69,3 +69,14 @@ override curl's internal binary output in terminal prevention:

	Note that the binary output may be caused by the response being compressed, in
	which case you may want to use the --compressed option.
+
+	Starting in curl 8.21.0, the separate globbing parts can be named and
+	referenced by their names. The case sensitive alphanumeric name is set
+	enclosed within angle brackets after the opening character. Examples:
+
+	curl "https://fun.example/{<num>one,two}.jpg" -o "save-#<num>"
+
+	curl "ftp://ftp.example/file[<range>1-100].txt" \
+	-o "save-#<range>.txt"
+
+	Referencing a named glob that is not set, causes an error.

src/tool_operate.c+6 -1

@@ -1067,7 +1067,12 @@ static CURLcode setup_outfile(struct OperationConfig *config,
	}
	else if(result) {
	/* bad globbing */
-	warnf("bad output glob");
+	if(state->urlglob.error) {
+	glob_show_error(&state->urlglob, u->outfile, tool_stderr, result);
+	config->synthetic_error = TRUE;
+	}
+	else
+	warnf("bad output glob");
	return result;
	}
	if(!*per->outfile) {

src/tool_urlglob.c+135 -53

@@ -56,6 +56,7 @@ static CURLcode glob_fixed(struct URLGlob glob, char fixed, size_t len)

	pat->c.set.palloc = 1;
	pat->c.set.size = 1;
+	pat->name = NULL; /* unnamed */
	return CURLE_OK;
	}

@@ -89,7 +90,7 @@ static int multiply(curl_off_t *amount, curl_off_t with)

	static CURLcode glob_set(struct URLGlob glob, const char *patternp,
	size_t posp, curl_off_t amount,
-	int globindex)
+	int globindex, const struct Curl_str *name)
	{
	/* processes a set expression with the point behind the opening '{'
	','-separated elements are collected until the next closing '}'
@@ -103,6 +104,7 @@ static CURLcode glob_set(struct URLGlob glob, const char *patternp,
	size_t size = 0;
	char **elem = NULL;
	size_t palloc = 0; /* start with this */
+	DEBUGASSERT(name);

	while(!done) {
	switch(*pattern) {
@@ -198,6 +200,15 @@ static CURLcode glob_set(struct URLGlob glob, const char *patternp,
	pat->c.set.size = size;
	pat->c.set.idx = 0;
	pat->c.set.palloc = palloc;
+	if(curlx_strlen(name)) {
+	pat->name = curlx_memdup0(curlx_str(name), curlx_strlen(name));
+	if(!pat->name) {
+	result = CURLE_OUT_OF_MEMORY;
+	goto error;
+	}
+	}
+	else
+	pat->name = NULL; /* no name */

	return CURLE_OK;
	error:
@@ -212,7 +223,7 @@ error:

	static CURLcode glob_range(struct URLGlob glob, const char *patternp,
	size_t posp, curl_off_t amount,
-	int globindex)
+	int globindex, const struct Curl_str *name)
	{
	/* processes a range expression with the point behind the opening '['
	- char range: e.g. "a-z]", "B-Q]"
@@ -224,8 +235,10 @@ static CURLcode glob_range(struct URLGlob glob, const char *patternp,
	const char pattern = patternp;
	const char *c;

+	DEBUGASSERT(name);
	pat = &glob->pattern[glob->pnum];
	pat->globindex = globindex;
+	pat->name = NULL; /* no name (so far) */

	if(ISALPHA(*pattern)) {
	/* character range detected */
@@ -340,6 +353,11 @@ static CURLcode glob_range(struct URLGlob glob, const char *patternp,
	return globerror(glob, "bad range specification", *posp,
	CURLE_URL_MALFORMAT);

+	if(curlx_strlen(name)) {
+	pat->name = curlx_memdup0(curlx_str(name), curlx_strlen(name));
+	if(!pat->name)
+	return CURLE_OUT_OF_MEMORY;
+	}
	*patternp = pattern;
	return CURLE_OK;
	}
@@ -407,12 +425,31 @@ static CURLcode add_glob(struct URLGlob *glob, size_t pos)
	return CURLE_OK;
	}

+	/* returns the named glob pattern (case sensitively) if it exists, otherwise
+	NULL
+	*/
+	static struct URLPattern glob_find_name(struct URLGlob glob,
+	struct Curl_str *name)
+	{
+	size_t i;
+	/* find the correct glob entry */
+	for(i = 0; i < glob->pnum; i++) {
+	if(glob->pattern[i].name &&
+	curlx_str_cmp(name, glob->pattern[i].name))
+	return &glob->pattern[i];
+	}
+	return NULL; /* no match */
+	}
+
+	#define MAX_GLOBNAME_LEN 64
+
	static CURLcode glob_parse(struct URLGlob glob, const char pattern,
	size_t pos, curl_off_t *amount)
	{
	/* processes a literal string component of a URL
	special characters '{' and '[' branch to set/range processing functions
	*/
+	const char ipattern = pattern; / start position */
	CURLcode result = CURLE_OK;
	int globindex = 0; /* count "actual" globs */

@@ -464,21 +501,39 @@ static CURLcode glob_parse(struct URLGlob glob, const char pattern,
	curlx_dyn_reset(&glob->buf);
	}
	else {
+	struct Curl_str name;
	if(!pattern) / done */
	break;
-	else if(*pattern == '{') {
-	/* process set pattern */
+	else if((pattern == '{') \|\| (pattern == '[')) {
+	bool set = (*pattern == '{');
+	const char *start;
	pattern++;
	pos++;
-	result = glob_set(glob, &pattern, &pos, amount, globindex++);
-	if(!result)
-	result = add_glob(glob, pos);
-	}
-	else if(*pattern == '[') {
-	/* process range pattern */
-	pattern++;
-	pos++;
-	result = glob_range(glob, &pattern, &pos, amount, globindex++);
+	start = pattern;
+	/* fetch the name, if provided */
+	if(curlx_str_single(&pattern, '<') \|\|
+	curlx_str_until(&pattern, &name, MAX_GLOBNAME_LEN, '>') \|\|
+	curlx_str_single(&pattern, '>')) {
+	/* Not a proper name. This is not reporting errors on syntax errors
+	on purpose: it means that if there is an existing use case that
+	uses what looks like a broken named-glob syntax (now introduced)
+	we let that function like before. */
+	curlx_str_init(&name);
+	pattern = start; /* reset any partial patch */
+	}
+	else {
+	/* check that the name is not already used */
+	struct URLPattern *p = glob_find_name(glob, &name);
+	if(p)
+	return globerror(glob, "Duplicate glob name", 2 + start - ipattern,
+	CURLE_URL_MALFORMAT);
+	}
+	if(set)
+	result = glob_set(glob, &pattern, &pos, amount, globindex++, &name);
+	else
+	result = glob_range(glob, &pattern, &pos, amount, globindex++,
+	&name);
+
	if(!result)
	result = add_glob(glob, pos);
	}
@@ -492,6 +547,26 @@ bool glob_inuse(struct URLGlob *glob)
	return glob->palloc ? TRUE : FALSE;
	}

+	/* a glob error has been confirmed, this outputs details about it to the set
+	error stream */
+	void glob_show_error(struct URLGlob glob, const char url, FILE *error,
+	CURLcode result)
+	{
+	char text[512];
+	const char *t;
+	if(glob->pos) {
+	curl_msnprintf(text, sizeof(text), "%s in position %zu:\n%s\n%*s^",
+	glob->error,
+	glob->pos, url, (int)glob->pos - 1, " ");
+	t = text;
+	}
+	else
+	t = glob->error;
+
+	/* send error description to the error-stream */
+	curl_mfprintf(error, "curl: (%d) %s\n", result, t);
+	}
+
	CURLcode glob_url(struct URLGlob glob, const char url, curl_off_t *urlnum,
	FILE *error)
	{
@@ -511,21 +586,8 @@ CURLcode glob_url(struct URLGlob glob, const char url, curl_off_t *urlnum,

	result = glob_parse(glob, url, 1, &amount);
	if(result) {
-	if(error && glob->error) {
-	char text[512];
-	const char *t;
-	if(glob->pos) {
-	curl_msnprintf(text, sizeof(text), "%s in URL position %zu:\n%s\n%*s^",
-	glob->error,
-	glob->pos, url, (int)glob->pos - 1, " ");
-	t = text;
-	}
-	else
-	t = glob->error;
-
-	/* send error description to the error-stream */
-	curl_mfprintf(error, "curl: (%d) %s\n", result, t);
-	}
+	if(error && glob->error)
+	glob_show_error(glob, url, error, result);
	*urlnum = 1;
	return result;
	}
@@ -547,6 +609,7 @@ void glob_cleanup(struct URLGlob *glob)
	curlx_safefree(glob->pattern[i].c.set.elem[elem]);
	curlx_safefree(glob->pattern[i].c.set.elem);
	}
+	curlx_safefree(glob->pattern[i].name);
	}
	curlx_safefree(glob->pattern);
	glob->palloc = 0;
@@ -643,6 +706,7 @@ CURLcode glob_match_url(char *output, const char filename,
	struct URLGlob glob, SANITIZEcode sc)
	{
	struct dynbuf dyn;
+	const char *ifilename = filename;
	*output = NULL;
	*sc = SANITIZE_ERR_OK;

@@ -650,11 +714,11 @@ CURLcode glob_match_url(char *output, const char filename,

	while(*filename) {
	CURLcode result = CURLE_OK;
+	struct URLPattern *pat = NULL;
	if(*filename == '#' && ISDIGIT(filename[1])) {
-	const char *ptr = filename;
+	/* a numbered glob reference */
+	const char *ptr = filename++;
	curl_off_t num;
-	struct URLPattern *pat = NULL;
-	filename++;
	if(!curlx_str_number(&filename, &num, glob->pnum) && num) {
	size_t i;
	num--; /* make it zero based */
@@ -666,31 +730,49 @@ CURLcode glob_match_url(char *output, const char filename,
	}
	}
	}
-
-	if(pat) {
-	switch(pat->type) {
-	case GLOB_SET:
-	if(pat->c.set.elem)
-	result = curlx_dyn_add(&dyn, pat->c.set.elem[pat->c.set.idx]);
-	break;
-	case GLOB_ASCII: {
-	char letter = (char)pat->c.ascii.letter;
-	result = curlx_dyn_addn(&dyn, &letter, 1);
-	break;
-	}
-	case GLOB_NUM:
-	result = curlx_dyn_addf(&dyn, "%0*" CURL_FORMAT_CURL_OFF_T,
-	pat->c.num.npad, pat->c.num.idx);
-	break;
-	default:
-	DEBUGASSERT(0);
+	if(!pat)
+	filename = ptr;
+	}
+	else if(*filename == '#' && (filename[1] == '<')) {
+	/* a named glob reference */
+	struct Curl_str name;
+	const char *ptr = filename;
+	filename += 2; /* pass both leading bytes */
+	if(!curlx_str_until(&filename, &name, MAX_GLOBNAME_LEN, '>') &&
+	!curlx_str_single(&filename, '>')) {
+	/* find the correct glob entry */
+	pat = glob_find_name(glob, &name);
+	if(!pat) {
+	/* when the name is given correctly, it needs to be an existing glob
+	name, which makes this an error */
	curlx_dyn_free(&dyn);
-	return CURLE_FAILED_INIT;
+	return globerror(glob, "no glob exists with this name",
+	filename - ifilename, CURLE_BAD_FUNCTION_ARGUMENT);
	}
	}
-	else
-	/* #[num] out of range, use the #[num] in the output */
-	result = curlx_dyn_addn(&dyn, ptr, filename - ptr);
+	if(!pat)
+	filename = ptr;
+	}
+	if(pat) {
+	switch(pat->type) {
+	case GLOB_SET:
+	if(pat->c.set.elem)
+	result = curlx_dyn_add(&dyn, pat->c.set.elem[pat->c.set.idx]);
+	break;
+	case GLOB_ASCII: {
+	char letter = (char)pat->c.ascii.letter;
+	result = curlx_dyn_addn(&dyn, &letter, 1);
+	break;
+	}
+	case GLOB_NUM:
+	result = curlx_dyn_addf(&dyn, "%0*" CURL_FORMAT_CURL_OFF_T,
+	pat->c.num.npad, pat->c.num.idx);
+	break;
+	default:
+	DEBUGASSERT(0);
+	curlx_dyn_free(&dyn);
+	return CURLE_FAILED_INIT;
+	}
	}
	else
	result = curlx_dyn_addn(&dyn, filename++, 1);

src/tool_urlglob.h+4 -0

@@ -33,6 +33,7 @@ typedef enum {

	struct URLPattern {
	globtype type;
+	char name; / if not NULL */
	int globindex; /* the number of this particular glob or -1 if not used
	within {} or [] */
	union {
@@ -71,6 +72,9 @@ struct URLGlob {
	size_t pos; /* column position of error or 0 */
	};

+	void glob_show_error(struct URLGlob glob, const char url, FILE *error,
+	CURLcode result);
+
	CURLcode glob_url(struct URLGlob glob, const char url, curl_off_t *urlnum,
	FILE *error);
	CURLcode glob_next_url(char *globbed, struct URLGlob glob);

tests/data/Makefile.am+1 -0

@@ -265,6 +265,7 @@ test2300 test2301 test2302 test2303 test2304 test2306 test2307 test2308 \
	test2309 \
	\
	test2400 test2401 test2402 test2403 test2404 test2405 test2406 test2407 \
+	test2408 test2409 test2410 test2411 \
	\
	test2500 test2501 test2502 test2503 test2504 test2505 test2506 \
	\

tests/data/test2408+92 -0

@@ -0,0 +1,92 @@
+	<?xml version="1.0" encoding="US-ASCII"?>
+	<testcase>
+	<info>
+	<keywords>
+	HTTP
+	HTTP GET
+	globbing
+	{} list
+	</keywords>
+	</info>
+	# Server-side
+	<reply>
+	<data crlf="headers" nocheck="yes">
+	HTTP/1.1 200 OK
+	Funny-head: yesyes
+	Content-Length: 4
+
+	moo
+	</data>
+	<data2 crlf="headers">
+	HTTP/1.1 200 OK
+	Funny-head: yesyes
+	Content-Length: 4
+
+	foo
+	</data2>
+	<data3 crlf="headers">
+	HTTP/1.1 200 OK
+	Funny-head: yesyes
+	Content-Length: 4
+
+	hoo
+	</data3>
+	</reply>
+
+	# Client-side
+	<client>
+	<server>
+	http
+	</server>
+	<name>
+	multiple requests using named {} globs in URL
+	</name>
+	<command option="no-output">
+	"%HOSTIP:%HTTPPORT/{%LTtest%GT%TESTNUMBER,%TESTNUMBER0002,%TESTNUMBER0003}" -o "%LOGDIR/dump-#%LTtest%GT"
+	</command>
+	</client>
+
+	# Verify data after the test has been "shot"
+	<verify>
+	<protocol crlf="headers">
+	GET /%TESTNUMBER HTTP/1.1
+	Host: %HOSTIP:%HTTPPORT
+	User-Agent: curl/%VERSION
+	Accept: /
+
+	GET /%TESTNUMBER0002 HTTP/1.1
+	Host: %HOSTIP:%HTTPPORT
+	User-Agent: curl/%VERSION
+	Accept: /
+
+	GET /%TESTNUMBER0003 HTTP/1.1
+	Host: %HOSTIP:%HTTPPORT
+	User-Agent: curl/%VERSION
+	Accept: /
+
+	</protocol>
+
+	<file name="%LOGDIR/dump-%TESTNUMBER" crlf="headers">
+	HTTP/1.1 200 OK
+	Funny-head: yesyes
+	Content-Length: 4
+
+	moo
+	</file>
+	<file2 name="%LOGDIR/dump-%TESTNUMBER0002" crlf="headers">
+	HTTP/1.1 200 OK
+	Funny-head: yesyes
+	Content-Length: 4
+
+	foo
+	</file2>
+	<file3 name="%LOGDIR/dump-%TESTNUMBER0003" crlf="headers">
+	HTTP/1.1 200 OK
+	Funny-head: yesyes
+	Content-Length: 4
+
+	hoo
+	</file3>
+
+	</verify>
+	</testcase>

tests/data/test2409+92 -0

@@ -0,0 +1,92 @@
+	<?xml version="1.0" encoding="US-ASCII"?>
+	<testcase>
+	<info>
+	<keywords>
+	HTTP
+	HTTP GET
+	globbing
+	{} list
+	</keywords>
+	</info>
+	# Server-side
+	<reply>
+	<data crlf="headers" nocheck="yes">
+	HTTP/1.1 200 swsbounce
+	Funny-head: yesyes
+	Content-Length: 4
+
+	moo
+	</data>
+	<data1 crlf="headers">
+	HTTP/1.1 200 swsbounce
+	Funny-head: yesyes
+	Content-Length: 4
+
+	foo
+	</data1>
+	<data2 crlf="headers">
+	HTTP/1.1 200 OK
+	Funny-head: yesyes
+	Content-Length: 4
+
+	hoo
+	</data2>
+	</reply>
+
+	# Client-side
+	<client>
+	<server>
+	http
+	</server>
+	<name>
+	multiple requests using named [] globs in URL
+	</name>
+	<command option="no-output">
+	"%HOSTIP:%HTTPPORT/hello[%LTtest%GT7-9]" -o "%LOGDIR/dump-#%LTtest%GT"
+	</command>
+	</client>
+
+	# Verify data after the test has been "shot"
+	<verify>
+	<protocol crlf="headers">
+	GET /hello7 HTTP/1.1
+	Host: %HOSTIP:%HTTPPORT
+	User-Agent: curl/%VERSION
+	Accept: /
+
+	GET /hello8 HTTP/1.1
+	Host: %HOSTIP:%HTTPPORT
+	User-Agent: curl/%VERSION
+	Accept: /
+
+	GET /hello9 HTTP/1.1
+	Host: %HOSTIP:%HTTPPORT
+	User-Agent: curl/%VERSION
+	Accept: /
+
+	</protocol>
+
+	<file name="%LOGDIR/dump-7" crlf="headers">
+	HTTP/1.1 200 swsbounce
+	Funny-head: yesyes
+	Content-Length: 4
+
+	moo
+	</file>
+	<file2 name="%LOGDIR/dump-8" crlf="headers">
+	HTTP/1.1 200 swsbounce
+	Funny-head: yesyes
+	Content-Length: 4
+
+	foo
+	</file2>
+	<file3 name="%LOGDIR/dump-9" crlf="headers">
+	HTTP/1.1 200 OK
+	Funny-head: yesyes
+	Content-Length: 4
+
+	hoo
+	</file3>
+
+	</verify>
+	</testcase>

tests/data/test2410+37 -0

@@ -0,0 +1,37 @@
+	<?xml version="1.0" encoding="US-ASCII"?>
+	<testcase>
+	<info>
+	<keywords>
+	HTTP
+	HTTP GET
+	globbing
+	{} list
+	</keywords>
+	</info>
+	# Server-side
+
+	# Client-side
+	<client>
+	<server>
+	http
+	</server>
+	<name>
+	duplicate named glob
+	</name>
+	<command option="no-output">
+	"%HOSTIP:%HTTPPORT/{%LTtest%GTA,B}{%LTtest%GTC,D}" -o "%LOGDIR/dump"
+	</command>
+	</client>
+
+	# Verify data after the test has been "shot"
+	<verify>
+	<stderr mode="text">
+	curl: (3) Duplicate glob name in position 30:
+	%HOSTIP:%HTTPPORT/{%LTtest%GTA,B}{%LTtest%GTC,D}
+	^
+	</stderr>
+	<errorcode>
+	3
+	</errorcode>
+	</verify>
+	</testcase>

tests/data/test2411+37 -0

@@ -0,0 +1,37 @@
+	<?xml version="1.0" encoding="US-ASCII"?>
+	<testcase>
+	<info>
+	<keywords>
+	HTTP
+	HTTP GET
+	globbing
+	{} list
+	</keywords>
+	</info>
+	# Server-side
+
+	# Client-side
+	<client>
+	<server>
+	http
+	</server>
+	<name>
+	reference a named glob not set
+	</name>
+	<command option="no-output">
+	"%HOSTIP:%HTTPPORT/{%LTtest%GTA,B}{%LTmoo%GTC,D}" -o "somewhere/#%LTfoo%GT"
+	</command>
+	</client>
+
+	# Verify data after the test has been "shot"
+	<verify>
+	<stderr mode="text">
+	curl: (43) no glob exists with this name in position 16:
+	somewhere/#%LTfoo%GT
+	^
+	</stderr>
+	<errorcode>
+	43
+	</errorcode>
+	</verify>
+	</testcase>

tests/data/test75+1 -1

@@ -31,7 +31,7 @@ HTTP, urlglob retrieval with bad range
	3
	</errorcode>
	<stdout mode="text">
-	curl: (3) bad range in URL position 47:
+	curl: (3) bad range in position 47:
	http://a-site-never-accessed.example.org/[2-1]
	^
	</stdout>

tests/data/test759+1 -1

@@ -18,7 +18,7 @@ glob '{,'

	# Verify data after the test has been "shot"
	<verify>
-	# curl: (3) unmatched brace in URL position 1:
+	# curl: (3) unmatched brace in position 1:
	<errorcode>
	3
	</errorcode>

tests/data/test761+2 -2

@@ -22,8 +22,8 @@ http://testingthis/%repeat[201 x {a}b]%
	3
	</errorcode>
	<stderr mode="text">
-	curl: (3) too many {} sets in URL position 403:
-	http://testingthis/%repeat[113 x {a}b]%{a
+	curl: (3) too many {} sets in position 403:
+	http://testingthis/%repeat[114 x {a}b]%{a
	</stderr>
	</verify>
	</testcase>