← Commit list

1750c4b27f

Daniel Stenberg

2026-09-23

SUCCESS Duration: 379s

Subject

test-linter: verify test cases

Binary

No binary (build failed)

Architecture & Compiler Flags

Architecture: x86_64-linux-gnu Compiler: gcc (Ubuntu 11.4.0-1ubuntu1~22.04.3) 11.4.0

—

Build Log

(log yok)

Commit Diff GitHub →

.github/workflows/linux.yml +6 -1 docs/tests/FILEFORMAT.md +7 -3 tests/Makefile.am +1 -0 tests/data/test1394 +1 -4 tests/data/test1604 +1 -4 tests/data/test1621 +1 -4 tests/data/test1622 +1 -4 tests/data/test1623 +1 -4 tests/data/test1720 +1 -4 tests/data/test1989 +0 -3 tests/runner.pm +8 -1 tests/test-linter.pl +351 -0

.github/workflows/linux.yml+6 -1

@@ -76,7 +76,7 @@ jobs:
	matrix:
	build:
	- name: 'awslc'
-	install_steps: awslc pytest
+	install_steps: awslc pytest test-linter
	LDFLAGS: -Wl,-rpath,/home/runner/awslc/lib
	configure: --with-openssl=/home/runner/awslc --enable-ech --enable-ntlm

@@ -926,6 +926,11 @@ jobs:
	make -C bld V=1 -C tests
	fi

+	- name: 'test-linter'
+	if: ${{ contains(matrix.build.install_steps, 'test-linter') }}
+	run:
+	cd tests && ./test-linter.pl ../bld
+
	- name: 'run tests'
	if: ${{ !contains(matrix.build.install_steps, 'skipall') && !contains(matrix.build.install_steps, 'skiprun') }}
	timeout-minutes: ${{ contains(matrix.build.install_packages, 'valgrind') && 20 \|\| 10 }}

docs/tests/FILEFORMAT.md+7 -3

@@ -244,8 +244,9 @@ for users to skip certain classes of tests, if desired, but a few are treated
	specially by the test harness or build system.

	When running a unit test and the keywords include `unittest`, the `<tool>`
-	section can be left empty to use the standard unit test tool name `unitN` where
-	`N` is the test number.
+	section can be left empty to use the standard unit test tool name `unitN`
+	where `N` is the test number. Unit tests means libcurl unit tests. The
+	command line tool's unit tests are marked as `tunittest`.

	The `test-ci` make target automatically skips test with the `flaky` keyword.

@@ -540,7 +541,7 @@ Features testable here are:
	- `typecheck`
	- `threadsafe`
	- `Unicode`
-	- `unittest`
+	- `unittest` - that feature needs to be enabled also for the tool unit tests marked as `tunittest`
	- `UnixSockets`
	- `verbose-strings`
	- `wakeup`
@@ -575,6 +576,9 @@ the `unit/` directory (if the tool name starts with `unit`).
	If the keyword `unittest` is set in the test case, the default name for the
	tool is `unit%TESTNUMBER`.

+	If the keyword `tunittest` is set in the test case, the default name for the
+	tool is `tool%TESTNUMBER`.
+
	If the keyword `libtest` is set in the test case, the default name for the
	tool is `lib%TESTNUMBER`.

tests/Makefile.am+1 -0

@@ -74,6 +74,7 @@ EXTRA_DIST = \
	servers.pm \
	sshhelp.pm \
	sshserver.pl \
+	test-linter.pl \
	testcurl.pl \
	testutil.pm \
	tftpserver.pl \

tests/data/test1394+1 -4

@@ -2,7 +2,7 @@
	<testcase>
	<info>
	<keywords>
-	unittest
+	tunittest
	</keywords>
	</info>

@@ -14,9 +14,6 @@ unittest
	<name>
	unit test for parse_cert_parameter()
	</name>
-	<tool>
-	tool%TESTNUMBER
-	</tool>
	</client>

	<verify>

tests/data/test1604+1 -4

@@ -2,7 +2,7 @@
	<testcase>
	<info>
	<keywords>
-	unittest
+	tunittest
	</keywords>
	</info>

@@ -14,8 +14,5 @@ unittest
	<name>
	Test Windows/MS-DOS filename sanitization
	</name>
-	<tool>
-	tool%TESTNUMBER
-	</tool>
	</client>
	</testcase>

tests/data/test1621+1 -4

@@ -2,7 +2,7 @@
	<testcase>
	<info>
	<keywords>
-	unittest
+	tunittest
	stripcredentials
	</keywords>
	</info>
@@ -15,8 +15,5 @@ unittest
	<name>
	unit tests for stripcredentials from URL
	</name>
-	<tool>
-	tool%TESTNUMBER
-	</tool>
	</client>
	</testcase>

tests/data/test1622+1 -4

@@ -2,7 +2,7 @@
	<testcase>
	<info>
	<keywords>
-	unittest
+	tunittest
	tool-progress
	</keywords>
	</info>
@@ -15,9 +15,6 @@ unittest
	<name>
	tool time and size progress output
	</name>
-	<tool>
-	tool%TESTNUMBER
-	</tool>
	</client>

	<verify>

tests/data/test1623+1 -4

@@ -2,7 +2,7 @@
	<testcase>
	<info>
	<keywords>
-	unittest
+	tunittest
	tool-sizeparser
	</keywords>
	</info>
@@ -15,9 +15,6 @@ unittest
	<name>
	the size parser for --limit-rate
	</name>
-	<tool>
-	tool%TESTNUMBER
-	</tool>
	</client>

	<verify>

tests/data/test1720+1 -4

@@ -2,7 +2,7 @@
	<testcase>
	<info>
	<keywords>
-	unittest
+	tunittest
	</keywords>
	</info>

@@ -14,9 +14,6 @@ unittest
	<name>
	create_dir_hierarchy()
	</name>
-	<tool>
-	tool%TESTNUMBER
-	</tool>
	</client>

	<verify>

tests/data/test1989+0 -3

@@ -8,9 +8,6 @@ urlapi
	</info>

	<client>
-	<tool>
-	lib%TESTNUMBER
-	</tool>
	<name>
	curl_url_dup preserves guessed_scheme
	</name>

tests/runner.pm+8 -1

@@ -862,7 +862,10 @@ sub singletest_run {
	$tool = $CMDLINE;
	$disablevalgrind = 1;
	}
-	elsif(!$tool && (!$keywords{"unittest"} && !$keywords{"libtest"})) {
+	elsif(!$tool &&
+	!$keywords{"unittest"} &&
+	!$keywords{"tunittest"} &&
+	!$keywords{"libtest"}) {
	# run curl, add suitable command line options
	my $inc = "";
	if((!$cmdhash{'option'}) \|\| ($cmdhash{'option'} !~ /no-include/)) {
@@ -912,6 +915,10 @@ sub singletest_run {
	$tool_name = "lib$testnum";
	$tool = $tool_name . exe_ext('TOOL');
	}
+	elsif($keywords{"tunittest"} && !$tool) {
+	$tool_name = "tool$testnum";
+	$tool = $tool_name . exe_ext('TOOL');
+	}

	if($tool =~ /^lib/) {
	$tool = "libtests" . exe_ext('TOOL');

tests/test-linter.pl+351 -0

@@ -0,0 +1,351 @@
+	#!/usr/bin/env perl
+	#***************************************************************************
+	# _ _ ____ _
+	# Project ___\| \| \| \| _ \\| \|
+	# / __\| \| \| \| \|_) \| \|
+	# \| (__\| \|_\| \| _ <\| \|___
+	# \___\|\___/\|_\| \_\_____\|
+	#
+	# Copyright (C) Daniel Stenberg, <[email protected]>, et al.
+	#
+	# This software is licensed as described in the file COPYING, which
+	# you should have received as part of this distribution. The terms
+	# are also available at https://curl.se/docs/copyright.html.
+	#
+	# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+	# copies of the Software, and permit persons to whom the Software is
+	# furnished to do so, under the terms of the COPYING file.
+	#
+	# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+	# KIND, either express or implied.
+	#
+	# SPDX-License-Identifier: curl
+	#
+	###########################################################################
+
+	# If the build is done out-of-source-tree, specify the build root dir as an
+	# argument to this script. Necessary to find the generated buildprotos.h
+	# header file.
+	my $root = "..";
+
+	if($ARGV[0]) {
+	$root = $ARGV[0];
+	}
+
+	# return all known test cases
+	sub alltests {
+	my @a;
+	open(M, "<./data/Makefile.am") \|\|
+	print STDERR "can't open data/Makefile.am\n";
+	while(<M>) {
+	if(/^ test(\d+)/) {
+	push @a, $1; # just the number
+	}
+	}
+	close(M);
+	return @a;
+	}
+
+	sub libtests {
+	my @a;
+	open(M, "<./libtest/Makefile.inc") \|\|
+	print STDERR "can't open Makefile.inc\n";
+	while(<M>) {
+	if(/^ *lib(\d+)\.c/) {
+	push @a, $1; # just the number
+	}
+	}
+	close(M);
+	return @a;
+	}
+
+	sub tunittests {
+	my @a;
+	open(M, "<./tunit/Makefile.inc") \|\|
+	print STDERR "can't open Makefile.inc\n";
+	while(<M>) {
+	if(/^ *tool(\d+)\.c/) {
+	push @a, $1; # just the number
+	}
+	}
+	close(M);
+	return @a;
+	}
+
+	sub unittests {
+	my @a;
+	open(M, "<./unit/Makefile.inc") \|\|
+	print STDERR "can't open Makefile.inc\n";
+	while(<M>) {
+	if(/^ *unit(\d+)\.c/) {
+	push @a, $1; # just the number
+	}
+	}
+	close(M);
+	return @a;
+	}
+
+	# provided a test number, return the keywords it uses
+	sub keywords {
+	my ($num) = @_;
+	open(my $fh, "<./data/test$num") \|\|
+	print STDERR "can't open test$num\n";
+	local $/;
+	my $c = <$fh>;
+	close($fh);
+
+	# Match and extract the <keywords> block
+	if ($c =~ /<keywords>\n(.*?)<\/keywords>/s) {
+	return split(/\n/, $1);
+	}
+	return undef;
+	}
+
+	# provided a test number, return the <tool> it uses
+	sub tool {
+	my ($num) = @_;
+	open(my $fh, "<./data/test$num") \|\|
+	print STDERR "can't open test$num\n";
+	local $/;
+	my $c = <$fh>;
+	close($fh);
+
+	# Match and extract the <tool> block
+	if ($c =~ /<tool>\n(.*?)<\/tool>/s) {
+	return split(/\n/, $1);
+	}
+	return undef;
+	}
+
+	# return all the function names that are prototyped here, as a hash
+	sub unitprotos {
+	my ($num) = @_;
+	my %protos;
+	open(my $fh, "<$root/lib/unitprotos.h") \|\|
+	print STDERR "can't open unitprotos.h\n";
+	while(<$fh>) {
+	if(/^UNITTEST .[ ]([a-z0-9_]+)\(/i) {
+	$protos{$1} = 1;
+	}
+	}
+	close($fh);
+	return %protos;
+	}
+
+	# check if this unit test actually uses any internal symbols
+	sub checkunit {
+	my ($num, $protref) = @_;
+	open(my $fh, "<./unit/unit$num.c") \|\|
+	print STDERR "can't open unit$num.c\n";
+	my $private = 0;
+	while(<$fh>) {
+	# check all function calls
+	while(/([a-z0-9_]+)\(/cig) {
+	my $f = $1;
+	#print "Check ($num) $f\n";
+	if($f =~ /^(Curl_\|curlx_)/) {
+	$private = 1;
+	last;
+	}
+	elsif($$protref{$f}) {
+	# it uses prototyped one
+	$private = 1;
+	last;
+	}
+	}
+	if(/struct (Curl_\|connectdata)/) {
+	$private = 1;
+	last;
+	}
+	elsif(/\bCURLPROTO_(WS\|WSS\|MQTTS\|SOCKS)\b/) {
+	$private = 1;
+	last;
+	}
+	}
+	close($fh);
+	if(!$private) {
+	print "Unit test $num uses no private functions or structs\n";
+	}
+	return !$private;
+	}
+
+	sub checkxml {
+	my ($num) = @_;
+	open(my $fh, "<./data/test$num") \|\|
+	print STDERR "can't open test$num\n";
+	my $bad = 1;
+	while(<$fh>) {
+	if(/^<\?xml/) {
+	$bad = 0;
+	last;
+	}
+	print "data/test$num lacks <xml> tag\n";
+	last;
+	}
+	close($fh);
+	return $bad;
+	}
+
+	my $error;
+	my @l = libtests();
+
+	print "- Tests with source code in libtests/ have 'libtest' as <keyword>\n";
+	for my $t (@l) {
+	my $lt = 0;
+	my @k = keywords($t);
+	for my $w (@k) {
+	if($w eq "libtest") {
+	$lt = 1;
+	}
+	}
+	if(!$lt) {
+	print "Test $t in data/test$t lacks the 'libtest' keyword\n";
+	$error++;
+	}
+	}
+
+	print "- Tests with source code in tunit/ have 'tunittest' as <keyword>\n";
+	my @l = tunittests();
+	for my $t (@l) {
+	my $lt = 0;
+	my @k = keywords($t);
+	for my $w (@k) {
+	if($w eq "tunittest") {
+	$lt = 1;
+	}
+	elsif($w eq "unittest") {
+	print "Test $t in data/test$t wrongly has the 'unittest' keyword\n";
+	}
+	}
+	if(!$lt) {
+	print "Test $t in data/test$t lacks the 'tunittest' keyword\n";
+	$error++;
+	}
+	}
+
+	print "- Tests with source code in unit/ have 'unittest' as <keyword>\n";
+	my @l = unittests();
+	for my $t (@l) {
+	my $lt = 0;
+	my @k = keywords($t);
+	for my $w (@k) {
+	if($w eq "unittest") {
+	$lt = 1;
+	}
+	elsif($w eq "tunittest") {
+	print "Test $t in data/test$t wrongly has the 'tunittest' keyword\n";
+	}
+	}
+	if(!$lt) {
+	print "Test $t in data/test$t lacks the 'unittest' keyword\n";
+	$error++;
+	}
+	}
+
+	my @a = alltests();
+	print "- Tests that set lib* in <tool> have 'libtest' as <keyword>\n";
+	for my $t (@a) {
+	my @k = tool($t);
+	for my $w (@k) {
+	if($w =~ /lib%TESTNUMBER/) {
+	print "test $t invokes default tool\n";
+	$error++;
+	}
+	elsif($w =~ /lib(\d+)/) {
+	my $num = $1;
+	if($num == $t) {
+	print "test $t invokes default tool\n";
+	$error++;
+	}
+	my $lt = 0;
+
+	# get the keywords for test $num
+	my @k = keywords($t);
+	for my $w (@k) {
+	if($w eq "libtest") {
+	$lt = 1;
+	}
+	}
+	if(!$lt) {
+	print "Test $t in data/test$t lacks the 'libtest' keyword\n";
+	$error++;
+	}
+	}
+	}
+	}
+
+	print "- Tests that set tool* in <tool> have 'tunittest' as <keyword>\n";
+	for my $t (@a) {
+	my @k = tool($t);
+	for my $w (@k) {
+	if($w =~ /tool%TESTNUMBER/) {
+	print "test $t invokes default tool\n";
+	$error++;
+	}
+	elsif($w =~ /tool(\d+)/) {
+	my $num = $1;
+	if($num == $t) {
+	print "test $t invokes default tool\n";
+	$error++;
+	}
+	my $lt = 0;
+
+	# get the keywords for test $num
+	my @k = keywords($num);
+	for my $w (@k) {
+	if($w eq "tunittest") {
+	$lt = 1;
+	}
+	}
+	if(!$lt) {
+	print "Test $t in data/test$t lacks the 'tunittest' keyword\n";
+	$error++;
+	}
+	}
+	}
+	}
+
+	print "- Tests that set unit* in <tool> have 'unittest' as <keyword>\n";
+	for my $t (@a) {
+	my @k = tool($t);
+	for my $w (@k) {
+	if($w =~ /unit%TESTNUMBER/) {
+	print "test $t invokes default tool\n";
+	$error++;
+	}
+	elsif($w =~ /unit(\d+)/) {
+	my $num = $1;
+	if($num == $t) {
+	print "test $t invokes default tool\n";
+	$error++;
+	}
+	my $lt = 0;
+
+	# get the keywords for test $num
+	my @k = keywords($num);
+	for my $w (@k) {
+	if($w eq "unittest") {
+	$lt = 1;
+	}
+	}
+	if(!$lt) {
+	print "Test $t in data/test$t lacks the 'unittest' keyword\n";
+	$error++;
+	}
+	}
+	}
+	}
+
+	print "- Unit tests actually use private symbols\n";
+	my @l = unittests();
+	my %protos = unitprotos();
+	for my $t (@l) {
+	$error += checkunit($t, \%protos);
+	}
+
+	print "- All tests have <xml> tag on first line\n";
+	for my $t (@a) {
+	$error += checkxml($t);
+	}
+
+	exit $error;