← Commit list

3f1cd809ee

Viktor Szakats
Viktor Szakats
2025-12-12
SUCCESS Duration: 220s

Subject

runtests: add options to set minimum number of tests, use them

Binary

No binary (build failed)

Architecture & Compiler Flags

Architecture: x86_64-linux-gnu Compiler: gcc (Ubuntu 11.4.0-1ubuntu1~22.04.3) 11.4.0

—

Build Log

(log yok)

Commit Diff GitHub →

.github/workflows/distcheck.yml +1 -0.github/workflows/linux-old.yml +1 -0.github/workflows/linux.yml +7 -3.github/workflows/macos.yml +11 -3.github/workflows/non-native.yml +1 -0.github/workflows/windows.yml +19 -15docs/runtests.md +6 -1tests/globalconfig.pm +2 -0tests/runtests.pl +18 -0
.github/workflows/distcheck.yml+1 -0
@@ -20,6 +20,7 @@ concurrency:
permissions: {}
env:
+ CURL_TEST_MIN: 1450
MAKEFLAGS: -j 5
jobs:
.github/workflows/linux-old.yml+1 -0
@@ -52,6 +52,7 @@ permissions: {}
env:
MAKEFLAGS: -j 5
CURL_CI: github
+ CURL_TEST_MIN: 1500
DEBIAN_FRONTEND: noninteractive
jobs:
.github/workflows/linux.yml+7 -3
@@ -36,6 +36,7 @@ permissions: {}
env:
MAKEFLAGS: -j 5
CURL_CI: github
+ CURL_TEST_MIN: 1600
CURL_CLANG_TIDYFLAGS: '-checks=-clang-analyzer-security.insecureAPI.bzero,-clang-analyzer-security.insecureAPI.strcpy,-clang-analyzer-optin.performance.Padding,-clang-analyzer-security.ArrayBound,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling,-clang-analyzer-valist.Uninitialized'
# renovate: datasource=github-tags depName=libressl/portable versioning=semver registryUrl=https://github.com
LIBRESSL_VERSION: 4.2.1
@@ -184,6 +185,7 @@ jobs:
configure: --with-openssl --disable-ipv6 --enable-debug --disable-unity --disable-libcurl-option --disable-digest-auth
- name: 'openssl https-only'
+ tflags: '--min=1150'
configure: >-
--with-openssl --enable-debug --disable-unity
--disable-dict --disable-gopher --disable-ldap --disable-telnet
@@ -195,13 +197,13 @@ jobs:
- name: 'openssl torture !FTP'
install_packages: libnghttp2-dev libssh2-1-dev libc-ares-dev
generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON
- tflags: '-t --shallow=25 !FTP'
+ tflags: '-t --shallow=25 !FTP --min=1450'
torture: true
- name: 'openssl torture FTP'
install_packages: libnghttp2-dev libssh2-1-dev libc-ares-dev
generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON
- tflags: '-t --shallow=20 FTP'
+ tflags: '-t --shallow=20 FTP --min=260'
torture: true
- name: 'openssl i686'
@@ -215,6 +217,7 @@ jobs:
--with-openssl --with-librtmp --with-libssh2 --with-libidn2 --enable-ares --enable-debug
- name: '!ssl !http !smtp !imap'
+ tflags: '--min=475'
configure: --without-ssl --enable-debug --disable-http --disable-smtp --disable-imap --disable-unity
- name: 'libressl Fil-C'
@@ -303,11 +306,12 @@ jobs:
LDFLAGS: -fsanitize=memory
LIBS: -ldl
configure: CC=clang --without-ssl --without-zlib --without-brotli --without-zstd --without-libpsl --without-nghttp2 --enable-debug
+ tflags: '--min=1480'
- name: 'event-based'
install_packages: libssh-dev
configure: --enable-debug --disable-shared --disable-threaded-resolver --with-libssh --with-openssl
- tflags: '-n --test-event'
+ tflags: '-n --test-event --min=1350'
- name: 'duphandle'
install_packages: libssh-dev
.github/workflows/macos.yml+11 -3
@@ -44,6 +44,7 @@ permissions: {}
env:
CURL_CI: github
+ CURL_TEST_MIN: 1700
MAKEFLAGS: -j 4
LDFLAGS: -w # suppress 'object file was built for newer macOS version than being linked' warnings
@@ -225,16 +226,20 @@ jobs:
compiler: clang
install: brotli zstd
configure: --without-ssl --with-brotli --with-zstd
+ tflags: '--min=1450'
xcode: '' # default Xcode. Set it once to silence actionlint.
- name: '!ssl !debug'
compiler: gcc-13
configure: --without-ssl
+ tflags: '--min=950'
- name: '!ssl'
compiler: clang
configure: --enable-debug --without-ssl
+ tflags: '--min=1500'
- name: '!ssl libssh2 AppleIDN'
compiler: clang
configure: --enable-debug --with-libssh2=/opt/homebrew/opt/libssh2 --without-ssl --with-apple-idn
+ tflags: '--min=1550'
- name: 'OpenSSL libssh c-ares'
compiler: clang
install: libssh
@@ -246,8 +251,10 @@ jobs:
- name: '!ssl c-ares'
compiler: clang
configure: --enable-debug --enable-ares --without-ssl
+ tflags: '--min=1500'
- name: '!ssl HTTP-only'
compiler: clang
+ tflags: '--min=950'
configure: >-
--enable-debug
--disable-alt-svc --disable-dict --disable-file --disable-ftp --disable-gopher --disable-imap
@@ -277,7 +284,7 @@ jobs:
- name: 'OpenSSL event-based'
compiler: clang
configure: --enable-debug --with-openssl=/opt/homebrew/opt/openssl
- tflags: --test-event
+ tflags: '--test-event --min=1300'
# cmake
- name: 'OpenSSL gsasl rtmp AppleIDN SecTrust +examples'
install: libnghttp3 libngtcp2 gsasl rtmpdump
@@ -328,18 +335,19 @@ jobs:
compiler: clang
install: rustls-ffi
generate: -DENABLE_DEBUG=ON -DCURL_USE_RUSTLS=ON -DUSE_ECH=ON -DCURL_DISABLE_LDAP=ON
+ tflags: '--min=1650'
- name: 'OpenSSL torture !FTP'
compiler: clang
install: libnghttp3
install_steps: torture
generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=/opt/homebrew/opt/openssl -DUSE_OPENSSL_QUIC=ON
- tflags: -t --shallow=25 !FTP
+ tflags: '-t --shallow=25 !FTP --min=1450'
- name: 'OpenSSL torture FTP'
compiler: clang
install: libnghttp3
install_steps: torture
generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=/opt/homebrew/opt/openssl -DUSE_OPENSSL_QUIC=ON
- tflags: -t --shallow=20 FTP
+ tflags: '-t --shallow=20 FTP --min=260'
exclude:
# opt out jobs from combinations that have the compiler set manually
- { compiler: llvm@18, build: { compiler: 'clang' } }
.github/workflows/non-native.yml+1 -0
@@ -37,6 +37,7 @@ permissions: {}
env:
CURL_CI: github
+ CURL_TEST_MIN: 1750
jobs:
netbsd:
.github/workflows/windows.yml+19 -15
@@ -37,6 +37,7 @@ permissions: {}
env:
CURL_CI: github
+ CURL_TEST_MIN: 1700
jobs:
cygwin:
@@ -197,26 +198,27 @@ jobs:
matrix:
include:
# MSYS
- - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '' , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-proxy', install: 'openssl-devel libssh2-devel', name: '!proxy' }
- - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver', install: 'openssl-devel libssh2-devel', name: 'default' }
- - { build: 'cmake' , sys: 'msys' , env: 'x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DENABLE_THREADED_RESOLVER=OFF', install: 'openssl-devel libssh2-devel', name: 'default' }
- - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '' , config: '--with-openssl', install: 'openssl-devel libssh2-devel', name: 'default R' }
+ - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '--min=1550', config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-proxy', install: 'openssl-devel libssh2-devel', name: '!proxy' }
+ - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver', install: 'openssl-devel libssh2-devel', name: 'default' }
+ - { build: 'cmake' , sys: 'msys' , env: 'x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DENABLE_THREADED_RESOLVER=OFF', install: 'openssl-devel libssh2-devel', name: 'default' }
+ - { build: 'autotools', sys: 'msys' , env: 'x86_64' , tflags: '' , config: '--with-openssl', install: 'openssl-devel libssh2-devel', name: 'default R' }
# MinGW
- - { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-curldebug --enable-static=no --without-zlib CPPFLAGS=-D_WIN32_WINNT=0x0501', install: 'mingw-w64-x86_64-openssl mingw-w64-x86_64-libssh2', name: 'default XP' }
- - { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: '' , config: '--enable-debug --with-openssl --enable-windows-unicode --enable-ares --with-openssl-quic --enable-shared=no', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-openssl mingw-w64-x86_64-nghttp3 mingw-w64-x86_64-libssh2', name: 'c-ares U' }
- - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel c-ares U' }
- - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '-t --shallow=13 !FTP', config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel U torture' }
+ - { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '--enable-debug --with-openssl --disable-threaded-resolver --disable-curldebug --enable-static=no --without-zlib CPPFLAGS=-D_WIN32_WINNT=0x0501', install: 'mingw-w64-x86_64-openssl mingw-w64-x86_64-libssh2', name: 'default XP' }
+ - { build: 'autotools', sys: 'mingw64' , env: 'x86_64' , tflags: '' , config: '--enable-debug --with-openssl --enable-windows-unicode --enable-ares --with-openssl-quic --enable-shared=no', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-openssl mingw-w64-x86_64-nghttp3 mingw-w64-x86_64-libssh2', name: 'c-ares U' }
+ - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '--min=1650', config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel c-ares U' }
+ # MinGW torture
+ - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: '-t --shallow=13 !FTP --min=1300', config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_ARES=ON', install: 'mingw-w64-x86_64-c-ares mingw-w64-x86_64-libssh2', type: 'Debug', name: 'schannel U torture' }
# WARNING: libssh uses hard-coded world-writable paths (/etc/..., ~/.ssh/) to
# read its configuration from, making it vulnerable to attacks on
# Windows. Do not use this component till there is a fix for these.
# https://github.com/curl/curl-for-win/blob/3951808deb04df9489ee17430f236ed54436f81a/libssh.sh#L6-L8
- - { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_GNUTLS=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON', install: 'mingw-w64-clang-x86_64-gnutls mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh', type: 'Debug', name: 'gnutls libssh' }
- - { build: 'cmake' , sys: 'clangarm64', env: 'clang-aarch64', tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_CURLDEBUG=ON', install: 'mingw-w64-clang-aarch64-libssh2', type: 'Release', name: 'schannel R TrackMemory', image: 'windows-11-arm' }
- - { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_OPENSSL=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON', install: 'mingw-w64-clang-x86_64-openssl mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh2', type: 'Release', name: 'openssl', chkprefill: '_chkprefill' }
- - { build: 'cmake' , sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_OPENSSL=ON', install: 'mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' }
- # { build: 'autotools', sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '--without-debug --with-schannel --enable-shared', install: 'mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' }
- - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DCMAKE_VERBOSE_MAKEFILE=ON', install: 'mingw-w64-x86_64-libssh2', type: 'Debug', cflags: '-DCURL_SCHANNEL_DEV_DEBUG', name: 'schannel dev debug', image: 'windows-2025' }
- - { build: 'cmake' , sys: 'mingw32' , env: 'i686' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', install: 'mingw-w64-i686-libssh2', type: 'Release', name: 'schannel R' }
+ - { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: '' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_GNUTLS=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON', install: 'mingw-w64-clang-x86_64-gnutls mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh', type: 'Debug', name: 'gnutls libssh' }
+ - { build: 'cmake' , sys: 'clangarm64', env: 'clang-aarch64', tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DENABLE_CURLDEBUG=ON', install: 'mingw-w64-clang-aarch64-libssh2', type: 'Release', name: 'schannel R TrackMemory', image: 'windows-11-arm' }
+ - { build: 'cmake' , sys: 'clang64' , env: 'clang-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_OPENSSL=ON -DENABLE_UNICODE=OFF -DUSE_NGTCP2=ON', install: 'mingw-w64-clang-x86_64-openssl mingw-w64-clang-x86_64-nghttp3 mingw-w64-clang-x86_64-ngtcp2 mingw-w64-clang-x86_64-libssh2', type: 'Release', name: 'openssl', chkprefill: '_chkprefill' }
+ - { build: 'cmake' , sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_OPENSSL=ON', install: 'mingw-w64-ucrt-x86_64-openssl mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' }
+ # { build: 'autotools', sys: 'ucrt64' , env: 'ucrt-x86_64' , tflags: 'skiprun' , config: '--without-debug --with-schannel --enable-shared', install: 'mingw-w64-ucrt-x86_64-libssh2', type: 'Release', test: 'uwp', name: 'schannel' }
+ - { build: 'cmake' , sys: 'mingw64' , env: 'x86_64' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON -DCMAKE_VERBOSE_MAKEFILE=ON', install: 'mingw-w64-x86_64-libssh2', type: 'Debug', cflags: '-DCURL_SCHANNEL_DEV_DEBUG', name: 'schannel dev debug', image: 'windows-2025' }
+ - { build: 'cmake' , sys: 'mingw32' , env: 'i686' , tflags: 'skiprun' , config: '-DENABLE_DEBUG=OFF -DBUILD_SHARED_LIBS=ON -DCURL_USE_SCHANNEL=ON -DENABLE_UNICODE=ON', install: 'mingw-w64-i686-libssh2', type: 'Release', name: 'schannel R' }
fail-fast: false
steps:
- uses: msys2/setup-msys2@fb197b72ce45fb24f17bf3f807a388985654d1f2 # v2.29.0
@@ -415,6 +417,7 @@ jobs:
run:
shell: msys2 {0}
env:
+ CURL_TEST_MIN: 1550
MAKEFLAGS: -j 5
MATRIX_DIR: '${{ matrix.dir }}'
strategy:
@@ -775,6 +778,7 @@ jobs:
type: 'Debug'
image: 'windows-11-arm'
openssh: 'OpenSSH-Windows'
+ tflags: '--min=1650'
config: >-
-DENABLE_DEBUG=ON
-DCURL_USE_SCHANNEL=ON
docs/runtests.md+6 -1
@@ -182,7 +182,12 @@ Lists all test case names.
## `-m=[seconds]`
-Set timeout for curl commands in tests
+Set timeout for curl commands in tests.
+
+## `--min=[count]`
+
+Set the minimum number of tests to run. This overrides the `CURL_TEST_MIN`
+environment variable, if any.
## `-n`
tests/globalconfig.pm+2 -0
@@ -50,6 +50,7 @@ BEGIN {
$LOCKDIR
$LOGDIR
$maxtime
+ $mintotal
$memanalyze
$MEMDUMP
$perlcmd
@@ -104,6 +105,7 @@ our $CURLVERSION=""; # curl's reported version number
our $CURLVERNUM=""; # curl's reported version number (without -DEV)
our $randseed = 0; # random number seed
our $maxtime; # curl command timeout override
+our $mintotal; # minimum number of tests to run
# paths
our $pwd = getcwd(); # current working directory
tests/runtests.pl+18 -0
@@ -2403,6 +2403,10 @@ while(@ARGV) {
my ($num)=($1);
$maxtime=$num;
}
+ elsif($ARGV[0] =~ /--min=(\d+)/) {
+ my ($num)=($1);
+ $mintotal=$num;
+ }
elsif($ARGV[0] eq "-n") {
# no valgrind
undef $valgrind;
@@ -2546,6 +2550,7 @@ Usage: runtests.pl [options] [test selection(s)]
-L path require an additional perl library file to replace certain functions
-l list all test case names/descriptions
-m=[seconds] set timeout for curl commands in tests
+ --min=[count] minimum number of tests to run.
-n no valgrind
--no-debuginfod disable the valgrind debuginfod functionality
-o variable=value set internal variable to the specified value
@@ -3332,6 +3337,19 @@ else {
}
}
+if(!$mintotal && $ENV{"CURL_TEST_MIN"}) {
+ $mintotal = $ENV{"CURL_TEST_MIN"};
+}
+if($mintotal) {
+ if($total < $mintotal) {
+ logmsg "TESTFAIL: number of tests run was below the minimum of: $mintotal\n";
+ exit 1;
+ }
+ else {
+ logmsg "TESTDONE: minimum number of tests was met: $mintotal\n";
+ }
+}
+
if(($total && (($ok+$ign) != $total)) || !$total || $unexpected) {
exit 1;
}