← Commit list

e7247d8697

Jay Satiro
Jay Satiro
2025-10-09
SUCCESS Duration: 156s

Subject

tool_operate: keep failed partial download for retry auto-resume

Binary

No binary (build failed)

Architecture & Compiler Flags

Architecture: x86_64-linux-gnu Compiler: gcc (Ubuntu 11.4.0-1ubuntu1~22.04.3) 11.4.0

—

Build Log

(log yok)

Commit Diff GitHub →

src/tool_operate.c +72 -2tests/data/Makefile.am +1 -1tests/data/test3035 +127 -0
src/tool_operate.c+72 -2
@@ -336,6 +336,32 @@ void single_transfer_cleanup(void)
glob_cleanup(&state->inglob);
}
+/* Helper function for retrycheck.
+ *
+ * This function is a prerequisite check used to determine whether or not some
+ * already downloaded data (ie out->bytes written) can be safely resumed in a
+ * subsequent transfer. The conditions are somewhat pedantic to avoid any risk
+ * of data corruption.
+ *
+ * Specific HTTP limitations (scheme, method, response code, etc) are checked
+ * in retrycheck if this prerequisite check is met.
+ */
+static bool is_outfile_auto_resumable(struct OperationConfig *config,
+ struct per_transfer *per,
+ CURLcode result)
+{
+ struct OutStruct *outs = &per->outs;
+ return config->use_resume && config->resume_from_current &&
+ config->resume_from >= 0 && outs->init == config->resume_from &&
+ outs->bytes > 0 && outs->filename && outs->s_isreg && outs->fopened &&
+ outs->stream && !ferror(outs->stream) &&
+ !config->customrequest && !per->uploadfile &&
+ (config->httpreq == TOOL_HTTPREQ_UNSPEC ||
+ config->httpreq == TOOL_HTTPREQ_GET) &&
+ /* CURLE_WRITE_ERROR could mean outs->bytes is not accurate */
+ result != CURLE_WRITE_ERROR && result != CURLE_RANGE_ERROR;
+}
+
static CURLcode retrycheck(struct OperationConfig *config,
struct per_transfer *per,
CURLcode result,
@@ -353,7 +379,6 @@ static CURLcode retrycheck(struct OperationConfig *config,
RETRY_FTP,
RETRY_LAST /* not used */
} retry = RETRY_NO;
- long response = 0;
if((CURLE_OPERATION_TIMEDOUT == result) ||
(CURLE_COULDNT_RESOLVE_HOST == result) ||
(CURLE_COULDNT_RESOLVE_PROXY == result) ||
@@ -378,6 +403,7 @@ static CURLcode retrycheck(struct OperationConfig *config,
scheme = proto_token(scheme);
if(scheme == proto_http || scheme == proto_https) {
/* This was HTTP(S) */
+ long response = 0;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
switch(response) {
@@ -404,6 +430,7 @@ static CURLcode retrycheck(struct OperationConfig *config,
} /* if CURLE_OK */
else if(result) {
const char *scheme;
+ long response = 0;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
curl_easy_getinfo(curl, CURLINFO_SCHEME, &scheme);
@@ -432,6 +459,7 @@ static CURLcode retrycheck(struct OperationConfig *config,
": HTTP error",
": FTP error"
};
+ bool truncate = TRUE; /* truncate output file */
if(RETRY_HTTP == retry) {
curl_easy_getinfo(curl, CURLINFO_RETRY_AFTER, &retry_after);
@@ -482,7 +510,49 @@ static CURLcode retrycheck(struct OperationConfig *config,
per->retry_remaining--;
- if(outs->bytes && outs->filename && outs->stream) {
+ /* Skip truncation of outfile if auto-resume is enabled for download and
+ the partially received data is good. Only for HTTP GET requests in
+ limited circumstances. */
+ if(is_outfile_auto_resumable(config, per, result)) {
+ long response = 0;
+ struct curl_header *header = NULL;
+ const char *method = NULL, *scheme = NULL;
+
+ curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_METHOD, &method);
+ curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
+ curl_easy_getinfo(curl, CURLINFO_SCHEME, &scheme);
+ scheme = proto_token(scheme);
+
+ if((scheme == proto_http || scheme == proto_https) &&
+ method && !strcmp(method, "GET") &&
+ ((response == 206 && config->resume_from) ||
+ (response == 200 &&
+ !curl_easy_header(curl, "Accept-Ranges", 0,
+ CURLH_HEADER, -1, &header) &&
+ !strcmp(header->value, "bytes")))) {
+
+ notef("Keeping %" CURL_FORMAT_CURL_OFF_T " bytes", outs->bytes);
+ if(fflush(outs->stream)) {
+ errorf("Failed to flush output file stream");
+ return CURLE_WRITE_ERROR;
+ }
+ if(outs->bytes >= CURL_OFF_T_MAX - outs->init) {
+ errorf("Exceeded maximum supported file size ("
+ "%" CURL_FORMAT_CURL_OFF_T " + "
+ "%" CURL_FORMAT_CURL_OFF_T ")",
+ outs->init, outs->bytes);
+ return CURLE_WRITE_ERROR;
+ }
+ truncate = FALSE;
+ outs->init += outs->bytes;
+ outs->bytes = 0;
+ config->resume_from = outs->init;
+ curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE,
+ config->resume_from);
+ }
+ }
+
+ if(truncate && outs->bytes && outs->filename && outs->stream) {
#ifndef __MINGW32CE__
struct_stat fileinfo;
tests/data/Makefile.am+1 -1
@@ -275,7 +275,7 @@ test3000 test3001 test3002 test3003 test3004 test3005 test3006 test3007 \
test3008 test3009 test3010 test3011 test3012 test3013 test3014 test3015 \
test3016 test3017 test3018 test3019 test3020 test3021 test3022 test3023 \
test3024 test3025 test3026 test3027 test3028 test3029 test3030 test3031 \
-test3032 test3033 test3034 \
+test3032 test3033 test3034 test3035 \
\
test3100 test3101 test3102 test3103 test3104 test3105 \
\
tests/data/test3035+127 -0
@@ -0,0 +1,127 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+Content-Range
+Resume
+retry
+</keywords>
+</info>
+
+# Server-side
+<reply>
+
+#
+# the first chunk
+#
+<data nocheck="yes" nonewline="yes">
+HTTP/1.1 200 OK swsbounce swsclose
+Accept-Ranges: bytes
+Content-Type: text/html
+Content-Length: 26
+
+abcde
+</data>
+
+#
+# the second chunk
+#
+<data1 nocheck="yes" nonewline="yes">
+HTTP/1.1 206 Partial Content swsbounce swsclose
+Content-Type: text/html
+Content-Length: 21
+Content-Range: bytes 5-25/26
+
+fghijk
+</data1>
+
+#
+# some nonsense that curl should ignore as unresumable
+#
+<data2 nocheck="yes">
+HTTP/1.1 404 Not Found swsbounce
+Content-Type: text/html
+Content-Length: 5
+
+body
+</data2>
+
+#
+# some more nonsense that curl should ignore as unresumable
+#
+<data3 nocheck="yes">
+HTTP/1.1 200 OK swsbounce
+Accept-Ranges: bytes
+Content-Type: text/html
+Content-Length: 30
+
+XXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+</data3>
+
+#
+# the third chunk
+#
+<data4 nocheck="yes" nonewline="yes">
+HTTP/1.1 206 Partial Content swsbounce swsclose
+Content-Type: text/html
+Content-Length: 15
+Content-Range: bytes 11-25/26
+
+lmnopqrstuvwxyz
+</data4>
+</reply>
+
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+HTTP retry failed download with keep data and auto-resume
+</name>
+<command option="no-output,no-include">
+--continue-at - --retry 4 --retry-delay 1 --retry-all-errors -o %LOGDIR/outfile%TESTNUMBER http://%HOSTIP:%HTTPPORT/%TESTNUMBER
+</command>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+User-Agent: curl/%VERSION
+Accept: */*
+
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Range: bytes=5-
+User-Agent: curl/%VERSION
+Accept: */*
+
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Range: bytes=11-
+User-Agent: curl/%VERSION
+Accept: */*
+
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Range: bytes=11-
+User-Agent: curl/%VERSION
+Accept: */*
+
+GET /%TESTNUMBER HTTP/1.1
+Host: %HOSTIP:%HTTPPORT
+Range: bytes=11-
+User-Agent: curl/%VERSION
+Accept: */*
+
+</protocol>
+
+<file1 name="%LOGDIR/outfile%TESTNUMBER" nonewline="yes">
+abcdefghijklmnopqrstuvwxyz
+</file1>
+
+</verify>
+</testcase>