5cefb455d4 — curl

Subject

quic: improve UDP GRO receives

Binary

No binary (build failed)

Architecture & Compiler Flags

Architecture: x86_64-linux-gnu Compiler: gcc (Ubuntu 11.4.0-1ubuntu1~22.04.3) 11.4.0

—

Build Log

(log yok)

Commit Diff GitHub →

lib/vquic/curl_ngtcp2.c +25 -18 lib/vquic/curl_quiche.c +40 -34 lib/vquic/vquic.c +26 -43 lib/vquic/vquic_int.h +6 -5

lib/vquic/curl_ngtcp2.c+25 -18

@@ -1731,37 +1731,43 @@ denied:
	return result;
	}

-	static CURLcode recv_pkt(const unsigned char *pkt, size_t pktlen,
-	struct sockaddr_storage *remote_addr,
-	socklen_t remote_addrlen, int ecn,
-	void *userp)
+	static CURLcode cf_ngtcp2_recv_pkts(const unsigned char *buf, size_t buflen,
+	size_t gso_size,
+	struct sockaddr_storage *remote_addr,
+	socklen_t remote_addrlen, int ecn,
+	void *userp)
	{
	struct pkt_io_ctx *pktx = userp;
	struct cf_ngtcp2_ctx *ctx = pktx->cf->ctx;
	ngtcp2_pkt_info pi;
	ngtcp2_path path;
+	size_t offset, pktlen;
	int rv;

	if(ecn)
-	CURL_TRC_CF(pktx->data, pktx->cf, "vquic_recv(len=%zu, ecn=%x)",
-	pktlen, ecn);
+	CURL_TRC_CF(pktx->data, pktx->cf, "vquic_recv(len=%zu, gso=%zu, ecn=%x)",
+	buflen, gso_size, ecn);
	ngtcp2_addr_init(&path.local, (struct sockaddr *)&ctx->q.local_addr,
	(socklen_t)ctx->q.local_addrlen);
	ngtcp2_addr_init(&path.remote, (struct sockaddr *)remote_addr,
	remote_addrlen);
	pi.ecn = (uint8_t)ecn;

-	rv = ngtcp2_conn_read_pkt(ctx->qconn, &path, &pi, pkt, pktlen, pktx->ts);
-	if(rv) {
-	CURL_TRC_CF(pktx->data, pktx->cf, "ingress, read_pkt -> %s (%d)",
-	ngtcp2_strerror(rv), rv);
-	cf_ngtcp2_err_set(pktx->cf, pktx->data, rv);
-
-	if(rv == NGTCP2_ERR_CRYPTO)
-	/* this is a "TLS problem", but a failed certificate verification
-	is a common reason for this */
-	return CURLE_PEER_FAILED_VERIFICATION;
-	return CURLE_RECV_ERROR;
+	for(offset = 0; offset < buflen; offset += gso_size) {
+	pktlen = ((offset + gso_size) <= buflen) ? gso_size : (buflen - offset);
+	rv = ngtcp2_conn_read_pkt(ctx->qconn, &path, &pi,
+	buf + offset, pktlen, pktx->ts);
+	if(rv) {
+	CURL_TRC_CF(pktx->data, pktx->cf, "ingress, read_pkt -> %s (%d)",
+	ngtcp2_strerror(rv), rv);
+	cf_ngtcp2_err_set(pktx->cf, pktx->data, rv);
+
+	if(rv == NGTCP2_ERR_CRYPTO)
+	/* this is a "TLS problem", but a failed certificate verification
+	is a common reason for this */
+	return CURLE_PEER_FAILED_VERIFICATION;
+	return CURLE_RECV_ERROR;
+	}
	}
	return CURLE_OK;
	}
@@ -1787,7 +1793,8 @@ static CURLcode cf_progress_ingress(struct Curl_cfilter *cf,
	if(result)
	return result;

-	return vquic_recv_packets(cf, data, &ctx->q, 1000, recv_pkt, pktx);
+	return vquic_recv_packets(cf, data, &ctx->q, 1000,
+	cf_ngtcp2_recv_pkts, pktx);
	}

	/**

lib/vquic/curl_quiche.c+40 -34

@@ -625,58 +625,63 @@ struct recv_ctx {
	int pkts;
	};

-	static CURLcode recv_pkt(const unsigned char *pkt, size_t pktlen,
-	struct sockaddr_storage *remote_addr,
-	socklen_t remote_addrlen, int ecn,
-	void *userp)
+	static CURLcode cf_quiche_recv_pkts(const unsigned char *buf, size_t buflen,
+	size_t gso_size,
+	struct sockaddr_storage *remote_addr,
+	socklen_t remote_addrlen, int ecn,
+	void *userp)
	{
	struct recv_ctx *r = userp;
	struct cf_quiche_ctx *ctx = r->cf->ctx;
	quiche_recv_info recv_info;
+	size_t pktlen, offset;
	ssize_t nread;

	(void)ecn;
-	++r->pkts;

	recv_info.to = (struct sockaddr *)&ctx->q.local_addr;
	recv_info.to_len = ctx->q.local_addrlen;
	recv_info.from = (struct sockaddr *)remote_addr;
	recv_info.from_len = remote_addrlen;

-	nread = quiche_conn_recv(ctx->qconn,
-	(unsigned char *)CURL_UNCONST(pkt), pktlen,
-	&recv_info);
-	if(nread < 0) {
-	if(QUICHE_ERR_DONE == nread) {
-	if(quiche_conn_is_draining(ctx->qconn)) {
-	CURL_TRC_CF(r->data, r->cf, "ingress, connection is draining");
-	return CURLE_RECV_ERROR;
+	for(offset = 0; offset < buflen; offset += gso_size) {
+	pktlen = ((offset + gso_size) <= buflen) ? gso_size : (buflen - offset);
+	nread = quiche_conn_recv(ctx->qconn,
+	(unsigned char *)CURL_UNCONST(buf + offset),
+	pktlen, &recv_info);
+	if(nread < 0) {
+	if(QUICHE_ERR_DONE == nread) {
+	if(quiche_conn_is_draining(ctx->qconn)) {
+	CURL_TRC_CF(r->data, r->cf, "ingress, connection is draining");
+	return CURLE_RECV_ERROR;
+	}
+	if(quiche_conn_is_closed(ctx->qconn)) {
+	CURL_TRC_CF(r->data, r->cf, "ingress, connection is closed");
+	return CURLE_RECV_ERROR;
+	}
+	CURL_TRC_CF(r->data, r->cf, "ingress, quiche is DONE");
+	return CURLE_OK;
	}
-	if(quiche_conn_is_closed(ctx->qconn)) {
-	CURL_TRC_CF(r->data, r->cf, "ingress, connection is closed");
+	else if(QUICHE_ERR_TLS_FAIL == nread) {
+	long verify_ok = SSL_get_verify_result(ctx->tls.ossl.ssl);
+	if(verify_ok != X509_V_OK) {
+	failf(r->data, "SSL certificate problem: %s",
+	X509_verify_cert_error_string(verify_ok));
+	return CURLE_PEER_FAILED_VERIFICATION;
+	}
+	failf(r->data, "ingress, quiche reports TLS fail");
	return CURLE_RECV_ERROR;
	}
-	CURL_TRC_CF(r->data, r->cf, "ingress, quiche is DONE");
-	return CURLE_OK;
-	}
-	else if(QUICHE_ERR_TLS_FAIL == nread) {
-	long verify_ok = SSL_get_verify_result(ctx->tls.ossl.ssl);
-	if(verify_ok != X509_V_OK) {
-	failf(r->data, "SSL certificate problem: %s",
-	X509_verify_cert_error_string(verify_ok));
-	return CURLE_PEER_FAILED_VERIFICATION;
+	else {
+	failf(r->data, "quiche reports error %zd on receive", nread);
+	return CURLE_RECV_ERROR;
	}
-	failf(r->data, "ingress, quiche reports TLS fail");
-	return CURLE_RECV_ERROR;
	}
-	else {
-	failf(r->data, "quiche reports error %zd on receive", nread);
-	return CURLE_RECV_ERROR;
+	else if((size_t)nread < pktlen) {
+	CURL_TRC_CF(r->data, r->cf, "ingress, quiche only read %zd/%zu bytes",
+	nread, pktlen);
	}
-	}
-	else if((size_t)nread < pktlen) {
-	CURL_TRC_CF(r->data, r->cf, "ingress, quiche only read %zd/%zu bytes",
-	nread, pktlen);
+	++r->pkts;
	}

	return CURLE_OK;
@@ -698,7 +703,8 @@ static CURLcode cf_process_ingress(struct Curl_cfilter *cf,
	rctx.data = data;
	rctx.pkts = 0;

-	result = vquic_recv_packets(cf, data, &ctx->q, 1000, recv_pkt, &rctx);
+	result = vquic_recv_packets(cf, data, &ctx->q, 1000,
+	cf_quiche_recv_pkts, &rctx);
	if(result)
	return result;

lib/vquic/vquic.c+26 -43

@@ -378,9 +378,16 @@ static CURLcode recvmmsg_packets(struct Curl_cfilter *cf,
	struct Curl_easy *data,
	struct cf_quic_ctx *qctx,
	size_t max_pkts,
-	vquic_recv_pkt_cb recv_cb, void userp)
+	vquic_recv_pkts_cb recv_cb, void userp)
	{
+	#if defined(__linux__) && defined(UDP_GRO)
	#define MMSG_NUM 16
+	#define UDP_GRO_CNT_MAX 64
+	#else
+	#define MMSG_NUM 64
+	#define UDP_GRO_CNT_MAX 1
+	#endif
+	#define MSG_BUF_SIZE (UDP_GRO_CNT_MAX * 1500)
	struct iovec msg_iov[MMSG_NUM];
	struct mmsghdr mmsg[MMSG_NUM];
	uint8_t msg_ctrl[MMSG_NUM * CMSG_SPACE(sizeof(int))];
@@ -390,17 +397,15 @@ static CURLcode recvmmsg_packets(struct Curl_cfilter *cf,
	char errstr[STRERROR_LEN];
	CURLcode result = CURLE_OK;
	size_t gso_size;
-	size_t pktlen;
-	size_t offset, to;
	char *sockbuf = NULL;
-	uint8_t (bufs)[641024] = NULL;
+	uint8_t (*bufs)[MSG_BUF_SIZE] = NULL;

	DEBUGASSERT(max_pkts > 0);
-	result = Curl_multi_xfer_sockbuf_borrow(data, MMSG_NUM * sizeof(bufs[0]),
+	result = Curl_multi_xfer_sockbuf_borrow(data, MMSG_NUM * MSG_BUF_SIZE,
	&sockbuf);
	if(result)
	goto out;
-	bufs = (uint8_t ()[641024])sockbuf;
+	bufs = (uint8_t (*)[MSG_BUF_SIZE])sockbuf;

	total_nread = 0;
	while(pkts < max_pkts) {
@@ -449,22 +454,12 @@ static CURLcode recvmmsg_packets(struct Curl_cfilter *cf,
	gso_size = mmsg[i].msg_len;
	}

-	for(offset = 0; offset < mmsg[i].msg_len; offset = to) {
-	++pkts;
-
-	to = offset + gso_size;
-	if(to > mmsg[i].msg_len) {
-	pktlen = mmsg[i].msg_len - offset;
-	}
-	else {
-	pktlen = gso_size;
-	}
-
-	result = recv_cb(bufs[i] + offset, pktlen, mmsg[i].msg_hdr.msg_name,
-	mmsg[i].msg_hdr.msg_namelen, 0, userp);
-	if(result)
-	goto out;
-	}
+	result = recv_cb(bufs[i], mmsg[i].msg_len, gso_size,
+	mmsg[i].msg_hdr.msg_name,
+	mmsg[i].msg_hdr.msg_namelen, 0, userp);
+	if(result)
+	goto out;
+	pkts += (mmsg[i].msg_len + gso_size - 1) / gso_size;
	}
	}

@@ -481,7 +476,7 @@ static CURLcode recvmsg_packets(struct Curl_cfilter *cf,
	struct Curl_easy *data,
	struct cf_quic_ctx *qctx,
	size_t max_pkts,
-	vquic_recv_pkt_cb recv_cb, void userp)
+	vquic_recv_pkts_cb recv_cb, void userp)
	{
	struct iovec msg_iov;
	struct msghdr msg;
@@ -494,8 +489,6 @@ static CURLcode recvmsg_packets(struct Curl_cfilter *cf,
	CURLcode result = CURLE_OK;
	uint8_t msg_ctrl[CMSG_SPACE(sizeof(int))];
	size_t gso_size;
-	size_t pktlen;
-	size_t offset, to;

	DEBUGASSERT(max_pkts > 0);
	for(pkts = 0, total_nread = 0, calls = 0; pkts < max_pkts;) {
@@ -542,20 +535,10 @@ static CURLcode recvmsg_packets(struct Curl_cfilter *cf,
	gso_size = nread;
	}

-	for(offset = 0; offset < nread; offset = to) {
-	++pkts;
-
-	to = offset + gso_size;
-	if(to > nread)
-	pktlen = nread - offset;
-	else
-	pktlen = gso_size;
-
-	result =
-	recv_cb(buf + offset, pktlen, msg.msg_name, msg.msg_namelen, 0, userp);
-	if(result)
-	goto out;
-	}
+	result = recv_cb(buf, nread, gso_size,
+	msg.msg_name, msg.msg_namelen, 0, userp);
+	if(result)
+	goto out;
	}

	out:
@@ -570,7 +553,7 @@ static CURLcode recvfrom_packets(struct Curl_cfilter *cf,
	struct Curl_easy *data,
	struct cf_quic_ctx *qctx,
	size_t max_pkts,
-	vquic_recv_pkt_cb recv_cb, void userp)
+	vquic_recv_pkts_cb recv_cb, void userp)
	{
	uint8_t buf[64*1024];
	int bufsize = (int)sizeof(buf);
@@ -611,8 +594,8 @@ static CURLcode recvfrom_packets(struct Curl_cfilter *cf,
	++pkts;
	++calls;
	total_nread += (size_t)nread;
-	result = recv_cb(buf, (size_t)nread, &remote_addr, remote_addrlen,
-	0, userp);
+	result = recv_cb(buf, (size_t)nread, (size_t)nread,
+	&remote_addr, remote_addrlen, 0, userp);
	if(result)
	goto out;
	}
@@ -629,7 +612,7 @@ CURLcode vquic_recv_packets(struct Curl_cfilter *cf,
	struct Curl_easy *data,
	struct cf_quic_ctx *qctx,
	size_t max_pkts,
-	vquic_recv_pkt_cb recv_cb, void userp)
+	vquic_recv_pkts_cb recv_cb, void userp)
	{
	CURLcode result;
	#ifdef HAVE_SENDMMSG

lib/vquic/vquic_int.h+6 -5

@@ -78,16 +78,17 @@ CURLcode vquic_flush(struct Curl_cfilter cf, struct Curl_easy data,
	struct cf_quic_ctx *qctx);


-	typedef CURLcode vquic_recv_pkt_cb(const unsigned char *pkt, size_t pktlen,
-	struct sockaddr_storage *remote_addr,
-	socklen_t remote_addrlen, int ecn,
-	void *userp);
+	typedef CURLcode vquic_recv_pkts_cb(const unsigned char *buf, size_t buflen,
+	size_t gso_size,
+	struct sockaddr_storage *remote_addr,
+	socklen_t remote_addrlen, int ecn,
+	void *userp);

	CURLcode vquic_recv_packets(struct Curl_cfilter *cf,
	struct Curl_easy *data,
	struct cf_quic_ctx *qctx,
	size_t max_pkts,
-	vquic_recv_pkt_cb recv_cb, void userp);
+	vquic_recv_pkts_cb recv_cb, void userp);

	#endif /* !USE_HTTP3 */