linux走内核协议栈发包极限测试

组网：

服务器(server)--------交换机--------服务器(client)

两套设备：

A：Dell服务器（Intel Xeon Gold 6354 CPU）+solaflare 10G网卡+思科10G交换机，这套约40w

B：树莓派4B（2台，主板集成1G网卡，8G内存，约600元一台）+ h3c家用千兆小交换机（约150元），这套约1500元

测试前，保证OS干干净净，无耗cpu的任务

场景一. A通过10G网卡交互测试，测试收包极限

启动收包server：

启动发包client（本场景是测试收包极限，所以用两台服务器分别起1个client）:

查看收包服务器的网卡丢包情况，已出现RX-DRP，这个说明内核CPU消化不了网卡送到DMA的报文：

查看下收包服务器server的cpu使用情况，core 0的soft占用100%就是内核协议栈部分，说明这里有瓶颈，跟上图是对的上的；

上图core 2是用户态收包线程，cpu还有少量空闲，查看下用户态线程的切换情况，这个应该是read、epoll_wait持续多次没数据触发了内核context switch；

再看看 socket缓冲区是否有满的情况，可以看出还是有少量满的情况出现，这里因为是只有少量的出现满的情况，通过调大socket缓冲区就可以解决；

如果是大量满那就是cpu不够用了，就要多线程通过so_reuseport同时来收包；

结论：上面是服务端已经出现丢包的情况了，用户进程约 624 000us收 1000 000个1024B大小的包，差不多可以认为1个包走内核协议栈上送到socket缓冲区约600ns+

场景二. A通过1G网卡交互测试，测试收包极限

common.c

#include <sys/time.h>
#include <time.h>
#include <stdlib.h>
#include <stdio.h>
#include "common.h"

//typedef unsigned long long uint64_t;

long long getustime(void)
{
    struct timeval tv;
    long long ust;

    gettimeofday(&tv, NULL);
    ust = ((long)tv.tv_sec)*1000000;
    ust += tv.tv_usec;
    return ust;
}

void getnstime(struct timespec *pstTP)
{
    clock_gettime(CLOCK_MONOTONIC, pstTP);
    return;
}

long getdiff_nstime(struct timespec *pstTP1, struct timespec *pstTP2)
{
    return (pstTP2->tv_sec - pstTP1->tv_sec)*1000000000 + pstTP2->tv_nsec - pstTP1->tv_nsec;
}


void outMembuf(void * buf, unsigned int uiLen)
{
    int i, count = 0;
	printf("buf len:%u\n",uiLen);
	for(i=0; i < uiLen; i++)
	{
		if (0 ==(count%16))
			printf("0x");
		printf("%02X", *(unsigned char *)(buf+i));
		count++;

        if (0 ==(count%8))
			printf("  ");	
		
		if (0 ==(count%16))
			printf("\n");	

	
	}
}

common.h

#ifndef COMMON_H_
#define COMMON_H_


#define ERROR_SUCCESS 0
#define ERROR_FAIL    1
#define IN
#define OUT
#define INOUT

#define CACHE_LINE   64

//#define HASHSIZE 1*1024

typedef unsigned long ulong_t;
long long getustime(void);
void getnstime(struct timespec *pstTP);
long getdiff_nstime(struct timespec *pstTP1, struct timespec *pstTP2);
void outMembuf(void * buf, unsigned int uiLen);

#endif

udp_server.c

#include <stdio.h>        // for printf() and fprintf()
#include <sys/socket.h>        // for socket(), bind(), and connect()
#include <arpa/inet.h>        // for sockaddr_in and inet_ntoa()
#include <stdlib.h>        // for atoi() and exit()
#include <string.h>        // for memset()
#include <unistd.h>        // for close()
#include <fcntl.h>        // for fcntl()
#include <errno.h>

#include <sys/epoll.h>

#define BUFSIZE 4096       // Size of receive buffer
#define STEPCOUNT 1000000
static int make_socket_non_blocking(int sfd)
{
    int flags, s;

    flags = fcntl(sfd, F_GETFL, 0);

    if (flags == -1) {
        perror("fcntl");
        return -1;
    }

    flags |= O_NONBLOCK;
    s = fcntl(sfd, F_SETFL, flags);

    if (s == -1) {
        perror("fcntl");
        return -1;
    }

    return 0;
}

int main(int argc, char *argv[])
{
    int sockfd;        // Socket descriptors for server
    int broadcast = 1;    // Socket Option.
    struct sockaddr_in srvaddr;    // Broadcast Server Address
    struct sockaddr_in cliaddr;    // Broadcast Response Client Address
    int cliaddr_len = sizeof(cliaddr);
    unsigned long uc_epollwait = 0, uc_rf = 0;
    long long t1,t2,diff;
    struct timespec stTP1,stTP2;
    char buffer[BUFSIZE];    // Input and Receive buffer
    int i;            // For loop use
    int running = 1;    // Main Loop
    unsigned long ulCount = 0;
    int rcvlen;
    int avgrcvlen = 0;
    int index = 0;

    int epfd;        // EPOLL File Descriptor.
    struct epoll_event ev;    // Used for EPOLL.
    struct epoll_event events[5];    // Used for EPOLL.
    int noEvents;        // EPOLL event number.

    printf("This process is a UDP process!.\n");


    if ((sockfd = socket(AF_INET, SOCK_DGRAM, 0)) == -1) 
    {
        perror("Create Sockfd Fail!!\n");
        exit(1);
    }

    make_socket_non_blocking(sockfd);

    // Reset the addresses
    memset(&srvaddr, 0, sizeof(srvaddr));

    // Set the addresses
    srvaddr.sin_family = AF_INET;
    srvaddr.sin_port = htons(atoi(argv[1]));
    srvaddr.sin_addr.s_addr = INADDR_ANY;

    if (bind(sockfd, (struct sockaddr *)&srvaddr, sizeof(srvaddr)) == -1) 
    {
        perror("bind");
        exit(1);
    }

    epfd = epoll_create(5);
    ev.data.fd = sockfd;
    ev.events = EPOLLIN | EPOLLET;
    epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev);

    // Add STDIN into the EPOLL set.
    ev.data.fd = STDIN_FILENO;
    ev.events = EPOLLIN | EPOLLET;
    epoll_ctl(epfd, EPOLL_CTL_ADD, STDIN_FILENO, &ev);
    getnstime(&stTP1);
    t1 = getustime();
    //getnstime(&stTP1);
    while (running) 
    {
        // Wait for events.
        noEvents = epoll_wait(epfd, events, FD_SETSIZE, -1);
        uc_epollwait ++;
        for (i = 0; i < noEvents; i++) 
        {
            if ((events[i].events & EPOLLIN) && (sockfd == events[i].data.fd))
            {
                while(1)
                {
                    //rcvlen = recvfrom(sockfd, buffer, BUFSIZE, 0, (struct sockaddr *)&cliaddr, (socklen_t *) & cliaddr_len);
                    rcvlen = read(events[i].data.fd, buffer, BUFSIZE);

                    uc_rf ++;
                    if (rcvlen <= 0)
                        break;
                    avgrcvlen += rcvlen;
                    ulCount ++;
                                

                    if (STEPCOUNT == ulCount)
                    {
                        avgrcvlen = avgrcvlen/STEPCOUNT;
                        ulCount = 0;

                        t2 = getustime();
                        diff = t2 - t1;

                        printf("%lu:recv avg len: %d, rcv %u pkt diff :%lluus, epollwait count:%lu, read count:%lu\n", index, avgrcvlen, STEPCOUNT, diff, uc_epollwait, uc_rf);

                        uc_rf = 0;
                        uc_epollwait = 0;
                        index ++;

                        t1 = getustime();
                    } 
                }
                          

                if (errno != EAGAIN)
                {
                    printf("Receive the brocast message response FAIL!!\n");
                    running = 0;
                }
            }
        }
    }

    close(sockfd);
    close(epfd);

    return 0;
}

udp_client.c

#include <sys/types.h>
#include <sys/socket.h>
#include <string.h>
#include <netinet/in.h>
#include <stdio.h>
#include <stdlib.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <errno.h>
#include <sys/time.h>

#include "common.h"

#define STEPCOUNT 1000000

int main(int argc, char **argv)
{
    int sockfd;
    struct sockaddr_in servaddr;
    char sendline[2048];
    unsigned long ulCount = 0;
    int ret;
    long long t1,t2,diff,maxdiff = 0;
    int iSendLen;


    /* init servaddr */
    bzero(&servaddr, sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_port = htons(atoi(argv[2]));
    servaddr.sin_addr.s_addr = inet_addr(argv[1]);
    iSendLen = atoi(argv[3]);
    sockfd = socket(AF_INET, SOCK_DGRAM, 0);

    if(connect(sockfd, (struct sockaddr *)&servaddr, sizeof(servaddr)) == -1)
    {
        printf("connect error\n");
        exit(1);
    }

    int c = 0;
    while(1)
    {
        ret = write(sockfd, sendline, iSendLen); 
        //ret = sendto(sockfd, sendline,20, 0, (struct sockaddr *)&servaddr,sizeof(servaddr));
        if (ret <0)
            printf("send error %d, %d, %s\n", ret, errno, strerror(errno));
        ulCount ++;
        if (STEPCOUNT == ulCount)
        {
        	ulCount = 0;
            t2 = getustime();
            diff = t2-t1;

            printf("send count %lu diff %lluus, pkt len:%d\n", STEPCOUNT, diff, iSendLen);
            t1 = getustime();
        }

    }

    return 0;
}

Tags：

转载：感谢比目鱼技术博客文章的认可，非常欢迎各位朋友分享，转载请说明文章出处“比目鱼技术博客bi2.com.cn”。http://www.bi2.com.cn/biancheng/53.html

很赞哦！ ()