最近碰到了一个问题，设备端上挂载的nginx服务通过c版fcgi提供服务，其中的长连接API经常连接的设备断网后其他设备无法连接上。
初步查看源代码调试日志发现网络断了nginx未检测到，这时就想到给nginx添加保活检测，配置中listen加上了，写模块给connection也加上，均发现无效果，半天摸不清头脑。先写个小小的测试程序测试保活特性吧，开始我测试的是服务端只收不发，客户端只发不收，居然达到了我要的效果，短短几秒之后端口就关闭了。后面测试了收发同时跑的时候，死活都达不到效果，这就郁闷了。
listen保活检测配置:
listen 80 so_keepalive=1:1:1; //so_keepalive= tcp_keepidle:tcp_keepintvl:tcp_keepcnt经过抓包后发现这两种情况的不同之处，请看截图：
这是只收不发的抓包截图，Keep-Alive严格按照设定走。
这是收发同时跑的抓包截图，Keep-Alive一个都没有，多出了TCP Retransmission，这个是超时重传的包。后面查了下为啥保活没激活，仔细瞧了下保活注解后注意到了 （ 如果2小时内在此套接口的任一方向都没有数据交换 ） 这是重点。在后面这种情况下需要去处理超时重发，但是超时重发貌似只能调整系统设置（暂时未找到怎么通过设置socket调整超时重发参数，知道的希望告诉我下）。
改系统中的设定如下，可以达到了效果。
echo "net.ipv4.tcp_retries2=5" >> /etc/sysctl.conf sysctl -p但是改系统参数配置影响范围有点大，这个是能是没有办法之后的方法了。之后翻找了下nginx中fastcgi流程，与源码对照的看了下，发现可以这么操作：
1.清零 SO_SNDBUF，，通过listen 80 sndbuf=0; 参数修改
2.降低fastcgi缓冲区大小，避免太多次的缓冲区操作导致超时延后
3.设置send_timeout 1
ps：1,2两点为了使协议栈和用户发送缓冲能尽快的装满，走入发送超时检测流程。
具体配置如下：
listen 80 sndbuf=0; ... location ~ \.push$ { send_timeout 1; fastcgi_buffer_size 200; fastcgi_buffers 2 200; fastcgi_busy_buffers_size 200; ... }nginx结构流程图.vsdx
nginx保活模块：https://pan.baidu.com/s/1EGLmNg5S9uXZloeLCvPOFQ
测试用c版代码:
server.c
#include <stdint.h> #include <stdio.h> #include <stdlib.h> #ifdef _WIN32 #define WIN32 1 #endif #ifdef WIN32 #include <winsock2.h> #include <windows.h> #include <mstcpip.h> #define close closesocket #else #include <unistd.h> #include <sys/select.h> #include <sys/time.h> #include <sys/types.h> #include <sys/socket.h> #include <netinet/in.h> #include <netinet/tcp.h> #include <fcntl.h> #include <errno.h> typedef int SOCKET; #endif #define LOGS_FULL(OPTION,ERRNO) \ printf("(%s %d %s) %s errno:%d",__FILE__,__LINE__,__FUNCTION__,OPTION,ERRNO); #define LOGS(OPTION,ERRNO) \ printf("%s errno:%d",OPTION,ERRNO); #define ERRORS(OPTION,ERRNO) LOGS(OPTION,ERRNO) #ifdef WIN32 int initNetwork() { WSADATA wsaData; int ret = WSAStartup(MAKEWORD(2,2), &wsaData); if (ret != 0) return 1; return 0; } int setnonblocking(SOCKET sock) { unsigned long ul=1; int ret=ioctlsocket(sock,FIONBIO,(unsigned long *)&ul); if(ret==SOCKET_ERROR) { } return ret; } int socket_keepalive(SOCKET socket) { int keep_alive = 1; int ret = setsockopt(socket, SOL_SOCKET, SO_KEEPALIVE, (char*)&keep_alive, sizeof(keep_alive)); if (ret == SOCKET_ERROR) { printf("SO_KEEPALIVE failed:%d\n", WSAGetLastError()); return -1; } struct tcp_keepalive in_keep_alive = {0}; unsigned long ul_in_len = sizeof(struct tcp_keepalive); struct tcp_keepalive out_keep_alive = {0}; unsigned long ul_out_len = sizeof(struct tcp_keepalive); unsigned long ul_bytes_return = 0; in_keep_alive.onoff = 1; in_keep_alive.keepaliveinterval = 5000; in_keep_alive.keepalivetime = 1000; ret = WSAIoctl(socket, SIO_KEEPALIVE_VALS, (LPVOID)&in_keep_alive, ul_in_len, (LPVOID)&out_keep_alive, ul_out_len, &ul_bytes_return, NULL, NULL); if (ret == SOCKET_ERROR) { printf("WSAIoctl KEEPALIVE failed:%d\n", WSAGetLastError()); return -1; } return 0; } #else int setnonblocking(SOCKET sock) { int opts = fcntl(sock,F_GETFL); if (opts < 0 ) { printf( "fcntl(sock,GETFL) " ); return -1; } opts = opts | O_NONBLOCK; if (fcntl(sock,F_SETFL,opts) < 0 ) { printf( "fcntl(sock,SETFL,opts) " ); return -1; } return 0; } int socket_keepalive(SOCKET socket) { int optval; socklen_t optlen = sizeof(int); optval = 1; int ret = setsockopt(socket, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen); if(ret != 0) { printf("SO_KEEPALIVE set failed:%d\n",errno); return -1; } #ifdef SOL_TCP //重复检查次数 optval = 5; ret = setsockopt(socket, SOL_TCP, TCP_KEEPCNT, &optval, optlen); if(ret != 0) { printf("SO_KEEPALIVE set failed:%d\n",errno); return -1; } //检查空闲 optval = 1; ret = setsockopt(socket, SOL_TCP, TCP_KEEPIDLE, &optval, optlen); if(ret != 0) { printf("SO_KEEPALIVE set failed:%d\n",errno); return -1; } //检查间隔 optval = 1; ret = setsockopt(socket, SOL_TCP, TCP_KEEPINTVL, &optval, optlen); if(ret != 0) { printf("SO_KEEPALIVE set failed:%d\n",errno); return -1; } #endif return 0; } int socket_sendbuf(SOCKET socket,int size){ return setsockopt(socket,SOL_SOCKET,SO_SNDBUF,(const char*)&size,sizeof(int)); } int socket_recvbuf(SOCKET socket,int size){ return setsockopt(socket,SOL_SOCKET,SO_RCVBUF,(const char*)&size,sizeof(int)); } int socket_linger(SOCKET socket,int onoff,int linger){ struct linger l; l.l_onoff = onoff; l.l_linger = linger; return setsockopt(socket, SOL_SOCKET, SO_LINGER,(const void *) &l, sizeof(struct linger)); } int socket_sendtimeout(SOCKET socket, int timeout){ return setsockopt(socket,SOL_SOCKET,SO_SNDTIMEO,&timeout,sizeof(timeout)); } int socket_recvtimeout(SOCKET socket, int timeout){ return setsockopt(socket,SOL_SOCKET,SO_RCVTIMEO,&timeout,sizeof(timeout)); } uint64_t time_microsecond(){ struct timeval tv; gettimeofday(&tv,NULL); return tv.tv_sec*1000000 + tv.tv_usec; } uint64_t time_millisecond(){ struct timeval tv; gettimeofday(&tv,NULL); return tv.tv_sec*1000 + tv.tv_usec/1000; } long time_second(){ struct timeval tv; gettimeofday(&tv,NULL); return tv.tv_sec; } double time_secondD(){ struct timeval tv; gettimeofday(&tv,NULL); double usec = (double)tv.tv_usec; return tv.tv_sec + usec/1000000; } #endif void init() { #ifdef WIN32 initNetwork(); #endif } int main(int argc,char *argv[]) { init(); SOCKET fd = socket(AF_INET,SOCK_STREAM,0); struct sockaddr_in server = {0}; server.sin_family = AF_INET; server.sin_addr.s_addr = inet_addr(argv[1]); server.sin_port = htons(atoi(argv[2])); int ret = bind(fd,&server,sizeof(struct sockaddr_in)); if(ret < 0) { printf("bind %d errno:%d\n",ret,errno); return -1; } // socket_keepalive(fd); // socket_sendbuf(fd,0); // socket_linger(fd,1,0); ret = listen(fd,100); if(ret == -1) { printf("listen %d errno:%d\n",ret,errno); close(fd); return -1; } int nfds = fd; fd_set readfds_cache; fd_set writefds_cache; FD_ZERO(&readfds_cache); FD_ZERO(&writefds_cache); FD_SET(fd,&readfds_cache); fd_set readfds; fd_set writefds; FD_ZERO(&readfds); FD_ZERO(&writefds); struct timeval timeout; timeout.tv_sec = 1; timeout.tv_usec = 0; printf("begin run...\n"); int fd_count = 1; int no_recv = 0; int no_send = 0; fd_set * ptrRecv = &readfds; fd_set * ptrSend = &writefds; while(1) { readfds = readfds_cache; writefds = writefds_cache; if(no_recv == 1) { // ptrRecv = NULL; } if(no_send == 1) { ptrSend = NULL; } int ret = select(nfds + 1,ptrRecv,ptrSend,NULL,&timeout); if(ret == -1) { if (errno == EINTR) continue; printf("select error:%d\n",errno); break; } if(ret == 0) { continue; } printf("%d select %d.\n",time_second(),ret); if(ptrRecv != NULL && FD_ISSET(fd,ptrRecv)) { printf("accept connection\n"); struct sockaddr_in client = {0}; socklen_t client_len = sizeof(struct sockaddr_in); SOCKET cfd = accept(fd,&client,&client_len); if(cfd == -1) { ERRORS("accept",errno); }else{ printf("accept %d\n",cfd); setnonblocking(cfd); socket_keepalive(cfd); // socket_sendbuf(cfd,0); // socket_linger(fd,1,0); if(!no_recv){ FD_SET(cfd,&readfds_cache); } if(!no_send){ FD_SET(cfd,&writefds_cache); } nfds = nfds>cfd?nfds:cfd; fd_count++; printf("accept end.\n"); } } int i = 0; for (; i <= nfds; i++) { if( i == fd) { continue; } if(ptrRecv != NULL && no_recv == 0) { if (FD_ISSET(i, ptrRecv)) { printf("recv %d check...\n",i); char peek; ret = recv(i,&peek,1,MSG_PEEK); if(ret != 1) { if(ret == 0 || (ret != 0 && EAGAIN != errno) ) { printf("recv %d check close.%d\n",i,errno); FD_CLR(i,&readfds_cache); FD_CLR(i,&writefds_cache); close(i); printf("check close %d.\n",i); fd_count--; continue; } printf("recv %d check :%d errno:%d\n",i,ret,errno); }else{ printf("recv %d ...\n",i); char buffer[65536]; ret = recv(i,buffer,65535,0); if(ret == -1) { if(EAGAIN != errno) { printf("recv %d close.%d\n",i,errno); FD_CLR(i,&readfds_cache); FD_CLR(i,&writefds_cache); close(i); printf("close %d.\n",i); fd_count--; continue; } }else if(ret == 0) { printf("recv %d close.\n",i); FD_CLR(i,&readfds_cache); FD_CLR(i,&writefds_cache); close(i); printf("close %d.\n",i); fd_count--; continue; }else{ buffer[ret] = 0x00; // printf("%s\n",buffer); printf("recv %i %d\n",i,ret); } } } } if(ptrSend != NULL && no_send == 0) { if(FD_ISSET(i, ptrSend)) { printf("send %d ...\n",i); ret = send(i,"client",6,0); if(ret == -1) { if(EAGAIN != errno) { printf("send %d close.%d\n",i,errno); FD_CLR(i,&readfds_cache); FD_CLR(i,&writefds_cache); close(i); printf("close %d.\n",i); fd_count--; } }else{ printf("send %d %d\n",i,ret); } } } } } return -1; }client:
#include <stdint.h> #include <stdio.h> #include <stdlib.h> #ifdef _WIN32 #define WIN32 1 #endif #ifdef WIN32 #include <winsock2.h> #include <windows.h> #include <mstcpip.h> #define close closesocket #else #include <unistd.h> #include <sys/select.h> #include <sys/time.h> #include <sys/types.h> #include <sys/socket.h> #include <netinet/in.h> #include <netinet/tcp.h> #include <fcntl.h> #include <errno.h> typedef int SOCKET; #endif #define LOGS_FULL(OPTION,ERRNO) \ printf("(%s %d %s) %s errno:%d",__FILE__,__LINE__,__FUNCTION__,OPTION,ERRNO); #define LOGS(OPTION,ERRNO) \ printf("%s errno:%d",OPTION,ERRNO); #define ERRORS(OPTION,ERRNO) LOGS(OPTION,ERRNO) #ifdef WIN32 int initNetwork() { WSADATA wsaData; int ret = WSAStartup(MAKEWORD(2,2), &wsaData); if (ret != 0) return 1; return 0; } int setnonblocking(SOCKET sock) { unsigned long ul=1; int ret=ioctlsocket(sock,FIONBIO,(unsigned long *)&ul); if(ret==SOCKET_ERROR) { } return ret; } int socket_keepalive(SOCKET socket) { int keep_alive = 1; int ret = setsockopt(socket, SOL_SOCKET, SO_KEEPALIVE, (char*)&keep_alive, sizeof(keep_alive)); if (ret == SOCKET_ERROR) { printf("SO_KEEPALIVE failed:%d\n", WSAGetLastError()); return -1; } struct tcp_keepalive in_keep_alive = {0}; unsigned long ul_in_len = sizeof(struct tcp_keepalive); struct tcp_keepalive out_keep_alive = {0}; unsigned long ul_out_len = sizeof(struct tcp_keepalive); unsigned long ul_bytes_return = 0; in_keep_alive.onoff = 1; in_keep_alive.keepaliveinterval = 5000; in_keep_alive.keepalivetime = 1000; ret = WSAIoctl(socket, SIO_KEEPALIVE_VALS, (LPVOID)&in_keep_alive, ul_in_len, (LPVOID)&out_keep_alive, ul_out_len, &ul_bytes_return, NULL, NULL); if (ret == SOCKET_ERROR) { printf("WSAIoctl KEEPALIVE failed:%d\n", WSAGetLastError()); return -1; } return 0; } #else int setnonblocking(SOCKET sock) { int opts = fcntl(sock,F_GETFL); if (opts < 0 ) { printf( "fcntl(sock,GETFL) " ); return -1; } opts = opts | O_NONBLOCK; if (fcntl(sock,F_SETFL,opts) < 0 ) { printf( "fcntl(sock,SETFL,opts) " ); return -1; } return 0; } int socket_keepalive(SOCKET socket) { int optval; socklen_t optlen = sizeof(int); optval = 1; int ret = setsockopt(socket, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen); if(ret != 0) { printf("SO_KEEPALIVE set failed:%d\n",errno); return -1; } #ifdef SOL_TCP //重复检查次数 optval = 5; ret = setsockopt(socket, SOL_TCP, TCP_KEEPCNT, &optval, optlen); if(ret != 0) { printf("SO_KEEPALIVE set failed:%d\n",errno); return -1; } //检查空闲 optval = 1; ret = setsockopt(socket, SOL_TCP, TCP_KEEPIDLE, &optval, optlen); if(ret != 0) { printf("SO_KEEPALIVE set failed:%d\n",errno); return -1; } //检查间隔 optval = 1; ret = setsockopt(socket, SOL_TCP, TCP_KEEPINTVL, &optval, optlen); if(ret != 0) { printf("SO_KEEPALIVE set failed:%d\n",errno); return -1; } #endif return 0; } int socket_sendbuf(SOCKET socket,int size){ return setsockopt(socket,SOL_SOCKET,SO_SNDBUF,(const char*)&size,sizeof(int)); } int socket_recvbuf(SOCKET socket,int size){ return setsockopt(socket,SOL_SOCKET,SO_RCVBUF,(const char*)&size,sizeof(int)); } int socket_linger(SOCKET socket,int onoff,int linger){ struct linger l; l.l_onoff = onoff; l.l_linger = linger; return setsockopt(socket, SOL_SOCKET, SO_LINGER,(const void *) &l, sizeof(struct linger)); } int socket_sendtimeout(SOCKET socket, int timeout){ return setsockopt(socket,SOL_SOCKET,SO_SNDTIMEO,&timeout,sizeof(timeout)); } int socket_recvtimeout(SOCKET socket, int timeout){ return setsockopt(socket,SOL_SOCKET,SO_RCVTIMEO,&timeout,sizeof(timeout)); } uint64_t time_microsecond(){ struct timeval tv; gettimeofday(&tv,NULL); return tv.tv_sec*1000000 + tv.tv_usec; } uint64_t time_millisecond(){ struct timeval tv; gettimeofday(&tv,NULL); return tv.tv_sec*1000 + tv.tv_usec/1000; } long time_second(){ struct timeval tv; gettimeofday(&tv,NULL); return tv.tv_sec; } double time_secondD(){ struct timeval tv; gettimeofday(&tv,NULL); double usec = (double)tv.tv_usec; return tv.tv_sec + usec/1000000; } #endif void init() { #ifdef WIN32 initNetwork(); #endif } int main(int argc,char *argv[]) { init(); SOCKET fd = socket(AF_INET,SOCK_STREAM,0); struct sockaddr_in server = {0}; server.sin_family = AF_INET; server.sin_addr.s_addr = inet_addr(argv[1]); server.sin_port = htons(atoi(argv[2])); int ret = connect(fd,&server,sizeof(struct sockaddr_in)); if(ret != 0) { printf("connect error:%d\n",errno); return -1; } int nfds = fd; fd_set readfds_cache; fd_set writefds_cache; FD_ZERO(&readfds_cache); FD_ZERO(&writefds_cache); FD_SET(fd,&readfds_cache); FD_SET(fd,&writefds_cache); fd_set readfds; fd_set writefds; FD_ZERO(&readfds); FD_ZERO(&writefds); struct timeval timeout; timeout.tv_sec = 1; timeout.tv_usec = 0; int fd_count = 1; int last_nfds = nfds; printf("begin run...\n"); int no_recv = 0; int no_send = 0; fd_set * ptrRecv = &readfds; fd_set * ptrSend = &writefds; while(1) { if(fd_count <= 0){ break; } nfds = last_nfds; readfds = readfds_cache; writefds = writefds_cache; ptrRecv = &readfds; ptrSend = &writefds; if(no_recv == 1) { ptrRecv = NULL; } if(no_send == 1) { ptrSend = NULL; } int ret = select(nfds + 1,ptrRecv,ptrSend,NULL,&timeout); if(ret == -1) { if (errno == EINTR) continue; printf("select error:%d\n",errno); break; } if(ret == 0) { continue; } printf("%d select %d.\n",time_second(),ret); int i = 0; for (; i <= nfds; i++) { if(ptrRecv != NULL && no_recv == 0) { if (FD_ISSET(i, ptrRecv)) { printf("recv %d check...\n",i); char peek; ret = recv(i,&peek,1,MSG_PEEK); if(ret != 1) { if(ret == 0 || (ret != 0 && EAGAIN != errno) ) { printf("recv %d check close.%d\n",i,errno); FD_CLR(i,&readfds_cache); FD_CLR(i,&writefds_cache); close(i); printf("check close %d.\n",i); fd_count--; continue; } printf("recv %d check :%d errno:%d\n",i,ret,errno); }else{ printf("recv %d ...\n",i); char buffer[65536]; ret = recv(i,buffer,65535,0); if(ret == -1) { if(EAGAIN != errno) { printf("recv %d close.%d\n",i,errno); FD_CLR(i,&readfds_cache); FD_CLR(i,&writefds_cache); close(i); printf("close %d.\n",i); fd_count--; continue; } }else if(ret == 0) { printf("recv %d close.\n",i); FD_CLR(i,&readfds_cache); FD_CLR(i,&writefds_cache); close(i); printf("close %d.\n",i); fd_count--; continue; }else{ buffer[ret] = 0x00; // printf("%s\n",buffer); printf("recv %i %d\n",i,ret); } } } } if(ptrSend != NULL && no_send == 0) { if(FD_ISSET(i, ptrSend)) { printf("send %d ...\n",i); ret = send(i,"client",6,0); if(ret == -1) { if(EAGAIN != errno) { printf("send %d close.%d\n",i,errno); FD_CLR(i,&readfds_cache); FD_CLR(i,&writefds_cache); close(i); printf("close %d.\n",i); fd_count--; } }else{ printf("send %d %d\n",i,ret); } } } } } return -1; }Makefile
CORSS_PREFIX := SYSROOT := $(warning $(CORSS_PREFIX)) $(warning $(SYSROOT)) CC = $(CORSS_PREFIX)gcc AR = $(CORSS_PREFIX)ar RM = rm -rf #原始目录 SRC_PATH :=. MODULES := #目标名 TARGET := #源文件 MODULES += $(wildcard $(SRC_PATH)/src/*.c) MODULES += $(wildcard $(SRC_PATH)/*.c) SRCS += $(MODULES) #中间文件 OBJS := $(SRCS:.c=.o) TARGET = MODULE_OBJS := $(MODULES:.c=.o) OBJ_TEST=win32/server.o OBJS_TEST=$(MODULE_OBJS) $(OBJ_TEST) TARGET_TEST=server OBJ_INFO=win32/client.o OBJS_INFO=$(MODULE_OBJS) $(OBJ_INFO) TARGET_INFO=client ALL_OBJS=$(OBJS) $(OBJS_TEST) $(OBJ_INFO) #动态库 LIBS := pthread #模块库文件 MOULE_LIBRARY_PATH = $(SYSROOT)/usr/lib/ $(SYSROOT)/usr/local/lib/ #头文件路径 INCLUDE_PATH :=. #动态库路径 LIBRARY_PATH := INCLUDE_PATH += $(SYSROOT)/usr/include INCLUDE_PATH += ./include INCLUDE_PATH += ./src INCLUDE_PATH += ./test INCLUDE_PATH += ./win32 LIBRARY_PATH += $(MOULE_LIBRARY_PATH) LIBRARY_PATH += ./ RELEASE = 1 BITS = #ifeq ( 1 , ${DBG_ENABLE} ) # CFLAGS += -D_DEBUG -O0 -g -DDEBUG=1 #endif CFLAGS = -Wall -DMAIN_TEST -DUSE_BOOL LFLAGS = #头文件 CFLAGS += $(foreach dir,$(INCLUDE_PATH),-I$(dir)) #库路径 LDFLAGS += $(foreach lib,$(LIBRARY_PATH),-L$(lib)) #库名 LDFLAGS += $(foreach lib,$(LIBS),-l$(lib)) #检查版本 ifeq ($(RELEASE),0) #debug CFLAGS += -g else #release CFLAGS += -O3 -DNDEBUG endif #检查位宽 ifeq ($(BITS),32) CFLAGS += -m32 LFLAGS += -m32 else ifeq ($(BITS),64) CFLAGS += -m64 LFLAGS += -m64 else endif endif $(warning $(OBJS)) #操作命令 all:clean build $(ALL_OBJS):%.o:%.c $(CC) $(CFLAGS) -c $^ -o $@ build_static:$(OBJS) # $(AR) -cru $(TARGET) $(OBJS) build_test:build_static $(OBJ_TEST) $(CC) $(CFLAGS) $(LFLAGS) -o $(TARGET_TEST) $(OBJ_TEST) $(LDFLAGS) build_info:build_static $(OBJ_INFO) $(CC) $(CFLAGS) $(LFLAGS) -o $(TARGET_INFO) $(OBJ_INFO) $(LDFLAGS) build:build_test build_info $(RM) $(ALL_OBJS) clean: echo $(SRCS) $(RM) $(ALL_OBJS) $(TARGET) $(TARGET_TEST) $(TARGET_INFO)makefile中CORSS_PREFIX和 SYSROOT是为了跨平台编译才增加的。
CORSS_PREFIX ：跨平台编译工具路径
SYSROOT：跨平台环境头文件及库路径
跨平台编译时需要配置如下:
#编译 make SYSROOT=/。。。/root CORSS_PREFIX=/。。。/prebuilts/toolschain/usr/bin/arm-linux-