Hi all,
i am having a problem using the epoll mechanism. I have written a small webserver [just serving an empty gif image] using epoll. When testing it with ab (apache benchmark tool) everything works fine, it runs without any problems even with several ab instances making requests for several days. The problem occurs when i run this small webserver in a real enviroment with 2.000 - 3.000 requests from different IPs. After about 1.000.000 requests the webserver process starts to consume 100% of CPU load and the rate of accepted connections drops to zero. Does anyone has any clue where to start watching for a solution ? If wanted i can post the source.
Regards,
Greg
epoll
Lets start out with some assertions where the bug is:
1. your code
2. the kernel
Given the low frequency of the error - my guess is your code.
You should try using strace or carefully walking thru your code
for race conditions and invalid assumptions.
Try and simplify what you are doing; try and slow down the code to try
and induce the error more frequently.
If its the kernel, same as above: produce as small a working test case
and either report the bug, or look at the source to see where the race condition is.
code of the server
Below you will finde the code to the "webserver":
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
char gif[43] = {
71, 73, 70, 56, 57, 97, 1, 0, 1, 0, -128, 0, 0, -95, -90, -95,
-95, -90, -95, 33, -7, 4, 1, 10, 0, 1, 0, 44, 0, 0, 0, 0, 1, 0,
1, 0, 0, 2, 2, 76, 1, 0, 59};
int set_socket_options(int fd);
int process_connection(int fd);
int set_socket_options(int fd) {
int flags = 1;
if (fd > 0) {
if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &flags, sizeof(flags)) == -1) {
return -1;
}
flags = 0;
if ( (flags = fcntl(fd, F_GETFL, 0)) == -1) {
return -1;
}
if ( (fcntl(fd, F_SETFL, flags | O_NONBLOCK)) == -1) {
return -1;
}
return fd;
}
return -1;
}
int process_connection(int fd) {
char buffer[1024];
ssize_t bytes = 0;
if ((bytes = recv(fd, &buffer, sizeof(char) * 1024, 0)) == -1) {
return -1;
}
send(fd, gif, sizeof(char) * 43, 0);
}
/**
* main
*
*/
int main( int argc, char **argv ) {
int socket_fd = -1;
int epoll_fd = -1;
struct sockaddr_in addr, addr_in;
socklen_t size = sizeof(struct sockaddr);
int port = 8080;
int max_connections = 4096;
int epoll_max_connections = max_connections;
struct epoll_event event, events[epoll_max_connections];
int count = 0;
int loop = 0;
struct rlimit limit;
if ((socket_fd = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
perror("error creating socket");
exit(EXIT_FAILURE);
}
if (set_socket_options(socket_fd) == -1) {
close(socket_fd);
perror("error setting socket to non blocking state");
exit(EXIT_FAILURE);
}
memset(&addr, 0, sizeof(struct sockaddr_in));
addr.sin_family = AF_INET;
addr.sin_port = htons(port);
addr.sin_addr.s_addr = htonl(INADDR_ANY);
if (bind(socket_fd, (struct sockaddr*)&addr, sizeof(struct sockaddr_in)) == -1) {
close(socket_fd);
perror("error binding socket");
exit(EXIT_FAILURE);
}
if (listen(socket_fd, max_connections) == -1) {
close(socket_fd);
perror("error listening on socket");
exit(EXIT_FAILURE);
}
if ((epoll_fd = epoll_create(epoll_max_connections)) == -1) {
close(socket_fd);
perror("error creating epoll");
exit(EXIT_FAILURE);
}
event.events = EPOLLIN | EPOLLPRI | EPOLLERR | EPOLLHUP;
event.data.fd = socket_fd;
if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, socket_fd, &event) == -1) {
close(socket_fd);
close(epoll_fd);
perror("error adding server socket to epoll");
exit(EXIT_FAILURE);
}
getrlimit(RLIMIT_NOFILE, &limit);
printf("max opened files allowed: %d\n", limit.rlim_cur);
while (1) {
count = epoll_wait(epoll_fd, events, epoll_max_connections, -1);
for (loop = 0; loop < count; loop++) {
if (events[loop].events & (EPOLLHUP | EPOLLERR)) {
perror("error in epoll event");
continue;
}
if (events[loop].events & EPOLLIN) {
if (events[loop].data.fd == socket_fd) {
int client_fd = -1;
if ((client_fd = accept(socket_fd, (struct sockaddr*)&addr_in, &size)) == -1) {
perror("error accepting client connection");
continue;
}
if (set_socket_options(client_fd) == -1) {
perror("error setting client socket options");
close(client_fd);
continue;
}
event.events = EPOLLIN | EPOLLPRI | EPOLLERR | EPOLLHUP;
event.data.fd = client_fd;
if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_fd, &event) == -1) {
perror("error adding client socket to epoll");
close(client_fd);
continue;
}
} else {
process_connection(events[loop].data.fd);
close(events[loop].data.fd);
event.data.fd = events[loop].data.fd;
epoll_ctl(epoll_fd, EPOLL_CTL_DEL, events[loop].data.fd, &event);
}
}
}
}
return EXIT_SUCCESS;
}
<pre>....</pre>
you should enclose program code in <pre>....</pre> to make it readable and non-damaged. the < character should be replaced by < to not get the rest of the line parsed as an invalid html tag. without indentation and with parts of the code missing few people will be inclined to read your code.
generally you should check if some function call/syscall failed just before the hang. there are still unchecked calls (e.g. close(), send()) and you could try a debugging mode with more logging and/or strace to get the full picture.
I have posted a corrected
I have posted a corrected version of the source. I hope this helps a bit to analize my problem.
#include <stdio.h> #include <stdlib.h> #include <string.h> #include <sys/socket.h> #include <netinet/in.h> #include <unistd.h> #include <fcntl.h> #include <sys/epoll.h> #include <sys/time.h> #include <sys/resource.h> #define BUFFER_LENGTH 1024 #define BUFFER_SIZE sizeof(char) * BUFFER_LENGTH #define GIF_BUFFER_SIZE sizeof(char) * 43 #define FLAGS_SIZE sizeof(int) #define MAX_CONNECTIONS 4096 #define PORT 8080 char gif[43] = { 71, 73, 70, 56, 57, 97, 1, 0, 1, 0, -128, 0, 0, -95, -90, -95, -95, -90, -95, 33, -7, 4, 1, 10, 0, 1, 0, 44, 0, 0, 0, 0, 1, 0, 1, 0, 0, 2, 2, 76, 1, 0, 59}; int set_socket_options(int fd); int process_connection(int fd); int set_socket_options(int fd) { int flags = 1; if (fd > 0) { if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &flags, FLAGS_SIZE) == -1) { return -1; } flags = 0; if ( (flags = fcntl(fd, F_GETFL, 0)) == -1) { return -1; } if ( (fcntl(fd, F_SETFL, flags | O_NONBLOCK)) == -1) { return -1; } return fd; } return -1; } int process_connection(int fd) { char buffer[BUFFER_LENGTH]; ssize_t bytes = 0; if ((bytes = recv(fd, &buffer, BUFFER_SIZE, 0)) == -1) { return -1; } if (send(fd, gif, GIF_BUFFER_SIZE, 0) == -1) { return -1; } } /** * main * */ int main( int argc, char **argv ) { int socket_fd = -1; int epoll_fd = -1; struct sockaddr_in addr, addr_in; socklen_t size = sizeof(struct sockaddr); int port = PORT; int max_connections = MAX_CONNECTIONS; int epoll_max_connections = max_connections; struct epoll_event event, events[epoll_max_connections]; int count = 0; int loop = 0; struct rlimit limit; if ((socket_fd = socket(AF_INET, SOCK_STREAM, 0)) == -1) { perror("error creating socket"); exit(EXIT_FAILURE); } if (set_socket_options(socket_fd) == -1) { close(socket_fd); perror("error setting socket to non blocking state"); exit(EXIT_FAILURE); } memset(&addr, 0, sizeof(struct sockaddr_in)); addr.sin_family = AF_INET; addr.sin_port = htons(port); addr.sin_addr.s_addr = htonl(INADDR_ANY); if (bind(socket_fd, (struct sockaddr*)&addr, sizeof(struct sockaddr_in)) == -1) { close(socket_fd); perror("error binding socket"); exit(EXIT_FAILURE); } if (listen(socket_fd, max_connections) == -1) { close(socket_fd); perror("error listening on socket"); exit(EXIT_FAILURE); } if ((epoll_fd = epoll_create(epoll_max_connections)) == -1) { close(socket_fd); perror("error creating epoll"); exit(EXIT_FAILURE); } event.events = EPOLLIN | EPOLLPRI | EPOLLERR | EPOLLHUP; event.data.fd = socket_fd; if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, socket_fd, &event) == -1) { close(socket_fd); close(epoll_fd); perror("error adding server socket to epoll"); exit(EXIT_FAILURE); } getrlimit(RLIMIT_NOFILE, &limit); printf("max opened files allowed: %d\n", limit.rlim_cur); while (1) { count = epoll_wait(epoll_fd, events, epoll_max_connections, -1); for (loop = 0; loop < count; loop++) { if ((events[loop].events & (EPOLLHUP | EPOLLERR)) && events[loop].data.fd != socket_fd) { perror("error in epoll event"); event.data.fd = events[loop].data.fd; if (epoll_ctl(epoll_fd, EPOLL_CTL_DEL, events[loop].data.fd, &event) == -1) { perror("error removing broken client socket from epoll"); } if (close(events[loop].data.fd) == -1) { perror("error closing broken client socket"); } continue; } if (events[loop].events & EPOLLIN) { if (events[loop].data.fd == socket_fd) { int client_fd = -1; if ((client_fd = accept(socket_fd, (struct sockaddr*)&addr_in, &size)) == -1) { perror("error accepting client connection"); continue; } if (set_socket_options(client_fd) == -1) { perror("error setting client socket options"); if (close(client_fd) == -1) { perror("error closing client socket"); } continue; } event.events = EPOLLIN | EPOLLPRI | EPOLLERR | EPOLLHUP; event.data.fd = client_fd; if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, client_fd, &event) == -1) { perror("error adding client socket to epoll"); if (close(client_fd) == -1) { perror("error closing client socket"); } continue; } } else { if (process_connection(events[loop].data.fd) == -1) { perror("error processing connection"); } event.data.fd = events[loop].data.fd; if (epoll_ctl(epoll_fd, EPOLL_CTL_DEL, events[loop].data.fd, &event) == -1) { perror("error removing processed client socket from epoll"); } if (close(events[loop].data.fd) == -1) { perror("error closing processed client socket"); } } } } } return EXIT_SUCCESS; }What about running strace on
What about running strace on your server and paste some output?
Looks to me like the writer
Looks to me like the writer is relying on recv() retrieving what is send()'t. This is not true - you need to packetise yourself. This can lead to deadlock or infinite cpu spinning as you dont necessarily do the right thing.
strace should show you the error of your ways.