Re: SO_REUSEPORT: strange kernel balancer behaviour

[trafdev]

It's like shared acceptor FD and N processes:

Listening proc:
bool HttpServer::Listen(unsigned short port, uint listen_backlog) {
   LOG4CXX_TRACE(kLogger, "Listen");
   if ((sockd_acceptor_ = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
       LOG4CXX_ERROR_ERRNO(kLogger, "socket");
       return false;
   }
   struct sockaddr_in sa_in;
   memset(&sa_in, 0, sizeof(sa_in));
   sa_in.sin_family = AF_INET;
   sa_in.sin_port = htons(port);
   sa_in.sin_addr.s_addr = htonl(INADDR_ANY);
   int yes = 1;
   if (setsockopt(sockd_acceptor_, SOL_SOCKET, SO_REUSEPORT, &yes, 
sizeof (yes)) == -1) {
       LOG4CXX_ERROR_ERRNO(kLogger, "setsockopt");
       return false;
   }
   if (bind(sockd_acceptor_, (const struct sockaddr*)&sa_in, 
sizeof(sa_in)) == -1) {
       LOG4CXX_ERROR_ERRNO(kLogger, "bind");
       return false;
   }
   if (listen(sockd_acceptor_, listen_backlog) == -1) {
       LOG4CXX_ERROR_ERRNO(kLogger, "socket listen");
       return false;
   }
   if (!fcntl_set(sockd_acceptor_, O_NONBLOCK))
       return false;

   sockd_acceptor_watcher_.set<HttpServer, &HttpServer::AcceptConnCb> 
(this);
   sockd_acceptor_watcher_.start(sockd_acceptor_, ev::READ);

   int wrk = thread::hardware_concurrency();
   while (--wrk) {
       pid_t pid = fork();
       if (pid == 0) {
           LOG4CXX_INFO(kLogger, "process " << wrk <<  " started");
           ev::default_loop().post_fork();
           ev::default_loop().run();
           return true;
       }
   }

   ev::default_loop().run();
   return true;
}


Accept conn callback:

void HttpServer::AcceptConnCb(ev::io &w, int revents) {
   LOG4CXX_TRACE(kLogger, "AcceptConnCb");
   CHECK_LIBEV_ERROR
   struct sockaddr_in addr;
   socklen_t addr_len = sizeof (addr);
   int sockd;
   if ((sockd = accept(sockd_acceptor_, (struct sockaddr *) &addr,
           &addr_len)) == -1) {
       if (errno == EAGAIN || errno == EWOULDBLOCK) {
           return;
       } else {
           LOG4CXX_ERROR_ERRNO(kLogger, "accept");
           return;
       }
   }
   if (!fcntl_set(sockd, O_NONBLOCK))
       return;
   char str[INET_ADDRSTRLEN];
   if (!inet_ntop(AF_INET, &(addr.sin_addr), str, INET_ADDRSTRLEN)) {
       LOG4CXX_ERROR(kLogger, "inet_ntop");
       return;
   }
   // Internal logic follows... ConnIn* c = 
ConnPool::Instance().Add(sockd, str);
   // c->Recv(this);
   return;
}

Accept conn callback is called in N processes on each connection, only 
one wins,
others exit by errno == EAGAIN case. Overhead is almost zero.
Problem is that "wins" distribution is far from equal.


On Mon Jul 22 23:37:57 2013, Adrian Chadd wrote:
eg:

* one process, one listen thread, multiple dispatch threads?
* n processes, one listen FD per process, all listening on the same IP:port?
* one process, each thread listening on the same IP:port?
* something else?

Thanks,



-adrian


_______________________________________________
freebsd-net@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-net
To unsubscribe, send any mail to "freebsd-net-unsubscr...@freebsd.org"