void
call_udpl_epoll(struct sockaddr_in servaddr, unsigned expected_size,
    unsigned *latency, size_t sender_buf_size)
{
    int sockfd;
    char buf[BUF_SIZE];

    struct timespec start;
    struct timespec end;

    int n, i;
    size_t sent_size=0;
    size_t max_send_size=0;

    struct epoll_event ev, events[MAX_EVENTS];
    int nfds, epollfd;

    struct timespec wait_for_server={.tv_sec=0, .tv_nsec=INTERVAL};

    assert(sender_buf_size <= BUF_SIZE);

    nanosleep(&wait_for_server, NULL);
    sockfd = socket(AF_INET, SOCK_DGRAM, 0);
    if (setNonblocking(sockfd) != 0) perror("setNonblocking");

    epollfd = epoll_create(16);
    if (epollfd == -1) {
        perror("epoll_create");
        exit(EXIT_FAILURE);
    }

    ev.events = EPOLLIN;
    ev.data.fd = sockfd;
    // need not care about blocking
    if (epoll_ctl(epollfd, EPOLL_CTL_ADD, sockfd, &ev) == -1) {
        perror("epoll_ctl: sockfd");
        exit(EXIT_FAILURE);
    }

    max_send_size = MIN(MAX_UDP_SIZE, MIN(expected_size, sender_buf_size));
    clock_gettime(CLOCK_MONOTONIC, &start);
    // send something first
    n = sendton(sockfd, buf, max_send_size, 0,
        (struct sockaddr *)&servaddr, sizeof(servaddr));
    if (n < 0)
        perror("client sendto");

    while (sent_size < expected_size) {
        nfds = epoll_wait(epollfd, events, MAX_EVENTS, -1);
        if (nfds == -1) {
            perror("epoll_pwait");
            exit(EXIT_FAILURE);
        }

        for (i = 0; i < nfds; ++i) {
            if (events[i].events & EPOLLIN) { // readable
                n = Readn(events[i].data.fd, buf, BUF_SIZE);
                sent_size += n; // only the echoed size is effective
                if (DEBUG) printf("%u / %u<<<<%u / %u echo rate\n", n, MIN(max_send_size, expected_size - sent_size), sent_size, expected_size);
                if (sendton(sockfd, buf, MIN(max_send_size, expected_size - sent_size), 0,
                    (struct sockaddr *)&servaddr, sizeof(servaddr)) < 0)
                    perror("client sendto");
            }
        }
    }
    clock_gettime(CLOCK_MONOTONIC, &end);
    *latency = ((end.tv_sec - start.tv_sec) * 1000000000 +
        end.tv_nsec - start.tv_nsec) / 2;
    close(sockfd);
}


void
call_udpt_epoll(struct sockaddr_in servaddr, unsigned expected_size,
    unsigned *latency, size_t sender_buf_size, size_t *received)
{
    int sockfd;
    char buf[BUF_SIZE];

    struct timespec start;
    struct timespec end;

    int n, i, ret;
    size_t sent_size=0;
    size_t max_send_size, send_size;

    struct epoll_event ev, events[MAX_EVENTS];
    int nfds, epollfd;

    struct timespec wait_for_server={.tv_sec=0, .tv_nsec=INTERVAL};

    assert(sender_buf_size <= BUF_SIZE);

    nanosleep(&wait_for_server, NULL);
    sockfd = socket(AF_INET, SOCK_DGRAM, 0);
    if (setNonblocking(sockfd) != 0) perror("setNonblocking");

    epollfd = epoll_create(16);
    if (epollfd == -1) {
        perror("epoll_create");
//.........这里部分代码省略.........

int network_glue(void)
{
    struct sockaddr_in s4;
    struct sockaddr_in6 s6;
    struct sockaddr_storage ss;
    int sfd, s;
    int efd;
    struct epoll_event event;
    socklen_t len;
    int optval;

    if (strlen(ip4) > 0) {
        s4.sin_family = AF_INET;
        inet_pton(AF_INET, ip4, &(s4.sin_addr));
        s4.sin_port = htons(port);
        memcpy(&ss, &s4, sizeof(s4));
    } else if (strlen(ip6) > 0) {
        s6.sin6_family = AF_INET6;
        inet_pton(AF_INET6, ip6, &(s6.sin6_addr));
        s6.sin6_port = htons(port);
        memcpy(&ss, &s6, sizeof(s6));
    } else {
        fprintf(stderr, "IPv4/IPv6 unspecified\n");
        return EXIT_FAILURE;
    }

    sfd = socket(ss.ss_family, SOCK_STREAM, 0);

    if (sfd == -1) {
        perror("socket() failed");
        return EXIT_FAILURE;
    }

    make_socket_non_blocking(sfd);

#ifndef WIN32
    int one = 1;
    if (setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)) == -1) {
        perror("setsockopt() failed");
        close(sfd);
        return EXIT_FAILURE;
    }
#endif

    if (bind(sfd, (struct sockaddr *)&ss, sizeof(ss)) < 0) {
        perror("bind() failed");
        close(sfd);
        return EXIT_FAILURE;
    }

    if (listen(sfd, num_daemons) < 0) {
        perror("listen() failed");
        close(sfd);
        return EXIT_FAILURE;
    }

    efd = epoll_create1(0);
    if (efd == -1) {
        perror("epoll_create() failed");
        close(sfd);
        return EXIT_FAILURE;
    }

    event.data.fd = sfd;
    event.events = EPOLLIN | EPOLLET;   // set edge-trigger
    s = epoll_ctl(efd, EPOLL_CTL_ADD, sfd, &event);
    if (s == -1) {
        perror("epoll_ctl(sfd) failed");
        close(sfd);
        return EXIT_FAILURE;
    }

    events = calloc(num_daemons, sizeof event);

    // the event loop
    while (1) {
        int n, i;

        n = epoll_wait(efd, events, num_daemons, -1);
        for (i = 0; i < n; i++) {
            if ((events[i].events & EPOLLERR) ||
                (events[i].events & EPOLLHUP) || (!(events[i].events & EPOLLIN))) {
                // an error has occured on this fd, or the socket is not
                // ready for reading 
                // (happens when the client closes the connection due to timeout)
                //fprintf(stderr, "epoll error\n");
                st.queries_timeout++;
                close(events[i].data.fd);
                continue;
            }

            else if (sfd == events[i].data.fd) {
                // we have a notification on the listening socket, which
                // means one or more incoming connections.
                while (1) {
                    struct sockaddr in_addr;
                    socklen_t in_len;
                    int infd;

                    in_len = sizeof in_addr;
//.........这里部分代码省略.........

void
fDns_poll(void)
{
again:

    fDns_result = 0;

    if (eventc == 0) {
//        fprintf(stderr, "(%s:%s:%i) epoll_wait\n", __FILE__, __func__, __LINE__);
        rc = epoll_wait (epoll_fd, events, MAXEVENTS, -1);
        if (rc < 1) {
            fDns_result = rc;
            return;
        }
        eventc = rc;
        ec = -1;
    }

//    fprintf(stderr, "(%s:%s:%i) eventc: %i \n", __FILE__, __func__, __LINE__, eventc);

    while (eventc) {
        ec++;

        e = events[ec].data.ptr;

//        fprintf(stderr, "(%s:%s:%i) e->fd: %i, ec: %i \n", __FILE__, __func__, __LINE__, e->fd, ec);

        if ((events[ec].events & EPOLLERR) || (events[ec].events & EPOLLHUP) || (!(events[ec].events & EPOLLIN))) {
            /* An error has occured on this fd, or the socket is not ready for reading (why were we notified then?) */
            if (e->fd == fDns_socket) {
                fprintf(stderr, "(%s:%s:%i) e->fd == fDns_socket \n", __FILE__, __func__, __LINE__);

            } else if (e->fd == fDns_notify_fd) {
                fprintf(stderr, "(%s:%s:%i) e->fd == fDns_notify_fd\n", __FILE__, __func__, __LINE__);
            } else {
//                fprintf(stderr, "(%s:%s:%i) fDns_socket: %i, fDns_inotify_fd %i, epoll_fd: %i, error on: %i\n", __FILE__, __func__, __LINE__, fDns_socket, fDns_inotify_fd, epoll_fd, e->fd);
                event_delete(e);
            }

            eventc--;

            continue;

        } else if (e->fd == fDns_notify_fd) {
            fprintf(stderr, "(%s:%s:%i) fDns_notify_fd... \n", __FILE__, __func__, __LINE__);

            fDns_notify();

            eventc--;
            continue;

        } else if (e->fd == fDns_socket) {
//            fprintf(stderr, "(%s:%s:%i) fDns_socket... \n", __FILE__, __func__, __LINE__);

            fDns_client_sa_len = sizeof(struct sockaddr_storage);
//            fprintf(stderr, "(%s:%s:%i) got an tcp connection... \n", __FILE__, __func__, __LINE__);
            /* We have a notification on the listening socket, which means one or more incoming connections. */
            while (1) {

                event_new->sas_len = sizeof(struct sockaddr_storage);

                fDns_socket_infd = accept(fDns_socket, (struct sockaddr *) &event_new->sas, &event_new->sas_len);
                if (fDns_socket_infd == -1) {
                    if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) {
                        /* We have processed all incoming connections. */
//                        fprintf(stderr, "(%s:%s:%i) accepted all incomming connections...\n", __FILE__, __func__, __LINE__);
                        break;
                    } else {
                        fprintf(stderr, "(%s:%s:%i) accept: '%s' \n", __FILE__, __func__, __LINE__, strerror(errno));
                        break;
                    }
                }

                if (event_new->sas_len > sizeof(struct sockaddr_storage)) {
                    fprintf(stderr, "(%s:%s:%i) event_new->sas_len %i > sizeof(struct sockaddr_storage) %lu \n", __FILE__, __func__, __LINE__, event_new->sas_len, sizeof(struct sockaddr_storage));
                    close(fDns_socket_infd);
                    continue;
                }

                rc = fDns_tcp_socket_non_blocking(fDns_socket_infd);
                if (rc == -1) {
                    close(fDns_socket_infd);
                    continue;
                }

                event_new->fd = fDns_socket_infd;
                event.data.ptr = event_new;
                event.events = EPOLLIN | EPOLLET;

                rc = epoll_ctl (epoll_fd, EPOLL_CTL_ADD, fDns_socket_infd, &event);
                if (rc == -1) {
                    fprintf(stderr, "(%s:%s:%i) epoll_ctl ADD: '%s' \n", __FILE__, __func__, __LINE__, strerror(errno));
                    close(fDns_socket_infd);
                    continue;
                }

                e = event_new;
                event_new = calloc(1, sizeof(struct event_struct));
                if (event_new == NULL) {
                    fprintf(stderr, "(%s:%s:%i) calloc of struct event_struct: '%s' \n", __FILE__, __func__, __LINE__, strerror(errno));
//.........这里部分代码省略.........

void close_conn( int epoll_fd, int sockfd )
{
    epoll_ctl( epoll_fd, EPOLL_CTL_DEL, sockfd, 0 );
    close( sockfd );
}

int resetEPOLL(int ed, int sd)
{
	epoll_ctl(ed, EPOLL_CTL_DEL, sd, NULL);
	return 0;
}

int main(int argc, const char* argv[])
{
	struct sockaddr_in sAddr;
	struct epoll_event ev;
	struct epoll_event evlist[MAX_EVENTS];
	int ready, i;

	int listensock = 0;

	int result=0, val = 0;

	//init epoll
	if( init_epoll() ){
		exit(-1);
	}
	printf("epoll instance created success! \n");

	//create socket
	if ( -1 == (listensock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)))  {
		perror("create socket error!");
		exit(1);
	}
	
	val = 1;
	result = setsockopt(listensock, SOL_SOCKET, SO_REUSEADDR, &val,sizeof(val));
	if( result < 0 ){
		perror("set socket option error!");
		exit(1);
	}
	
	sAddr.sin_family = AF_INET;
	sAddr.sin_port = htons(g_server_port);
	sAddr.sin_addr.s_addr = INADDR_ANY;

	//now bind the address and port
	result = bind(listensock, (struct sockaddr*)&sAddr, sizeof(sAddr));
	if ( result < 0 ) {
		perror("bind error!");
		exit(1);
	}

	//print server ready info
	printf("Server get ready at port : %d \n", g_server_port);

	result = listen(listensock, 5);
	if( result < 0 ) {
		perror("listen error!");
		exit(1);
	}
	
	setnonblocking(listensock); //set nonblocking

	ev.events = EPOLLIN|EPOLLET; //edge-trigged
        ev.data.fd = listensock;
        if( epoll_ctl(epfd, EPOLL_CTL_ADD, listensock, &ev) == -1 ){
		perror("epoll_ctl add error!");
	        exit(-1);
	}

	//now we will epoll_wait the event
	//
	while(1) {
		//Fetch up to MAX_EVENTS items from the ready list
                //printf("About to epoll_wait() \n");
                ready = epoll_wait(epfd, evlist, MAX_EVENTS, -1);
		if( ready == -1 ){
			if( errno == EINTR ){
				printf("Get a intr signal!\n");
				continue;
			} else {
				perror("epoll_wait error!");
				exit(-1);
			}

		}

		for(i=0;i<ready;i++){
		/*	printf("fd=%d, events: %s%s%s%s \n",evlist[i].data.fd,
				(evlist[i].events & EPOLLIN)? "EPOLLIN ":"",
				(evlist[i].events & EPOLLOUT)? "EPOLLOUT ":"",
				(evlist[i].events & EPOLLHUP)? "EPOLLHUP ":"",
				(evlist[i].events & EPOLLERR)? "EPOLLERR ":"");
		*/
			//deal with events
			if( evlist[i].events & (EPOLLIN|EPOLLPRI) ){
				if( evlist[i].data.fd == listensock ){
					handle_accept(evlist[i].data.fd);
				} else {
					handle_read(evlist[i].data.ptr);
				}

			}
			else if( evlist[i].events & EPOLLOUT){
				handle_write(evlist[i].data.ptr);
			}
			else if( evlist[i].events &(EPOLLHUP|EPOLLERR)){
				printf("Client on descriptor #%d disconnetcted. on epoll_wait() \n", evlist[i].data.fd);
				close_free_echo_data(evlist[i].data.ptr);//free echo data	
			}
		}//end of for ready
//.........这里部分代码省略.........

int
main(int argc, char *argv[])
{
    int epfd, ready, fd, s, j, numOpenFds;
    struct epoll_event ev;
    struct epoll_event evlist[MAX_EVENTS];
    char buf[MAX_BUF];

    if (argc < 2 || strcmp(argv[1], "--help") == 0)
        usageErr("%s file...\n", argv[0]);

    epfd = epoll_create(argc - 1);
    if (epfd == -1)
        errExit("epoll_create");

    /* Open each file on command line, and add it to the "interest
       list" for the epoll instance */

    for (j = 1; j < argc; j++) {
        fd = open(argv[j], O_RDONLY);
        if (fd == -1)
            errExit("open");
        printf("Opened \"%s\" on fd %d\n", argv[j], fd);

        ev.events = EPOLLIN;            /* Only interested in input events */
        ev.data.fd = fd;
        if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev) == -1)
            errExit("epoll_ctl");
    }

    numOpenFds = argc - 1;

    while (numOpenFds > 0) {

        /* Fetch up to MAX_EVENTS items from the ready list of the
           epoll instance */

        printf("About to epoll_wait()\n");
        ready = epoll_wait(epfd, evlist, MAX_EVENTS, -1);
        if (ready == -1) {
            if (errno == EINTR)
                continue;               /* Restart if interrupted by signal */
            else
                errExit("epoll_wait");
        }

        printf("Ready: %d\n", ready);

        /* Deal with returned list of events */

        for (j = 0; j < ready; j++) {
            printf("  fd=%d; events: %s%s%s\n", evlist[j].data.fd,
                    (evlist[j].events & EPOLLIN)  ? "EPOLLIN "  : "",
                    (evlist[j].events & EPOLLHUP) ? "EPOLLHUP " : "",
                    (evlist[j].events & EPOLLERR) ? "EPOLLERR " : "");

            if (evlist[j].events & EPOLLIN) {
                s = read(evlist[j].data.fd, buf, MAX_BUF);
                if (s == -1)
                    errExit("read");
                printf("    read %d bytes: %.*s\n", s, s, buf);

            } else if (evlist[j].events & (EPOLLHUP | EPOLLERR)) {

                /* After the epoll_wait(), EPOLLIN and EPOLLHUP may both have
                   been set. But we'll only get here, and thus close the file
                   descriptor, if EPOLLIN was not set. This ensures that all
                   outstanding input (possibly more than MAX_BUF bytes) is
                   consumed (by further loop iterations) before the file
                   descriptor is closed. */

                printf("    closing fd %d\n", evlist[j].data.fd);
                if (close(evlist[j].data.fd) == -1)
                    errExit("close");
                numOpenFds--;
            }
        }
    }

    printf("All file descriptors closed; bye\n");
    exit(EXIT_SUCCESS);
}

int32_t CNetworkUtil::UnregisterEvent( const int32_t epoll_fd, const int32_t fd)
{
	return epoll_ctl( epoll_fd, EPOLL_CTL_DEL, fd, NULL );
}

int Looper::addFd(int fd, int ident, int events, ALooper_callbackFunc callback, void* data) {
#if DEBUG_CALLBACKS
    ALOGD("%p ~ addFd - fd=%d, ident=%d, events=0x%x, callback=%p, data=%p", this, fd, ident,
            events, callback, data);
#endif

    if (! callback) {
        if (! mAllowNonCallbacks) {
            LOGE("Invalid attempt to set NULL callback but not allowed for this looper.");
            return -1;
        }

        if (ident < 0) {
            LOGE("Invalid attempt to set NULL callback with ident <= 0.");
            return -1;
        }
    }

#ifdef LOOPER_USES_EPOLL
    int epollEvents = 0;
    if (events & ALOOPER_EVENT_INPUT) epollEvents |= EPOLLIN;
    if (events & ALOOPER_EVENT_OUTPUT) epollEvents |= EPOLLOUT;

    { // acquire lock
        AutoMutex _l(mLock);

        Request request;
        request.fd = fd;
        request.ident = ident;
        request.callback = callback;
        request.data = data;

        struct epoll_event eventItem;
        memset(& eventItem, 0, sizeof(epoll_event)); // zero out unused members of data field union
        eventItem.events = epollEvents;
        eventItem.data.fd = fd;

        ssize_t requestIndex = mRequests.indexOfKey(fd);
        if (requestIndex < 0) {
            int epollResult = epoll_ctl(mEpollFd, EPOLL_CTL_ADD, fd, & eventItem);
            if (epollResult < 0) {
                LOGE("Error adding epoll events for fd %d, errno=%d", fd, errno);
                return -1;
            }
            mRequests.add(fd, request);
        } else {
            int epollResult = epoll_ctl(mEpollFd, EPOLL_CTL_MOD, fd, & eventItem);
            if (epollResult < 0) {
                LOGE("Error modifying epoll events for fd %d, errno=%d", fd, errno);
                return -1;
            }
            mRequests.replaceValueAt(requestIndex, request);
        }
    } // release lock
#else
    int pollEvents = 0;
    if (events & ALOOPER_EVENT_INPUT) pollEvents |= POLLIN;
    if (events & ALOOPER_EVENT_OUTPUT) pollEvents |= POLLOUT;

    wakeAndLock(); // acquire lock

    struct pollfd requestedFd;
    requestedFd.fd = fd;
    requestedFd.events = pollEvents;

    Request request;
    request.fd = fd;
    request.ident = ident;
    request.callback = callback;
    request.data = data;
    ssize_t index = getRequestIndexLocked(fd);
    if (index < 0) {
        mRequestedFds.push(requestedFd);
        mRequests.push(request);
    } else {
        mRequestedFds.replaceAt(requestedFd, size_t(index));
        mRequests.replaceAt(request, size_t(index));
    }

    mLock.unlock(); // release lock
#endif
    return 1;
}

static void do_TCP_epoll(TCP_Server *TCP_server)
{
#define MAX_EVENTS 16
    struct epoll_event events[MAX_EVENTS];
    int nfds;

    while ((nfds = epoll_wait(TCP_server->efd, events, MAX_EVENTS, 0)) > 0) {
        int n;

        for (n = 0; n < nfds; ++n) {
            sock_t sock = events[n].data.u64 & 0xFFFFFFFF;
            int status = (events[n].data.u64 >> 32) & 0xFFFF, index = (events[n].data.u64 >> 48);

            if ((events[n].events & EPOLLERR) || (events[n].events & EPOLLHUP)) {
                switch (status) {
                    case TCP_SOCKET_LISTENING: {
                        //should never happen
                        break;
                    }

                    case TCP_SOCKET_INCOMING: {
                        kill_TCP_connection(&TCP_server->incomming_connection_queue[index]);
                        break;
                    }

                    case TCP_SOCKET_UNCONFIRMED: {
                        kill_TCP_connection(&TCP_server->unconfirmed_connection_queue[index]);
                        break;
                    }

                    case TCP_SOCKET_CONFIRMED: {
                        kill_accepted(TCP_server, index);
                        break;
                    }
                }

                continue;
            }


            if (!(events[n].events & EPOLLIN)) {
                continue;
            }

            switch (status) {
                case TCP_SOCKET_LISTENING: {
                    //socket is from socks_listening, accept connection
                    struct sockaddr_storage addr;
                    unsigned int addrlen = sizeof(addr);
                    sock_t sock_new;

                    sock_new = accept(sock, (struct sockaddr *)&addr, &addrlen);

                    int index_new = TCP_server->incomming_connection_queue_index % MAX_INCOMMING_CONNECTIONS;

                    if (!accept_connection(TCP_server, sock_new)) {
                        break;
                    }

                    struct epoll_event ev = {
                        .events = EPOLLIN | EPOLLET,
                        .data.u64 = sock_new | ((uint64_t)TCP_SOCKET_INCOMING << 32) | ((uint64_t)index_new << 48)
                    };

                    if (epoll_ctl(TCP_server->efd, EPOLL_CTL_ADD, sock_new, &ev) == -1) {
                        kill_TCP_connection(&TCP_server->incomming_connection_queue[index_new]);
                        break;
                    }

                    break;
                }

                case TCP_SOCKET_INCOMING: {
                    int index_new;

                    if ((index_new = do_incoming(TCP_server, index)) != -1) {
                        events[n].events = EPOLLIN | EPOLLET;
                        events[n].data.u64 = sock | ((uint64_t)TCP_SOCKET_UNCONFIRMED << 32) | ((uint64_t)index_new << 48);

                        if (epoll_ctl(TCP_server->efd, EPOLL_CTL_MOD, sock, &events[n]) == -1) {
                            kill_TCP_connection(&TCP_server->unconfirmed_connection_queue[index_new]);
                            break;
                        }
                    }

                    break;
                }

                case TCP_SOCKET_UNCONFIRMED: {
                    int index_new;

                    if ((index_new = do_unconfirmed(TCP_server, index)) != -1) {
                        events[n].events = EPOLLIN | EPOLLET;
                        events[n].data.u64 = sock | ((uint64_t)TCP_SOCKET_CONFIRMED << 32) | ((uint64_t)index_new << 48);

                        if (epoll_ctl(TCP_server->efd, EPOLL_CTL_MOD, sock, &events[n]) == -1) {
                            //remove from confirmed connections
                            kill_accepted(TCP_server, index_new);
                            break;
                        }
//.........这里部分代码省略.........

TCP_Server *new_TCP_server(uint8_t ipv6_enabled, uint16_t num_sockets, uint16_t *ports, uint8_t *public_key,
                           uint8_t *secret_key, Onion *onion)
{
    if (num_sockets == 0 || ports == NULL)
        return NULL;

    if (networking_at_startup() != 0) {
        return NULL;
    }

    TCP_Server *temp = calloc(1, sizeof(TCP_Server));

    if (temp == NULL)
        return NULL;

    temp->socks_listening = calloc(num_sockets, sizeof(sock_t));

    if (temp->socks_listening == NULL) {
        free(temp);
        return NULL;
    }

#ifdef TCP_SERVER_USE_EPOLL
    temp->efd = epoll_create1(0);

    if (temp->efd == -1) {
        free(temp);
        return NULL;
    }

#endif

    uint8_t family;

    if (ipv6_enabled) {
        family = AF_INET6;
    } else {
        family = AF_INET;
    }

    uint32_t i;
#ifdef TCP_SERVER_USE_EPOLL
    struct epoll_event ev;
#endif

    for (i = 0; i < num_sockets; ++i) {
        sock_t sock = new_listening_TCP_socket(family, ports[i]);

        if (sock_valid(sock)) {
#ifdef TCP_SERVER_USE_EPOLL
            ev.events = EPOLLIN;
            ev.data.u64 = sock | ((uint64_t)TCP_SOCKET_LISTENING << 32);

            if (epoll_ctl(temp->efd, EPOLL_CTL_ADD, sock, &ev) == -1) {
                continue;
            }

#endif

            temp->socks_listening[temp->num_listening_socks] = sock;
            ++temp->num_listening_socks;
        }
    }

    if (temp->num_listening_socks == 0) {
        free(temp);
        return NULL;
    }

    if (onion) {
        temp->onion = onion;
        set_callback_handle_recv_1(onion, &handle_onion_recv_1, temp);
    }

    memcpy(temp->public_key, public_key, crypto_box_PUBLICKEYBYTES);
    memcpy(temp->secret_key, secret_key, crypto_box_SECRETKEYBYTES);

    bs_list_init(&temp->accepted_key_list, crypto_box_PUBLICKEYBYTES, 8);

    return temp;
}

// Set of threads which talk to client over the connection for doing the needful
// processing. Note that once fd is assigned to a thread all the work on that fd
// is done by that thread. Fair fd usage is expected of the client. First thread
// is special - also does accept [listens for new connections]. It is the only
// thread which does it.
void *
thr_demarshal(void *arg)
{
	cf_socket_cfg *s, *ls;
	// Create my epoll fd, register in the global list.
	struct epoll_event ev;
	int nevents, i, n, epoll_fd;
	cf_clock last_fd_print = 0;

#if defined(USE_SYSTEMTAP)
	uint64_t nodeid = g_config.self_node;
#endif

	// Early stage aborts; these will cause faults in process scope.
	cf_assert(arg, AS_DEMARSHAL, CF_CRITICAL, "invalid argument");
	s = &g_config.socket;
	ls = &g_config.localhost_socket;

#ifdef USE_JEM
	int orig_arena;
	if (0 > (orig_arena = jem_get_arena())) {
		cf_crash(AS_DEMARSHAL, "Failed to get original arena for thr_demarshal()!");
	} else {
		cf_info(AS_DEMARSHAL, "Saved original JEMalloc arena #%d for thr_demarshal()", orig_arena);
	}
#endif

	// Figure out my thread index.
	pthread_t self = pthread_self();
	int thr_id;
	for (thr_id = 0; thr_id < MAX_DEMARSHAL_THREADS; thr_id++) {
		if (0 != pthread_equal(g_demarshal_args->dm_th[thr_id], self))
			break;
	}

	if (thr_id == MAX_DEMARSHAL_THREADS) {
		cf_debug(AS_FABRIC, "Demarshal thread could not figure own ID, bogus, exit, fu!");
		return(0);
	}

	// First thread accepts new connection at interface socket.
	if (thr_id == 0) {
		demarshal_file_handle_init();
		epoll_fd = epoll_create(EPOLL_SZ);
		if (epoll_fd == -1)
			cf_crash(AS_DEMARSHAL, "epoll_create(): %s", cf_strerror(errno));

		memset(&ev, 0, sizeof (ev));
		ev.events = EPOLLIN | EPOLLERR | EPOLLHUP;
		ev.data.fd = s->sock;
		if (0 > epoll_ctl(epoll_fd, EPOLL_CTL_ADD, s->sock, &ev))
			cf_crash(AS_DEMARSHAL, "epoll_ctl(): %s", cf_strerror(errno));
		cf_info(AS_DEMARSHAL, "Service started: socket %s:%d", s->addr, s->port);

		if (ls->sock) {
			ev.events = EPOLLIN | EPOLLERR | EPOLLHUP;
			ev.data.fd = ls->sock;
			if (0 > epoll_ctl(epoll_fd, EPOLL_CTL_ADD, ls->sock, &ev))
			  cf_crash(AS_DEMARSHAL, "epoll_ctl(): %s", cf_strerror(errno));
			cf_info(AS_DEMARSHAL, "Service also listening on localhost socket %s:%d", ls->addr, ls->port);
		}
	}
	else {
		epoll_fd = epoll_create(EPOLL_SZ);
		if (epoll_fd == -1)
			cf_crash(AS_DEMARSHAL, "epoll_create(): %s", cf_strerror(errno));
	}

	g_demarshal_args->epoll_fd[thr_id] = epoll_fd;
	cf_detail(AS_DEMARSHAL, "demarshal thread started: id %d", thr_id);

	int id_cntr = 0;

	// Demarshal transactions from the socket.
	for ( ; ; ) {
		struct epoll_event events[EPOLL_SZ];

		cf_detail(AS_DEMARSHAL, "calling epoll");

		nevents = epoll_wait(epoll_fd, events, EPOLL_SZ, -1);

		if (0 > nevents) {
			cf_debug(AS_DEMARSHAL, "epoll_wait() returned %d ; errno = %d (%s)", nevents, errno, cf_strerror(errno));
		}

		cf_detail(AS_DEMARSHAL, "epoll event received: nevents %d", nevents);

		uint64_t now_ns = cf_getns();
		uint64_t now_ms = now_ns / 1000000;

		// Iterate over all events.
		for (i = 0; i < nevents; i++) {
			if ((s->sock == events[i].data.fd) || (ls->sock == events[i].data.fd)) {
				// Accept new connections on the service socket.
				int csocket = -1;
//.........这里部分代码省略.........

void CepollServer::Run()
{
    //
    static struct epoll_event event,events[EPOLL_SIZE];
    while(true)
    {
        int epoll_events_count = epoll_wait(m_efd,events,EPOLL_SIZE,-1);
        for(int i=0;i<epoll_events_count;i++)
        {
            int userfd = events[i].data.fd;
            if(userfd==m_listenfd)//new user
            {
                sockaddr_in userAddr;
                socklen_t userAddr_len = sizeof(userAddr);
                int newuserfd = accept(m_listenfd,(sockaddr*)&userAddr,&userAddr_len);
                printf("client connection from: %s:% d userfd = %d \n",
                                inet_ntoa(userAddr.sin_addr),
                                ntohs(userAddr.sin_port),
                                newuserfd);
                char name[255]={0};
                read(newuserfd,name,sizeof(name)-1);
                user_fd_map.insert(make_pair(name,newuserfd));
                addfd(newuserfd,name);

            }
            else if(events[i].events&EPOLLIN)
            {
                char buf[1024]={0};
                string strbuf;
                int counter=0;
                while(true)
                {
                    counter = read(userfd,buf,sizeof(buf));
                    if(counter<0)
                        break;
                    strbuf.append(buf,counter);
                    memset(buf,0,sizeof(buf));

                    if(counter<sizeof(buf))
                    {
                        break;
                    }
                }
                if(counter==-1)
                {
                    if(errno==EAGAIN||errno==EWOULDBLOCK)
                    {
                        continue;//buf is empty
                    }
                    else
                    {
                        perror("read failed!");
                    }
                }
                if(counter==0)
                {
                    cout<<"connect is disconnecting!"<<endl;
                    close(userfd);
                    epoll_ctl(m_efd,EPOLL_CTL_DEL,userfd,&events[i]);
                    continue;
                }
                //cout<<"receive buf: "<<strbuf<<endl;
                auto fromfd = strbuf.find_last_of("-");
                auto tofd = strbuf.find_first_of("-");
                auto iter =user_fd_map.find(strbuf.substr(0,tofd));

                //cout<<tofd->second<<endl;
                if(iter!=user_fd_map.end())
                {
                    //string s = strbuf.substr(tofd+1,fromfd-1);
                    strbuf = strbuf.substr(fromfd+1)+"--say-you->: "+strbuf.substr(tofd+1,fromfd-tofd-1);
                    fd_msg_map.insert(make_pair(iter->second,strbuf));
                    event.data.fd=iter->second;
                    event.events = EPOLLOUT | EPOLLET;
                    epoll_ctl(m_efd,EPOLL_CTL_MOD,event.data.fd,&event);
                }
                else//unknown user
                {
                    //TODO:
                    //strbuf clear;
                    strbuf.clear();
                }


            }
            else if(events[i].events&EPOLLOUT)
            {
                char buf[255]={0};
                auto a = fd_msg_map.find(userfd);
                string strtemp;//deal message
                if(a!=fd_msg_map.end())
                {
                    strtemp = a->second;
                    strcpy(buf,strtemp.c_str());
                    fd_msg_map.erase(a);
                    int  pos=0;
                    int  need_be_write_count=sizeof(buf);
                    int counter;
                    while(1)
                    {
//.........这里部分代码省略.........

int serverStart() {
    int pid = fork();
    if(pid ==0 ){
        return 0;
    }

    int serverFd;

    //创建服务器fd
    serverFd = socket(PF_INET, SOCK_STREAM, 0);
    setnonblocking(serverFd);

    //创建epoll，并把serverFd放入监听队列
    int epFd = epoll_create(EPOLL_SIZE);
    struct epoll_event ev, evs[EVENT_ARR];
    ev.data.fd = serverFd;
    ev.events = EPOLLIN | EPOLLET;
    epoll_ctl(epFd, EPOLL_CTL_ADD, serverFd, &ev);

    //绑定服务器端口
    struct sockaddr_in serverAddr;
    socklen_t serverLen = sizeof (struct sockaddr);
    serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    serverAddr.sin_port = htons(PORT);
    serverAddr.sin_family = AF_INET;
    //    if (bind(serverFd, (struct sockaddr *) &serverAddr, sizeof (struct sockaddr))== -1) {
    //        perror("bind");
    //        exit(1);
    //    }
    if (bind(serverFd, (struct sockaddr *) &serverAddr, sizeof (struct sockaddr)) == -1) {
        printf("bind() fail:%s.\n", strerror(errno));
        exit(-1);
    } else {
        printf("bind() success.\n");
    }

    //打开监听
    if (listen(serverFd, BACK_QUEUE)) {
        printf("Listen fail.\n");
        exit(-1);
    } else {
        printf("listen() success.\n");
    }

    //死循环处理
    int clientFd;
    struct sockaddr_in clientAddr;
    socklen_t clientLen;
    char buf[BUF_SIZE];

    while (1) {
        //等待epoll事件的到来，最多取EVENT_ARR个事件
        int nfds = epoll_wait(epFd, evs, EVENT_ARR, 2);
        //处理事件
        for (int i = 0; i < nfds; i++) {
            if (evs[i].data.fd == serverFd && evs[i].data.fd & EPOLLIN) {
                //如果是serverFd,表明有新连接连入
                if ((clientFd = accept(serverFd,
                        (struct sockaddr *) &clientAddr, &clientLen)) < 0) {
                    printf("accept fail.\n");
                }
                printf("Connect from %s:%d\n", inet_ntoa(clientAddr.sin_addr),htons(clientAddr.sin_port));
                setnonblocking(clientFd);
                //注册accept()到的连接
                ev.data.fd = clientFd;
                ev.events = EPOLLIN | EPOLLET;
                epoll_ctl(epFd, EPOLL_CTL_ADD, clientFd, &ev);
            } else if (evs[i].events & EPOLLIN) {
                //如果不是serverFd,则是client的可读
                if ((clientFd = evs[i].data.fd) > 0) {
                    int len = handle_message(clientFd);
                    //先进行试探性读取
                    //int len = read(clientFd, buf, BUF_SIZE);
                    if (len > 0) {
                        continue;
                    } else if (len == 0) {
                        //出发了EPOLLIN事件，却没有可以读取的，表示断线
                        printf("Client closed at %d\n", clientFd);
                        epoll_ctl(epFd, EPOLL_CTL_DEL, clientFd, &ev);
                        close(clientFd);
                        evs[i].data.fd = -1;
                        break;
                    } else if (len == EAGAIN) {
                        continue;
                    } else {
                        //client读取出错
                        printf("read() fail.");
                    }
                }
            } else {
                printf("other event.\n");
            }
        }
    }

    return 0;
}

int handle_read(void* p)
{
	int nread = 0, be_close = 0;
	struct epoll_event ev;
	struct echo_data* ptr = (struct echo_data*)p;
	int fd = ptr->fd;
	char* buf = ptr->rptr;
	char ch;
	int unused = 0;

	//already full, return. and in write mode, to change the trigger
	if( ptr->wptr > ptr->rptr ){//unused space!
		unused = ptr->wptr - ptr->rptr -1; //one byte not used!
	} else {
		unused = ptr->size - (ptr->rptr - ptr->buffer) ;
		if( ptr->wptr == ptr->buffer ){
			unused -= 1; //last one byte not used!
		}
	}
	buf = ptr->rptr;
	
	//printf("read -->\n");

	while(unused>0){
		nread = recv(fd, buf, unused, 0);
		if( nread == -1 ) { //error occur
			if( (errno == EAGAIN) || (errno == EWOULDBLOCK) ) { //no more data to read!
				//printf("\n ----> \n");
			}
			else{
				perror("recv error!");
				//return -1;
				be_close = 1; //error!
			}
			break;
		}
		else if( nread ==0 ){//peer already close
			be_close = 1; //use to write , so do not close it!
			//printf("recv() and get peer closed!\n ");
			break;
		} else {
			ch = buf[nread-1];
			buf[nread-1] = '\0';
			printf("%s%c",buf,ch);
			buf[nread-1] = ch;
			
			//adjuct rptr
			if( ptr->rptr + nread >= ptr->buffer + ptr->size ){
				ptr->rptr = ptr->buffer;
			} else {
				ptr->rptr += nread;
			}
			buf = ptr->rptr; //
			if( ptr->wptr > ptr->rptr ){
				unused = ptr->wptr - ptr->rptr -1; //one byte not used!
			} else {
				unused = ptr->size - (ptr->rptr - ptr->buffer);
				if( ptr->wptr == ptr->buffer ){
					unused -= 1; //last one byte not used!
				}
			}
		}

	}; //end of while

	if( be_close ) {
		printf("Client with descriptor #%d disconnected! in recv() \n",fd);
		close_free_echo_data(ptr);
		//epoll_ctl(epfd, EPOLL_CTL_DEL,fd,NULL); //close will automatically remove it from epoll instance!
	} else {
		ev.data.ptr = ptr;
		ev.events = EPOLLOUT|EPOLLET;
		if( unused > 0 ){
			ev.events |= (EPOLLIN|EPOLLPRI); //can read more!
		}
		if( -1 == epoll_ctl(epfd, EPOLL_CTL_MOD, fd,&ev)) {
			perror("epoll_ctl mod error");
			return -1;
		}
	}
	 
	return 0;
}

Looper::Looper(bool allowNonCallbacks) :
        mAllowNonCallbacks(allowNonCallbacks), mSendingMessage(false),
        mResponseIndex(0), mNextMessageUptime(LLONG_MAX) {
    int wakeFds[2];
    int result = pipe(wakeFds);
    LOG_ALWAYS_FATAL_IF(result != 0, "Could not create wake pipe.  errno=%d", errno);

    mWakeReadPipeFd = wakeFds[0];
    mWakeWritePipeFd = wakeFds[1];

    result = fcntl(mWakeReadPipeFd, F_SETFL, O_NONBLOCK);
    LOG_ALWAYS_FATAL_IF(result != 0, "Could not make wake read pipe non-blocking.  errno=%d",
            errno);

    result = fcntl(mWakeWritePipeFd, F_SETFL, O_NONBLOCK);
    LOG_ALWAYS_FATAL_IF(result != 0, "Could not make wake write pipe non-blocking.  errno=%d",
            errno);

#ifdef LOOPER_USES_EPOLL
    // Allocate the epoll instance and register the wake pipe.
    mEpollFd = epoll_create(EPOLL_SIZE_HINT);
    LOG_ALWAYS_FATAL_IF(mEpollFd < 0, "Could not create epoll instance.  errno=%d", errno);

    struct epoll_event eventItem;
    memset(& eventItem, 0, sizeof(epoll_event)); // zero out unused members of data field union
    eventItem.events = EPOLLIN;
    eventItem.data.fd = mWakeReadPipeFd;
    result = epoll_ctl(mEpollFd, EPOLL_CTL_ADD, mWakeReadPipeFd, & eventItem);
    LOG_ALWAYS_FATAL_IF(result != 0, "Could not add wake read pipe to epoll instance.  errno=%d",
            errno);
#else
    // Add the wake pipe to the head of the request list with a null callback.
    struct pollfd requestedFd;
    requestedFd.fd = mWakeReadPipeFd;
    requestedFd.events = POLLIN;
    mRequestedFds.push(requestedFd);

    Request request;
    request.fd = mWakeReadPipeFd;
    request.callback = NULL;
    request.ident = 0;
    request.data = NULL;
    mRequests.push(request);

    mPolling = false;
    mWaiters = 0;
#endif

#ifdef LOOPER_STATISTICS
    mPendingWakeTime = -1;
    mPendingWakeCount = 0;
    mSampledWakeCycles = 0;
    mSampledWakeCountSum = 0;
    mSampledWakeLatencySum = 0;

    mSampledPolls = 0;
    mSampledZeroPollCount = 0;
    mSampledZeroPollLatencySum = 0;
    mSampledTimeoutPollCount = 0;
    mSampledTimeoutPollLatencySum = 0;
#endif
}

int handle_write(void* p)
{
	int be_close = 0;
	struct echo_data* pt