Re: Developing Proxy Server - Don't have enough speed

From: Jignesh (tosc8_at_yahoo.co.in)
Date: 05/06/03 

Date: 6 May 2003 05:14:15 -0700

"David Schwartz" <davids@webmaster.com> wrote in message news:<b91big$aht$1@nntp.webmaster.com>...
> "Jignesh" <tosc8@yahoo.co.in> wrote in message
> news:a4f50e86.0305030401.6a5d1556@posting.google.com...
>
> > We are developing transparant proxy server on FreeBSD 4.7
>
> > Problem
> > -- we are testing speed of Network Address Translation operation which
> > is about 7 KB/S which is extreamly slow.
> > -- we have configure SQUID proxy server to mesure speed of internet
> > which is about 700 KB/S
> > -- Earlier we thought that less speed is because of LIBNET Libaray so
> > that we remove everything concern to LIBNET & used SOCKET APIs only.
> > -- we have tested same code on LINUX 7.1 which is giving speed of
> > about 300 KB/S which is quite reasonable.
> >
> > please suggest any solution so that speed of Network Address
> > Translation can be increase.
>
> You have to figure out why it's slow before you can make it faster.
> Without seeing your code, there's no way to know. Try profiling it and
> figure out what the problem is.
>
> Is the CPU maxed? If yes, figure out what the CPU is busy doing and then
> try to optimize that. If not, figure out why not -- what is your process
> waiting for?
>
> Sometimes 'strace' or 'ktrace' is useful to figure out where the process
> is spending its time. If it's maxed in kernel CPU, what kernel call is it
> spinning on? If it's not using the CPU, is it blocked in sleep? select?
> what?
>
> You need to figure it out. Maybe the Hash/Map/Vector implementation
> you're using is incredibly bad for the way you're using it. Who knows.
>
> DS

Hello Mr. David,

Here we are sending our latest code for your consideration.

we have done additional testing in following way.

1) We have profile our code & optimise code as much as we can but
there is not much diffrence in speed (still it is 7 KB/S).

To know speed of Intranet

2) Configure NAT on our FreeBSD box, which is giving us speed of
around 700 KB/S.
3) Configure SQUID on our FreeBSD box which is giving us speed of
around 600 KB/S.

We want to build application similar to IPFW - natd as a part of
project.
We aiming to get around 500 KB/S speed of our proxy server. so help
us in finding out bootleneck of our code or suggest any alternative
way.

Here is code.

we are loading ipfw and dropping incoming packets from WAN.

#kldload ipfw && ipfw -q add 65000 allow all from any to any
#ipfw add 1000 drop tcp from any 80 to 192.168.0.9 in via rl0

/* proxyServer.c */

short int status=1;
int sockfd=-1;
int network=-1;

int main(int argc, char *argv[])
{
        pthread_t t_outgoing,t_incoming;
        pthread_attr_t a_thread_attribute;
        pthread_create(&t_outgoing,NULL,(void *)&main_outgoing,NULL);
        pthread_create(&t_incoming,NULL,(void *)&main_incoming,NULL);
        pthread_join(t_outgoing,NULL);
        pthread_join(t_incoming,NULL);
        return 0;
}

------------------------------------------------------------------------------------

For Sending packet we have used two Method.

1) Use LIBNET libaray for packet formation,checksum & sending.
2) Use SOCKET API for sending packet & manual checksum.

Both Method gives almost same speed.

This file convert every packet source ip 192.168.0.9 (Which is our
FreeBSD box).

/* outgoing.c */

extern short int status;
void got_outgoing_packet(u_char *args, const struct pcap_pkthdr
*header, const u_char *packet);
void send_outgoing_packet(char *,int,char[16],u_int16_t);
unsigned short in_cksum(unsigned short *addr, int len);

void got_outgoing_packet(u_char *args, const struct pcap_pkthdr
*header, const u_char *packet1) {
   struct ether_header *eth_hdr;
   struct ip *ip_hdr;
   struct tcphdr *tcp;
   char *payload; /* Packet payload */
   
   int size_ethernet = sizeof(struct ether_header);
   int size_ip = sizeof(struct ip);
   int size_tcp = sizeof(struct tcphdr);
   int len=0;
   
   char *ptr_src1,*ptr_dest1,*ptr_buffer,*ptr1;
   u_int16_t src_port1,dest_port1;
   
   struct in_addr new_saddr;
   char test_src1[] = "172.16.2.2";
   char test_dest1[] = "192.168.0.5";
   
   u_int16_t size_packet;
   char *new_packet;
   static int count = 1; /* Just a counter of how many packets we've
had */
   
   struct tcp_pseudo
   {
      struct in_addr src_addr;
      struct in_addr dst_addr;
      unsigned char dummy;
      unsigned char proto;
      unsigned short length;
   } pseudohead;
   
   struct help_checksum /*struct for checksum calculation*/
   {
      struct tcp_pseudo pshd;
      struct tcphdr tcphd;
      char tcpdata[1024];
   } tcp_chk_construct;
   
   struct in_addr addr1;
   char *checksum_packet;
     
   /* -- Define our packet's attributes -- */
   eth_hdr = (struct ether_header*)(packet1);
   ip_hdr = (struct ip*)(packet1 + size_ethernet);
   tcp = (struct tcphdr*)(packet1 + size_ethernet + size_ip);
   size_tcp = (tcp->th_off)*4;
   payload = (u_char *)(packet1 + size_ethernet + size_ip + size_tcp);
   len = ntohs(ip_hdr->ip_len) - size_ip - size_tcp;
   
   if(ip_hdr->ip_p == IPPROTO_TCP) {
      addr1 = ip_hdr->ip_src;
      ptr_buffer = inet_ntoa(addr1);
      ptr_src1 = (char *) malloc(strlen(ptr_buffer)+1);
      strcpy(ptr_src1,ptr_buffer);
      src_port1 = ntohs(tcp->th_sport);
      
      addr1 = ip_hdr->ip_dst;
      ptr_buffer = inet_ntoa(addr1);
      ptr_dest1 = (char *) malloc(strlen(ptr_buffer)+1);
      strcpy(ptr_dest1,ptr_buffer);
      dest_port1 = ntohs(tcp->th_dport);
      
      //Discards packets of ssh.
      if(src_port1 ==22 || dest_port1 ==22) {
         free(ptr_src1);
         free(ptr_dest1);
         //count++;
         return;
      }
      
      if((strcmp(ptr_src1,test_src1) ==0)) {
         if((strcmp(ptr_dest1,test_dest1) ==0)) {
            printf("%d\t%s:%d\t%s:%d\t%ld\t%ld\n", count, ptr_src1,
src_port1, ptr_dest1, dest_port1,
ntohl(tcp->th_seq),ntohl(tcp->th_ack));

            //Method 1 - Using Libnet
            send_tcp_packet(ip_hdr,tcp,size_tcp,len,"192.168.0.9",
src_port1, ptr_dest1, dest_port1);
            
            
            //Method 2 - Using Socket & Manual CheckSum.

            new_saddr.s_addr = inet_addr("192.168.0.9");
            ip_hdr->ip_src = new_saddr;
            ip_hdr->ip_src.s_addr = inet_addr("192.168.0.9");
            ip_hdr->ip_len = ntohs(ip_hdr->ip_len);
            ip_hdr->ip_off = ntohs(ip_hdr->ip_off);
            
            ip_hdr->ip_sum = 0;
            tcp->th_sum = 0;
            size_packet = ip_hdr->ip_len;
            new_packet = (char *)ip_hdr;
            
            pseudohead.src_addr=ip_hdr->ip_src;
            pseudohead.dst_addr=ip_hdr->ip_dst;
            pseudohead.dummy=0;
            pseudohead.proto=ip_hdr->ip_p;
            pseudohead.length=htons(size_tcp+len);
            
            checksum_packet = (char *) malloc(sizeof(struct
tcp_pseudo)+size_tcp+len);
            memcpy(checksum_packet,&pseudohead,sizeof(struct
tcp_pseudo));
            memcpy(checksum_packet+sizeof(struct
tcp_pseudo),tcp,size_tcp);
            memcpy(checksum_packet+sizeof(struct
tcp_pseudo)+size_tcp,payload,len);
            
            tcp->th_sum=in_cksum((unsigned short
*)checksum_packet,sizeof(struct tcp_pseudo)+size_tcp+len);
            
            ip_hdr->ip_sum = in_cksum((unsigned short
*)ip_hdr,size_ip);
            send_outgoing_packet(new_packet, size_packet, ptr_dest1,
dest_port1);
            free(checksum_packet);
         }
      }
      free(ptr_src1);
      free(ptr_dest1);
   }
   count++;
}

//Used in Method-1
void send_tcp_packet(struct ip *ip_hdr,struct tcphdr *tcp_hdr, int
size_tcp,int len_payload,char *s_ip,short unsigned int s_port,char
*d_ip,short unsigned int d_port){
   int c;
   extern int network;
   struct in_addr new_saddr,new_daddr;
   
   int len_packet = LIBNET_IP_H + size_tcp + len_payload;

   if(network == -1) {
      network = libnet_open_raw_sock(IPPROTO_RAW);
      if (network == -1) {
         libnet_error(LIBNET_ERR_FATAL, "Can't open network.\n");
      }
   }
   new_saddr.s_addr = inet_addr(s_ip);
   ip_hdr->ip_src = new_saddr;
   tcp_hdr->th_sport=htons(s_port);
   
   new_daddr.s_addr = inet_addr(d_ip);
   ip_hdr->ip_dst = new_daddr;
   tcp_hdr->th_dport=htons(d_port);
      
   if (libnet_do_checksum((u_char *)ip_hdr, IPPROTO_TCP,
size_tcp+len_payload) == -1) {
      libnet_error(LIBNET_ERR_FATAL, "libnet_do_checksum failed\n");
   }
      
   c = libnet_write_ip(network, (u_char *)ip_hdr, len_packet);
   if (c < len_packet) {
      libnet_error(LN_ERR_WARNING, "libnet_write_ip only wrote %d
bytes\n", c);
   }
}

//Used in Method-2
unsigned short
in_cksum(unsigned short *addr, int len)
{
        int nleft = len;
        int sum = 0;
        unsigned short *w = addr;
        unsigned short answer = 0;

   while (nleft > 1) {
                sum += *w++;
                nleft -= 2;
        }

        if (nleft == 1) {
                *(unsigned char *)(&answer) = *(unsigned char *)w ;
                sum += answer;
        }
        sum = (sum >> 16) + (sum & 0xffff); /* add hi 16 to low 16 */
        sum += (sum >> 16); /* add carry */
        answer = ~sum; /* truncate to 16 bits */
        return(answer);
}

//Used in Method-2
void send_outgoing_packet(char *start,int size,char *d_addr,u_int16_t
d_port)
{
       struct sockaddr_in address;
       int on = 1;
       extern int sockfd;

       if(sockfd <0) {
          if((sockfd = socket(AF_INET,SOCK_RAW,IPPROTO_RAW)) < 0){
                  perror("Socket creation");
          }
          if((setsockopt(sockfd,IPPROTO_IP,IP_HDRINCL,&on,sizeof(int)))
< 0){
                  perror("setsockopt");
          }
       }

       memset(&address,'\0',sizeof(address));
       address.sin_family = AF_INET;
       address.sin_port = htons(d_port);
       address.sin_addr.s_addr = inet_addr(d_addr);

       if(sendto(sockfd,start,(size_t) size,0,(struct sockaddr
*)&address,sizeof(struct sockaddr))== -1){
         perror("send");
       }
}

int main_outgoing()
{
   //char *dev;
   char dev[] = "rl1";
   bpf_u_int32 mask;
   bpf_u_int32 net;
   
   char errbuf[PCAP_ERRBUF_SIZE];
   
   pcap_t *handle;
   struct pcap_pkthdr header;
   const u_char *packet;
   
   struct bpf_program filter;
   char filter_app[] = "";
   
   int datalink; /* Type of datalink SLIP/PPP/Ethernet */
            
   puts("Outgoing Thread Created");
   
   pcap_lookupnet(dev,&net,&mask,errbuf);
   
   if((handle = pcap_open_live(dev, BUFSIZ, 1, 500, errbuf))== NULL) {
      printf("Error opening device: %s", errbuf);
   }
   
   if((datalink = pcap_datalink(handle)) < 0 ) {
      printf("Datalink error : %s", pcap_geterr(handle));
   }
   
   if( pcap_compile(handle, &filter, filter_app, 0, net) == -1) {
      printf("pcap_compile borqed\n");
      exit(1);
   }
   
   if (pcap_setfilter(handle, &filter) == -1) {
      printf("pcap_setfilter said 'eat ***'\n");
      exit(1);
   }
      
   while(status) {
      while((packet = (char *) pcap_next(handle, &header)) == NULL);
      got_outgoing_packet("", &header,packet);
   }
   
   pcap_close(handle);
        return(0);
}

------------------------------------------------------------------------------------

This file convert every packet Destination ip 172.16.2.2 (Which is our
Test Machine).

/* incoming.c */

extern short int status;
void got_incoming_packet(u_char *args, const struct pcap_pkthdr
*header, const u_char *packet);
void send_incoming_packet(char *,int,char[16],u_int16_t);

void got_incoming_packet(u_char *args, const struct pcap_pkthdr
*header, const u_char *packet1) {
   struct ether_header *eth_hdr;
   struct ip *ip_hdr;
   struct tcphdr *tcp;
   char *payload; /* Packet payload */
   
   int size_ethernet = sizeof(struct ether_header);
   int size_ip = sizeof(struct ip);
   int size_tcp = sizeof(struct tcphdr);
   int len=0;
   
   char *ptr_src1,*ptr_dest1,*ptr_buffer,*ptr1;
   u_int16_t src_port1,dest_port1;
   
   struct in_addr new_saddr;
   char test_src1[] = "192.168.0.5";
   char test_dest1[] = "192.168.0.9";
   
   u_int16_t size_packet;
   char *new_packet;
        
   static int count = 1; /* Just a counter of how
many packets we've had */
   
   struct tcp_pseudo /*the tcp pseudo header*/
   {
      struct in_addr src_addr;
      struct in_addr dst_addr;
      unsigned char dummy;
      unsigned char proto;
      unsigned short length;
   } pseudohead;
   
   struct help_checksum /*struct for checksum calculation*/
   {
      struct tcp_pseudo pshd;
      struct tcphdr tcphd;
      char tcpdata[1024];
   } tcp_chk_construct;
   
   struct in_addr addr1;
   char *checksum_packet;
        
   /* -- Define our packet's attributes -- */
   eth_hdr = (struct ether_header*)(packet1);
   ip_hdr = (struct ip*)(packet1 + size_ethernet);
   tcp = (struct tcphdr*)(packet1 + size_ethernet + size_ip);
   size_tcp = (tcp->th_off)*4;
   payload = (u_char *)(packet1 + size_ethernet + size_ip + size_tcp);
   len = ntohs(ip_hdr->ip_len) - size_ip - size_tcp;
   
   if(ip_hdr->ip_p == IPPROTO_TCP) {
      addr1 = ip_hdr->ip_src;
      ptr_buffer = inet_ntoa(addr1);
      ptr_src1 = (char *) malloc(strlen(ptr_buffer)+1);
      strcpy(ptr_src1,ptr_buffer);
      src_port1 = ntohs(tcp->th_sport);
      
      addr1 = ip_hdr->ip_dst;
      ptr_buffer = inet_ntoa(addr1);
      ptr_dest1 = (char *) malloc(strlen(ptr_buffer)+1);
      strcpy(ptr_dest1,ptr_buffer);
      dest_port1 = ntohs(tcp->th_dport);
   
      //Discards packets of ssh.
      if(src_port1 ==22 || dest_port1 ==22) {
         free(ptr_src1);
         free(ptr_dest1);
         //count++;
         return;
      }
      
      if((strcmp(ptr_src1,test_src1) ==0) /*||
(strcmp(ptr_src1,test_dest1) ==0)*/) {
         if(/*(strcmp(ptr_dest1,test_src1) ==0) ||*/
(strcmp(ptr_dest1,test_dest1) ==0)) {
            printf("%d\t%s:%d\t%s:%d\t%ld\t%ld\n", count, ptr_src1,
src_port1, ptr_dest1, dest_port1,
ntohl(tcp->th_seq),ntohl(tcp->th_ack));

            //Method 1 - Using Libnet
            send_tcp_packet(ip_hdr,tcp,size_tcp,len,ptr_src1,
src_port1, "172.16.2.2", dest_port1);
            
            //Method 2 - Using Socket & Manual CheckSum.
            new_saddr.s_addr = inet_addr("172.16.2.2");
            ip_hdr->ip_dst = new_saddr;
            ip_hdr->ip_len = ntohs(ip_hdr->ip_len);
            ip_hdr->ip_off = ntohs(ip_hdr->ip_off);
            
            ip_hdr->ip_sum = 0;
            
            tcp->th_sum = 0;
            
            size_packet = ip_hdr->ip_len;
            new_packet = (char *)ip_hdr;
            
            pseudohead.src_addr=ip_hdr->ip_src;
            pseudohead.dst_addr=ip_hdr->ip_dst;
            pseudohead.dummy=0;
            pseudohead.proto=ip_hdr->ip_p;
            pseudohead.length=htons(size_tcp+len);
            
            checksum_packet = (char *) malloc(sizeof(struct
tcp_pseudo)+size_tcp+len);
            memcpy(checksum_packet,&pseudohead,sizeof(struct
tcp_pseudo));
            memcpy(checksum_packet+sizeof(struct
tcp_pseudo),tcp,size_tcp);
            memcpy(checksum_packet+sizeof(struct
tcp_pseudo)+size_tcp,payload,len);
            
            tcp->th_sum=in_cksum((unsigned short
*)checksum_packet,sizeof(struct tcp_pseudo)+size_tcp+len);
            ip_hdr->ip_sum = in_cksum((unsigned short
*)ip_hdr,size_ip);
            send_incoming_packet(new_packet, size_packet, ptr_dest1,
dest_port1);
            free(checksum_packet);
         }
      }

      free(ptr_src1);
      free(ptr_dest1);
   }
   count++;
   if(count>1000) status=0;
}

void send_incoming_packet(char *start,int size,char *d_addr,u_int16_t
d_port)
{
       struct sockaddr_in address;
       int on = 1;
       extern int sockfd;

       if(sockfd <0) {
          if((sockfd = socket(AF_INET,SOCK_RAW,IPPROTO_TCP)) < 0) {
                  perror("Socket creation");
          }
          if((setsockopt(sockfd,IPPROTO_IP,IP_HDRINCL,&on,sizeof(int)))
< 0) {
                  perror("setsockopt");
          }
       }
       memset(&address,'\0',sizeof(address));

       address.sin_family = AF_INET;
       address.sin_port = htons(d_port);
       address.sin_addr.s_addr = inet_addr(d_addr);

       if(sendto(sockfd,start,size,0,(struct sockaddr
*)&address,sizeof(struct sockaddr))<size){
         perror("send");
       }
}

int main_incoming()
{
   char dev[] = "rl0";
   bpf_u_int32 mask;
   bpf_u_int32 net;
   
   char errbuf[PCAP_ERRBUF_SIZE];
   
   pcap_t *handle;
   struct pcap_pkthdr header;
   const u_char *packet;
   
   struct bpf_program filter;
   char filter_app[] = "";
   
   int datalink; /* Type of datalink SLIP/PPP/Ethernet */
   puts("Incoming Thread Created");
         
   pcap_lookupnet(dev,&net,&mask,errbuf);
   
   if((handle = pcap_open_live(dev, BUFSIZ, 0, 500, errbuf))== NULL) {
      printf("Error opening device: %s", errbuf);
   }
   
   if((datalink = pcap_datalink(handle)) < 0 ) {
      printf("Datalink error : %s", pcap_geterr(handle));
   }
   
   if( pcap_compile(handle, &filter, filter_app, 0, net) == -1) {
      printf("pcap_compile borqed\n");
      exit(1);
   }
   if (pcap_setfilter(handle, &filter) == -1) {
      printf("pcap_setfilter said 'eat ***'\n");
      exit(1);
   }
      
   while(status) {
      while((packet = (char *) pcap_next(handle, &header)) == NULL);
      got_incoming_packet("", &header,packet);
   }
   
   pcap_close(handle);
        return(0);
}