ipvs的数据报处理

net/ipv4/ipvs/ip_vs_proto_tcp.c
 struct ip_vs_protocol ip_vs_protocol_tcp = {
     .name =            "TCP",
     .protocol =        IPPROTO_TCP,
     .num_states =        IP_VS_TCP_S_LAST,   //协议的状态,比如ESTABLISHED/LISTEN/LAST等
     .dont_defrag =        0,      //禁止重组***,ah/esp=1,tcp/udp=0
     .appcnt =        ATOMIC_INIT(0), //连接数
     .init =            ip_vs_tcp_init, //协议初始化的函数名
     .exit =            ip_vs_tcp_exit, //协议退出时的函数名
     .register_app =        tcp_register_app,   //注册函数
     .unregister_app =    tcp_unregister_app, //注销函数
     .conn_schedule =    tcp_conn_schedule,  //建立连接函数
     .conn_in_get =        tcp_conn_in_get,    //
     .conn_out_get =        tcp_conn_out_get,   //
     .snat_handler =        tcp_snat_handler,   //snat
     .dnat_handler =        tcp_dnat_handler,   //dnat
     .csum_check =        tcp_csum_check,     //
     .state_name =        tcp_state_name,     //
     .state_transition =    tcp_state_transition,   //
     .app_conn_bind =    tcp_app_conn_bind,      //
     .debug_packet =        ip_vs_tcpudp_debug_packet,  //
     .timeout_change =    tcp_timeout_change,         //
     .set_state_timeout =    tcp_set_state_timeout,  //
 };

net/ipv4/ipvs/ip_vs_ctl.c
 /*
  *    Add a service into the service hash table
  */
 static int
 ip_vs_add_service(struct ip_vs_service_user *u, struct ip_vs_service **svc_p)
 {
     int ret = 0;
     struct ip_vs_scheduler *sched = NULL;   //调度控制结构
     struct ip_vs_service *svc = NULL;       //虚拟服务结构
  
     /* increase the module use count */
     ip_vs_use_count_inc();
  
     /* Lookup the scheduler by 'u->sched_name' */
     sched = ip_vs_scheduler_get(u->sched_name); //看看将要添加的服务的调度模块是否存在;如果不存在,尝试加载该模块;如果加载失败,则返回NULL; 这样做能避免不必要的模块被加载,总共有ip_vs_rr.o、ip_vs_wrr.o、ip_vs_lc.o、ip_vs_wlc.o、ip_vs_lblc.o、ip_vs_lblcr.o、ip_vs_dh.o、ip_vs_sh.o、ip_vs_sed.o和ip_vs_nq.o这样10个模块,在我们这个例子中,使用的是ip_vs_rr(轮叫调度Round-Robin Scheduling)
     if (sched == NULL) {
         IP_VS_INFO("Scheduler module ip_vs_%s not found\n",
                u->sched_name);
         ret = -ENOENT;
         goto out_mod_dec;
     }
  
     svc = kzalloc(sizeof(struct ip_vs_service), GFP_ATOMIC); //看来需要为加入ip_vs_svc_table[]准备内存空间了
     if (svc == NULL) {
         IP_VS_DBG(1, "ip_vs_add_service: kmalloc failed.\n");
         ret = -ENOMEM;
         goto out_err;
     }
  
     /* I'm the first user of the service */
     atomic_set(&svc->usecnt, 1);        //使用计数器
     atomic_set(&svc->refcnt, 0);        //引用计数器
  
     svc->protocol = u->protocol;        //协议,地址,端口,防火墙标记,svc标记,超时时长,掩码等的赋值
     svc->addr = u->addr;
     svc->port = u->port;
     svc->fwmark = u->fwmark;
     svc->flags = u->flags;
     svc->timeout = u->timeout * HZ;
     svc->netmask = u->netmask;
  
     INIT_LIST_HEAD(&svc->destinations); //初始化svc的rs列表,现在还没有任何rs,所以先将该双向链表初始化,便于以后rs的添加
     rwlock_init(&svc->sched_lock);      //读写锁初始化
     spin_lock_init(&svc->stats.lock);   //互斥锁初始化
  
     /* Bind the scheduler */
     ret = ip_vs_bind_scheduler(svc, sched); //初始化svc的调度模块,我们的例子里用的是rr模块.将对svc->scheduler赋值,并且运行svc->scheduler->init_service(),对于rr来说,就是执行ip_vs_rr_init_svc(),将svc->sched_data = &svc->destinations;
     if (ret)
         goto out_err;
     sched = NULL;
  
     /* Update the virtual service counters */
     if (svc->port == FTPPORT)           //对于ftp和端口为0的svc计数
         atomic_inc(&ip_vs_ftpsvc_counter);
     else if (svc->port == 0)
         atomic_inc(&ip_vs_nullsvc_counter);
  
     ip_vs_new_estimator(&svc->stats);       //new一个评估性能的estimator
     ip_vs_num_services++;                   //svc计数器
  
     /* Hash the service into the service table */
     write_lock_bh(&__ip_vs_svc_lock);       //上锁
     ip_vs_svc_hash(svc);                    //将svc加入ip_vs_svc_table[]或者ip_vs_svc_fwm_table[]
     write_unlock_bh(&__ip_vs_svc_lock);     //解锁
  
     *svc_p = svc;
     return 0;
  
   out_err:              //出错时,注销掉已经做过的操作
     if (svc != NULL) {
         if (svc->scheduler)
             ip_vs_unbind_scheduler(svc);
         if (svc->inc) {
             local_bh_disable();
             ip_vs_app_inc_put(svc->inc);
             local_bh_enable();
         }
         kfree(svc);
     }
     ip_vs_scheduler_put(sched);
  
   out_mod_dec:
     /* decrease the module use count */
     ip_vs_use_count_dec();
  
     return ret;     //返回错误号
 }

net/ipv4/ipvs/ip_vs_ctl.c
 /*
  *    Add a destination into an existing service
  */
 static int
 ip_vs_add_dest(struct ip_vs_service *svc, struct ip_vs_dest_user *udest)
 {
     struct ip_vs_dest *dest;
     __be32 daddr = udest->addr;
     __be16 dport = udest->port;
     int ret;
  
     EnterFunction(2);
  
     if (udest->weight < 0) {        //权重
         IP_VS_ERR("ip_vs_add_dest(): server weight less than zero\n");
         return -ERANGE;
     }
  
     if (udest->l_threshold > udest->u_threshold) {
         IP_VS_ERR("ip_vs_add_dest(): lower threshold is higher than "
               "upper threshold\n");
         return -ERANGE;
     }
  
     /*
      * Check if the dest already exists in the list
      */
     dest = ip_vs_lookup_dest(svc, daddr, dport);    //该svc上是否已存在相同的rs
     if (dest != NULL) {
         IP_VS_DBG(1, "ip_vs_add_dest(): dest already exists\n");
         return -EEXIST;
     }
  
     /*
      * Check if the dest already exists in the trash and
      * is from the same service
      */
     dest = ip_vs_trash_get_dest(svc, daddr, dport); //判断该rs是否在ip_vs_dest_trash[]里(原来添加过,后来被删除了,就会被放入ip_vs_dest_trash中)
     if (dest != NULL) {     //如果是,则将rs从ip_vs_dest_trash移动到ip_vs_rtable
         IP_VS_DBG(3, "Get destination %u.%u.%u.%u:%u from trash, "
               "dest->refcnt=%d, service %u/%u.%u.%u.%u:%u\n",
               NIPQUAD(daddr), ntohs(dport),
               atomic_read(&dest->refcnt),
               dest->vfwmark,
               NIPQUAD(dest->vaddr),
               ntohs(dest->vport));
         __ip_vs_update_dest(svc, dest, udest);
  
         /*
          * Get the destination from the trash
          */
         list_del(&dest->n_list);    //将dest从ip_vs_dest_trash里删除
  
         ip_vs_new_estimator(&dest->stats);
  
         write_lock_bh(&__ip_vs_svc_lock);
  
         /*
          * Wait until all other svc users go away.
          */
         IP_VS_WAIT_WHILE(atomic_read(&svc->usecnt) > 1);
  
         list_add(&dest->n_list, &svc->destinations);    //将dest.nlist加入到svc->destinations链表中
         svc->num_dests++;
  
         /* call the update_service function of its scheduler */
         svc->scheduler->update_service(svc); //运行svc的scheduler->update_service()函数,对于rr调度来说,就是ip_vs_rr_update_svc()
  
         write_unlock_bh(&__ip_vs_svc_lock);
         return 0;
     }
  
     /*
      * Allocate and initialize the dest structure
      */
     ret = ip_vs_new_dest(svc, udest, &dest);    //将rs加入ip_vs_rtable中
     if (ret) {
         return ret;
     }
  
     /*
      * Add the dest entry into the list
      */
     atomic_inc(&dest->refcnt);
  
     write_lock_bh(&__ip_vs_svc_lock);
  
     /*
      * Wait until all other svc users go away.
      */
     IP_VS_WAIT_WHILE(atomic_read(&svc->usecnt) > 1);
  
     list_add(&dest->n_list, &svc->destinations);    //将dest->n_list加入到svc->destinations链表中
     svc->num_dests++;
  
     /* call the update_service function of its scheduler */
     svc->scheduler->update_service(svc);    //运行svc的scheduler->update_service()函数,对于rr调度来说,就是ip_vs_rr_update_svc()
  
     write_unlock_bh(&__ip_vs_svc_lock);
  
     LeaveFunction(2);
  
     return 0;
 }

net/ipv4/ipvs/ip_vs_core.c
     if (unlikely(!cp)) {        //如果在ip_vs_conn_tab中找不到该连接(也就是该连接是第一次访问vs的话)
         int v;
  
         if (!pp->conn_schedule(skb, pp, &v, &cp))   //利用该协议定义的conn_schedule函数为skb选择合适的rs,并根据skb,pp生成一个新的cp.并将cp添加到ip_vs_conn_tab中.rs的选择请查看相应协议的conn_schedule函数,比如tcp_conn_schedule()
             return v;                               //添加失败时,返回错误码
     }

net/ipv4/ipvs/ip_vs_proto_tcp.c
 static int
 tcp_conn_schedule(struct sk_buff *skb,
           struct ip_vs_protocol *pp,
           int *verdict, struct ip_vs_conn **cpp)
 {
     struct ip_vs_service *svc;
     struct tcphdr _tcph, *th;
  
     th = skb_header_pointer(skb, ip_hdrlen(skb), sizeof(_tcph), &_tcph);    //从skb中得到tcp层头部其实位置,如果头部信息有部分不在当前内存页,则将头部的所有信息先copy到_tcph缓冲区,然后将th指向该缓存区,以保证数据的完整性
     if (th == NULL) {
         *verdict = NF_DROP;
         return 0;
     }
  
     if (th->syn &&          //如果th是同步包(syn),请求建立连接
         (svc = ip_vs_service_get(skb->mark, ip_hdr(skb)->protocol,  //而且需要访问的虚拟服务存在的情况下
                      ip_hdr(skb)->daddr, th->dest))) {
         if (ip_vs_todrop()) {               //先判断本机是否已经超载,用来防止DoS攻击
             /*
              * It seems that we are very loaded.
              * We have to drop this packet :(
              */
             ip_vs_service_put(svc); //如果是,则释放svc的计数,返回0,将verdict设为丢弃
             *verdict = NF_DROP;
             return 0;
         }
  
         /*
          * Let the virtual server select a real server for the
          * incoming connection, and create a connection entry.
          */
         *cpp = ip_vs_schedule(svc, skb);    //此函数将(一)执行svc->scheduler->schedule(),选出dest(真实服务器);(二)kmem_cache_zalloc一个cp;(三)将skb,svc,dest的信息写入到cp中,其中包括将I.cp的目标地址:端口设置为rs的地址:端口;II.设置根据dest.flags设置cp->packet_xmit;(五)再将cp加入到连接列表ip_vs_conn_tab中;将cp返回给cpp
         if (!*cpp) {
             *verdict = ip_vs_leave(svc, skb, pp);
             return 0;
         }
         ip_vs_service_put(svc); //释放svc的计数器
     }
     return 1;
 }

net/ipv4/ipvs/ip_vs_core.c
     if (cp->packet_xmit)                //调用cp的packet_xmit()将数据传送出去,函数是在建立cp的时候,由ip_vs_bind_xmit(cp),根据dest->flags(真实服务器的标记)来决定的,有5种方法ip_vs_nat_xmit,ip_vs_tunnel_xmit,ip_vs_dr_xmit,ip_vs_null_xmit,ip_vs_bypass_xmit
         ret = cp->packet_xmit(skb, cp, pp);
         /* do not touch skb anymore */

net/ipv4/ipvs/ip_vs_xmit.c
 /*
  *      NAT transmitter (only for outside-to-inside nat forwarding)
  *      Not used for related ICMP
  */
 int
 ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
            struct ip_vs_protocol *pp)
 {
     struct rtable *rt;        /* Route to the other host */
     int mtu;
     struct iphdr *iph = ip_hdr(skb);
  
     EnterFunction(10);
  
     /* check if it is a connection of no-client-port */
     if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT)) {      //如果cp中没有客户端端口信息,则从skb中得到cport,并写入到cp中
         __be16 _pt, *p;
         p = skb_header_pointer(skb, iph->ihl*4, sizeof(_pt), &_pt);
         if (p == NULL)
             goto tx_error;
         ip_vs_conn_fill_cport(cp, *p);
         IP_VS_DBG(10, "filled cport=%d\n", ntohs(*p));
     }
  
     if (!(rt = __ip_vs_get_out_rt(cp, RT_TOS(iph->tos))))   //根据cp的目标地址(rip:rport)选择下一跳的路由(发给同网段的哪个mac地址),tos用来指定数据的参数,比如优先级,传输率等
         goto tx_error_icmp;
  
     /* MTU checking */
     mtu = dst_mtu(&rt->u.dst);              //查找网卡的mtu值,如果skb->len大于mtu,而且skb中网络层的标记中表明不能分片的话,发出icmp包(目的地址不可达,frag_needed,mtu)
     if ((skb->len > mtu) && (iph->frag_off & htons(IP_DF))) {
         ip_rt_put(rt);
         icmp_send(skb, ICMP_DEST_UNREACH,ICMP_FRAG_NEEDED, htonl(mtu));
         IP_VS_DBG_RL_PKT(0, pp, skb, 0, "ip_vs_nat_xmit(): frag needed for");
         goto tx_error;
     }
  
     /* copy-on-write the packet before mangling it */
     if (!skb_make_writable(skb, sizeof(struct iphdr))) //内核2.6以后,skb可能存在内存分页,如果skb的头部是分页存放的,不能直接操作,需要将多个分页合并到一个连续的内存空间里(线性化处理).方法就是copy到一个新的地址上,然后将原来的分页空间释放掉.这里看到,也不是处理全部skb,只用处理sizeof(struct iphdr)就可以了
         goto tx_error_put;
  
     if (skb_cow(skb, rt->u.dst.dev->hard_header_len))   // 扩充skb头部空间以容纳硬件MAC头数据
         goto tx_error_put;
  
     /* drop old route */
     dst_release(skb->dst);  //释放skb当前的路由cache
     skb->dst = &rt->u.dst;  //更新路由信息,
  
     /* mangle the packet */
     if (pp->dnat_handler && !pp->dnat_handler(skb, pp, cp)) //修改目的端口为真实目的服务器端口,并重新计算skb的校验和(如果有端口的话,比如tcp/udp协议)
  
         goto tx_error;
     ip_hdr(skb)->daddr = cp->daddr;     //修改目的地址为真实目的服务器地址,并重新计算skb的校验和
     ip_send_check(ip_hdr(skb));
  
     IP_VS_DBG_PKT(10, pp, skb, 0, "After DNAT");
  
     /* FIXME: when application helper enlarges the packet and the length
        is larger than the MTU of outgoing device, there will be still
        MTU problem. */
  
     /* Another hack: avoid icmp_send in ip_fragment */
     skb->local_df = 1;
  
     IP_VS_XMIT(skb, rt);        //发送skb到output
  
     LeaveFunction(10);
     return NF_STOLEN;
  
   tx_error_icmp:
     dst_link_failure(skb);
   tx_error:
     LeaveFunction(10);
     kfree_skb(skb);
     return NF_STOLEN;
   tx_error_put:
     ip_rt_put(rt);
     goto tx_error;
 }

net/ipv4/ipvs/ip_vs_xmit.c
 /*
  *   IP Tunneling transmitter
  *
  *   This function encapsulates the packet in a new IP packet, its
  *   destination will be set to cp->daddr. Most code of this function
  *   is taken from ipip.c.
  *
  *   It is used in VS/TUN cluster. The load balancer selects a real
  *   server from a cluster based on a scheduling algorithm,
  *   encapsulates the request packet and forwards it to the selected
  *   server. For example, all real servers are configured with
  *   "ifconfig tunl0 <Virtual IP Address> up". When the server receives
  *   the encapsulated packet, it will decapsulate the packet, processe
  *   the request and return the response packets directly to the client
  *   without passing the load balancer. This can greatly increase the
  *   scalability of virtual server.
  *
  *   Used for ANY protocol
  */
 int
 ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
           struct ip_vs_protocol *pp)
 {
     struct rtable *rt;            /* Route to the other host */
     struct net_device *tdev;        /* Device to other host */
     struct iphdr  *old_iph = ip_hdr(skb);
     u8     tos = old_iph->tos;
     __be16 df = old_iph->frag_off;
     sk_buff_data_t old_transport_header = skb->transport_header;
     struct iphdr  *iph;            /* Our new IP header */
     unsigned int max_headroom;        /* The extra header space needed */
     int    mtu;
  
     EnterFunction(10);      //debug
  
     if (skb->protocol != htons(ETH_P_IP)) {         //只处理太网的ip协议,其他不管(ATM,ARP,IPX...)
         IP_VS_DBG_RL("ip_vs_tunnel_xmit(): protocol error, "
                  "ETH_P_IP: %d, skb protocol: %d\n",
                  htons(ETH_P_IP), skb->protocol);
         goto tx_error;
     }
  
     if (!(rt = __ip_vs_get_out_rt(cp, RT_TOS(tos))))    //根据cp的目标地址(rip:rport)选择下一跳的路由(发给同网段的哪个mac地址),tos用来指定数据的参数,比如优先级,传输率等
         goto tx_error_icmp;
  
     tdev = rt->u.dst.dev;       //出口设备
  
     mtu = dst_mtu(&rt->u.dst) - sizeof(struct iphdr);   //由于需要在原有数据报上加一个新的网络层头部结构,所以网卡能传输的最大mut要减少一个iphdr大小
     if (mtu < 68) {                 //mtu过小的话,释放rt内存空间,并报错
         ip_rt_put(rt);
         IP_VS_DBG_RL("ip_vs_tunnel_xmit(): mtu less than 68\n");
         goto tx_error;
     }
     if (skb->dst)
         skb->dst->ops->update_pmtu(skb->dst, mtu);      //**如果skb->dst不为空,则将skb->dst的mtu值修改为新的值(减少了一个iphdr长度),这个mtu会影响到数据分片的结果
  
     df |= (old_iph->frag_off & htons(IP_DF));
  
     if ((old_iph->frag_off & htons(IP_DF))              //如果数据报的传输需要分片,而skb的标记位显示该数据不能分片,则释放rt空间后,报错
         && mtu < ntohs(old_iph->tot_len)) {
         icmp_send(skb, ICMP_DEST_UNREACH,ICMP_FRAG_NEEDED, htonl(mtu));
         ip_rt_put(rt);
         IP_VS_DBG_RL("ip_vs_tunnel_xmit(): frag needed\n");
         goto tx_error;
     }
  
     /*
      * Okay, now see if we can stuff it in the buffer as-is.
      */
     max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct iphdr);      //得到能够使用的最大网络层头部空间大小(tdev的加上刚从mtu里挖过来的iphdr
  
     if (skb_headroom(skb) < max_headroom            //如果skb的空间不够,则...;否则clone一个skb,再...;如果clone失败,那就shared(skb),再...  <这里真麻烦啊>
         || skb_cloned(skb) || skb_shared(skb)) {
         struct sk_buff *new_skb =
             skb_realloc_headroom(skb, max_headroom);    //将skb的头部信息复制到新的更大的new_skb内存空间上去
         if (!new_skb) {
             ip_rt_put(rt);
             kfree_skb(skb);
             IP_VS_ERR_RL("ip_vs_tunnel_xmit(): no memory\n");
             return NF_STOLEN;
         }
         kfree_skb(skb);         //释放掉原来小的skb指向的内存
         skb = new_skb;          //将新的new_skb交给skb
         old_iph = ip_hdr(skb);  //用old_iph指向新的skb的网络层头部
     }
  
     skb->transport_header = old_transport_header;
  
     /* fix old IP header checksum */
     ip_send_check(old_iph);         //重新old_iph计算网络层校验值,
  
     skb_push(skb, sizeof(struct iphdr));    //在skb的头部skb->data之前加入sizeof(struct iphdr)的空间,也就是将skb的头部加了一个iphdr的空间,使得之前的old_hdr被封装了.
     skb_reset_network_header(skb);          //由于skb空间改变,整理下skb的指针
     memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));       //用0初始化(&(IPCB(skb)->opt)之后sizeof(IPCB(skb)->opt)大小的内存空间
  
     /* drop old route */
     dst_release(skb->dst);          //释放skb->dst
     skb->dst = &rt->u.dst;          //将skb->dst指向新的路由信息
  
     /*
      *    Push down and install the IPIP header.
      */
     iph            =    ip_hdr(skb);            //对添加了新网络层头部的skb取iph(刚分配的内存空间),然后对iph进行赋值,产生一个新的ip头
     iph->version        =    4;
     iph->ihl        =    sizeof(struct iphdr)>>2;
     iph->frag_off        =    df;
     iph->protocol        =    IPPROTO_IPIP;
     iph->tos        =    tos;
     iph->daddr        =    rt->rt_dst;
     iph->saddr        =    rt->rt_src;
     iph->ttl        =    old_iph->ttl;
     ip_select_ident(iph, &rt->u.dst, NULL);
  
     /* Another hack: avoid icmp_send in ip_fragment */
     skb->local_df = 1;
  
     ip_local_out(skb);  //将skb发往output
  
     LeaveFunction(10);
  
     return NF_STOLEN;
  
   tx_error_icmp:
     dst_link_failure(skb);
   tx_error:
     kfree_skb(skb);
     LeaveFunction(10);
     return NF_STOLEN;
 }

net/ipv4/ipvs/ip_vs_xmit.c
 /*
  *      Direct Routing transmitter
  *      Used for ANY protocol
  */
 int
 ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
           struct ip_vs_protocol *pp)
 {
     struct rtable *rt;            /* Route to the other host */
     struct iphdr  *iph = ip_hdr(skb);
     int    mtu;
  
     EnterFunction(10);
  
     if (!(rt = __ip_vs_get_out_rt(cp, RT_TOS(iph->tos))))   //根据cp的目标地址(rip:rport)选择下一跳的路由(发给同网段的哪个mac地址),tos用来指定数据的参数,比如优先级,传输率等
         goto tx_error_icmp;
  
     /* MTU checking */
     mtu = dst_mtu(&rt->u.dst);  //查找网卡的mtu值,如果skb->len大于mtu,而且skb中网络层的标记中表明不能分片的话,发出icmp包(目的地址不可达,frag_needed,mtu)
     if ((iph->frag_off & htons(IP_DF)) && skb->len > mtu) {
         icmp_send(skb, ICMP_DEST_UNREACH,ICMP_FRAG_NEEDED, htonl(mtu));
         ip_rt_put(rt);
         IP_VS_DBG_RL("ip_vs_dr_xmit(): frag needed\n");
         goto tx_error;
     }
  
     /*
      * Call ip_send_check because we are not sure it is called
      * after ip_defrag. Is copy-on-write needed?
      */
     if (unlikely((skb = skb_share_check(skb, GFP_ATOMIC)) == NULL)) {       //检查skb是否正在被其他模块使用
         ip_rt_put(rt);
         return NF_STOLEN;
     }
     ip_send_check(ip_hdr(skb));     //计算网络层头部校验值
  
     /* drop old route */
     dst_release(skb->dst);          //释放skb原有路由
     skb->dst = &rt->u.dst;          //将skb路由指向刚生成的dst(rt->u.dst),这样下一跳的mac地址就修改为rs的mac了
  
     /* Another hack: avoid icmp_send in ip_fragment */
     skb->local_df = 1;
  
     IP_VS_XMIT(skb, rt);            //发送skb到OUTPUT,用(rt)->u.dst.dev作为出口网络设备
  
     LeaveFunction(10);
     return NF_STOLEN;
  
   tx_error_icmp:
     dst_link_failure(skb);
   tx_error:
     kfree_skb(skb);
     LeaveFunction(10);
     return NF_STOLEN;
 }