Load balancing of upstream mechanism in Nginx

<code><span>typedef</span><span>struct</span> {
    <span>/* 负载均衡的类型 */</span>
    ngx_http_upstream_init_pt        init_upstream;
    <span>/* 负载均衡类型的初始化函数 */</span>
    ngx_http_upstream_init_peer_pt   init;
    <span>/* 指向 ngx_http_upstream_rr_peers_t 结构体 */</span><span>void</span>                            *data;
} ngx_http_upstream_peer_t;</code>

<code><span>/* 服务器结构体 */</span><span>typedef</span><span>struct</span> {
    <span>/* 指向存储 IP 地址的数组，因为同一个域名可能会有多个 IP 地址 */</span>
    ngx_addr_t                      *addrs;
    <span>/* IP 地址数组中元素个数 */</span>
    ngx_uint_t                       naddrs;
    <span>/* 权重 */</span>
    ngx_uint_t                       weight;
    <span>/* 最大失败次数 */</span>
    ngx_uint_t                       max_fails;
    <span>/* 失败时间阈值 */</span>
    time_t                           fail_timeout;

    <span>/* 标志位，若为 1，表示不参与策略选择 */</span><span>unsigned</span>                         down:<span>1</span>;
    <span>/* 标志位，若为 1，表示为备用服务器 */</span><span>unsigned</span>                         backup:<span>1</span>;
} ngx_http_upstream_server_t;</code>

<code>typedef <span>struct</span> {
    <span>/* 后端服务器 IP 地址 */</span><span>struct</span> sockaddr                *sockaddr;
    <span>/* 后端服务器 IP 地址的长度 */</span>
    socklen_t                       socklen;
    <span>/* 后端服务器的名称 */</span>
    ngx_str_t                       name;

    <span>/* 后端服务器当前的权重 */</span>
    ngx_int_t                       current_weight;
    <span>/* 后端服务器有效权重 */</span>
    ngx_int_t                       effective_weight;
    <span>/* 配置项所指定的权重 */</span>
    ngx_int_t                       weight;

    <span>/* 已经失败的次数 */</span>
    ngx_uint_t                      fails;
    <span>/* 访问时间 */</span>
    time_t                          accessed;
    time_t                          <span>checked</span>;

    <span>/* 最大失败次数 */</span>
    ngx_uint_t                      max_fails;
    <span>/* 失败时间阈值 */</span>
    time_t                          fail_timeout;

    <span>/* 后端服务器是否参与策略，若为1，表示不参与 */</span>
    ngx_uint_t                      down;          <span>/* unsigned  down:1; */</span><span>#<span>if</span> (NGX_HTTP_SSL)</span>
    ngx_ssl_session_t              *ssl_session;   <span>/* local to a process */</span><span>#<span>endif</span></span>
} ngx_http_upstream_rr_peer_t;</code>

<code><span>typedef</span><span>struct</span> ngx_http_upstream_rr_peers_s  ngx_http_upstream_rr_peers_t;

<span>struct</span> ngx_http_upstream_rr_peers_s {
    <span>/* 竞选队列中后端服务器的数量 */</span>
    ngx_uint_t                      number;

 <span>/* ngx_mutex_t                    *mutex; */</span><span>/* 所有后端服务器总的权重 */</span>
    ngx_uint_t                      total_weight;

    <span>/* 标志位，若为 1，表示后端服务器仅有一台，此时不需要选择策略 */</span><span>unsigned</span>                        single:<span>1</span>;
    <span>/* 标志位，若为 1，表示所有后端服务器总的权重等于服务器的数量 */</span><span>unsigned</span>                        weighted:<span>1</span>;

    ngx_str_t                      *name;

    <span>/* 后端服务器的链表 */</span>
    ngx_http_upstream_rr_peers_t   *next;

    <span>/* 特定的后端服务器 */</span>
    ngx_http_upstream_rr_peer_t     peer[<span>1</span>];
};</code>

<code><span>typedef</span> struct {
    ngx_http_upstream_rr_peers_t   *peers;
    ngx_uint_t                      current;
    uintptr_t                      *tried;
    uintptr_t                       <span><span>data</span>;</span>
} ngx_http_upstream_rr_peer_data_t;</code>

<code><span>for</span> (i = <span>0</span>; i < umcf->upstreams.nelts; i++) {  

    init = uscfp[i]<span>-></span>peer.init_upstream ? 
        uscfp[i]<span>-></span>peer.<span>init_upstream</span>:                                  ngx_http_upstream_init_round_robin;  

    <span>if</span> (init(cf, uscfp[i]) != NGX_OK) {  
            <span>return</span> NGX_CONF_ERROR;  
    }  
}  </code>

<code>static void  
ngx_http_upstream_init_request(ngx_http_request_t *r)  
{  
<span>...</span><span>if</span> (uscf->peer.init(r, uscf) != NGX_OK) {  
        ngx_http_upstream_finalize_request(r, u,  
                                           NGX_HTTP_INTERNAL_SERVER_ERROR);  
        <span>return</span>;  
    }  

    ngx_http_upstream_connect(r, u);  
}  </code>

<code>ngx_int_t  
ngx_event_connect_peer(ngx_peer_connection_t *pc)  
{  
<span>...</span>/* 调用 ngx_http_upstream_get_round_robin_peer  */
   rc = pc->get(pc, pc->data);  
   <span>if</span> (rc != NGX_OK) {  
       <span>return</span> rc;  
   }  

  s = ngx_socket(pc->sockaddr->sa_family, SOCK_STREAM, <span>0</span>);  
<span>...</span>}  </code>

<code>static void  
ngx_http_upstream_send_request(ngx_http_request_t *r, ngx_http_upstream_t *u)  
{  
<span>...</span><span>if</span> (!u->request_sent && ngx_http_upstream_test_connect(c) != NGX_OK) {  
        /* 测试连接失败 */ 
        ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_ERROR);  
        <span>return</span>;  
    }  
<span>...</span>}  </code>

<code><span>/* 初始化服务器负载均衡列表 */</span>
ngx_int_t
ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
    ngx_http_upstream_srv_conf_t *us)
{
    ngx_url_t                      u;
    ngx_uint_t                     i, j, n, w;
    ngx_http_upstream_server_t    *<span>server</span>;
    ngx_http_upstream_rr_peers_t  *peers, *backup;

    <span>/* 设置 ngx_http_upstream_peer_t 结构体中 init 的回调方法 */</span>
    us->peer.init = ngx_http_upstream_init_round_robin_peer;

    <span>/* 第一种情况：若 upstream 机制中有配置后端服务器 */</span><span>if</span> (us->servers) {
        <span>/* ngx_http_upstream_srv_conf_t us 结构体成员 servers 是一个指向服务器数组 ngx_array_t 的指针，*/</span><span>server</span> = us->servers->elts;

        n = <span>0</span>;
        w = <span>0</span>;
        <span>/* 在这里说明下：一个域名可能会对应多个 IP 地址，upstream 机制中把一个 IP 地址看作一个后端服务器 */</span><span>/* 遍历服务器数组中所有后端服务器，统计非备用后端服务器的 IP 地址总个数(即非备用后端服务器总的个数) 和 总权重 */</span><span>for</span> (i = <span>0</span>; i < us->servers->nelts; i++) {
            <span>/* 若当前服务器是备用服务器，则 continue 跳过以下检查，继续检查下一个服务器 */</span><span>if</span> (<span>server</span>[i].backup) {ngx_http_upstream_peer_t
                <span>continue</span>;
            }

            <span>/* 统计所有非备用后端服务器 IP 地址总的个数(即非备用后端服务器总的个数) */</span>
            n += <span>server</span>[i].naddrs;
            <span>/* 统计所有非备用后端服务器总的权重 */</span>
            w += <span>server</span>[i].naddrs * <span>server</span>[i].weight;
        }

        <span>/* 若 upstream 机制中配置项指令没有设置后端服务器，则出错返回 */</span><span>if</span> (n == <span>0</span>) {
            ngx_log_error(NGX_LOG_EMERG, cf->log, <span>0</span>,
                          <span>"no servers in upstream \"%V\" in %s:%ui"</span>,
                          &us->host, us->file_name, us->line);
            <span>return</span> NGX_ERROR;
        }

        <span>/* 值得注意的是：备用后端服务器列表 和 非备用后端服务器列表 是分开挂载的，因此需要分开设置 */</span><span>/* 为非备用后端服务器分配内存空间 */</span>
        peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
                              + sizeof(ngx_http_upstream_rr_peer_t) * (n - <span>1</span>));
        <span>if</span> (peers == NULL) {
            <span>return</span> NGX_ERROR;
        }

        <span>/* 初始化非备用后端服务器列表 ngx_http_upstream_rr_peers_t 结构体 */</span>
        peers->single = (n == <span>1</span>);<span>/* 表示只有一个非备用后端服务器 */</span>
        peers->number = n;<span>/* 非备用后端服务器总的个数 */</span>
        peers->weighted = (w != n);<span>/* 设置默认权重为 1 或 0 */</span>
        peers->total_weight = w;<span>/* 设置非备用后端服务器总的权重 */</span>
        peers->name = &us->host;<span>/* 非备用后端服务器名称 */</span>        n = <span>0</span>;

        <span>/* 遍历服务器数组中所有后端服务器，初始化非备用后端服务器 */</span><span>for</span> (i = <span>0</span>; i < us->servers->nelts; i++) {
            <span>if</span> (<span>server</span>[i].backup) {<span>/* 若为备用服务器则 continue 跳过 */</span><span>continue</span>;
            }
            <span>/* 以下关于 ngx_http_upstream_rr_peer_t 结构体中三个权重值的说明 */</span><span>/*
             * effective_weight 相当于质量(来源于配置文件配置项的 weight)，current_weight 相当于重量。
             * 前者反应本质，一般是不变的。current_weight 是运行时的动态权值，它的变化基于 effective_weight。
             * 但是 effective_weight 在其对应的 peer 服务异常时，会被调低，
             * 当服务恢复正常时，effective_weight 会逐渐恢复到实际值（配置项的weight）;
             */</span><span>/* 遍历非备用后端服务器所对应 IP 地址数组中的所有 IP 地址(即一个后端服务器域名可能会对应多个 IP 地址) */</span><span>for</span> (j = <span>0</span>; j < <span>server</span>[i].naddrs; j++) {
                <span>/* 为每个非备用后端服务器初始化 */</span>
                peers->peer[n].sockaddr = <span>server</span>[i].addrs[j].sockaddr;<span>/* 设置非备用后端服务器 IP 地址 */</span>
                peers->peer[n].socklen = <span>server</span>[i].addrs[j].socklen;<span>/* 设置非备用后端服务器 IP 地址长度 */</span>
                peers->peer[n].name = <span>server</span>[i].addrs[j].name;<span>/* 设置非备用后端服务器域名 */</span>
                peers->peer[n].weight = <span>server</span>[i].weight;<span>/* 设置非备用后端服务器配置项权重 */</span>
                peers->peer[n].effective_weight = <span>server</span>[i].weight;<span>/* 设置非备用后端服务器有效权重 */</span>
                peers->peer[n].current_weight = <span>0</span>;<span>/* 设置非备用后端服务器当前权重 */</span>
                peers->peer[n].max_fails = <span>server</span>[i].max_fails;<span>/* 设置非备用后端服务器最大失败次数 */</span>
                peers->peer[n].fail_timeout = <span>server</span>[i].fail_timeout;<span>/* 设置非备用后端服务器失败时间阈值 */</span>
                peers->peer[n].down = <span>server</span>[i].down;<span>/* 设置非备用后端服务器 down 标志位，若该标志位为 1，则不参与策略 */</span>
                n++;
            }
        }

        <span>/*
         * 将非备用服务器列表挂载到 ngx_http_upstream_srv_conf_t 结构体成员结构体
         * ngx_http_upstream_peer_t peer 的成员 data 中；
         */</span>
        us->peer.data = peers;

        <span>/* backup servers */</span>        n = <span>0</span>;
        w = <span>0</span>;

         <span>/* 遍历服务器数组中所有后端服务器，统计备用后端服务器的 IP 地址总个数(即备用后端服务器总的个数) 和 总权重 */</span><span>for</span> (i = <span>0</span>; i < us->servers->nelts; i++) {
            <span>if</span> (!<span>server</span>[i].backup) {
                <span>continue</span>;
            }

            n += <span>server</span>[i].naddrs;<span>/* 统计所有备用后端服务器的 IP 地址总的个数 */</span>
            w += <span>server</span>[i].naddrs * <span>server</span>[i].weight;<span>/* 统计所有备用后端服务器总的权重 */</span>
        }

        <span>if</span> (n == <span>0</span>) {<span>/* 若没有备用后端服务器，则直接返回 */</span><span>return</span> NGX_OK;
        }

        <span>/* 分配备用服务器列表的内存空间 */</span>
        backup = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
                              + sizeof(ngx_http_upstream_rr_peer_t) * (n - <span>1</span>));
        <span>if</span> (backup == NULL) {
            <span>return</span> NGX_ERROR;
        }

        peers->single = <span>0</span>;
        <span>/* 初始化备用后端服务器列表 ngx_http_upstream_rr_peers_t 结构体 */</span>
        backup->single = <span>0</span>;
        backup->number = n;
        backup->weighted = (w != n);
        backup->total_weight = w;
        backup->name = &us->host;

        n = <span>0</span>;

        <span>/* 遍历服务器数组中所有后端服务器，初始化备用后端服务器 */</span><span>for</span> (i = <span>0</span>; i < us->servers->nelts; i++) {
            <span>if</span> (!<span>server</span>[i].backup) {<span>/* 若是非备用后端服务器，则 continue 跳过当前后端服务器，检查下一个后端服务器 */</span><span>continue</span>;
            }

            <span>/* 遍历备用后端服务器所对应 IP 地址数组中的所有 IP 地址(即一个后端服务器域名可能会对应多个 IP 地址) */</span><span>for</span> (j = <span>0</span>; j < <span>server</span>[i].naddrs; j++) {
                backup->peer[n].sockaddr = <span>server</span>[i].addrs[j].sockaddr;<span>/* 设置备用后端服务器 IP 地址 */</span>
                backup->peer[n].socklen = <span>server</span>[i].addrs[j].socklen;<span>/* 设置备用后端服务器 IP 地址长度 */</span>
                backup->peer[n].name = <span>server</span>[i].addrs[j].name;<span>/* 设置备用后端服务器域名 */</span>
                backup->peer[n].weight = <span>server</span>[i].weight;<span>/* 设置备用后端服务器配置项权重 */</span>
                backup->peer[n].effective_weight = <span>server</span>[i].weight;<span>/* 设置备用后端服务器有效权重 */</span>
                backup->peer[n].current_weight = <span>0</span>;<span>/* 设置备用后端服务器当前权重 */</span>
                backup->peer[n].max_fails = <span>server</span>[i].max_fails;<span>/* 设置备用后端服务器最大失败次数 */</span>
                backup->peer[n].fail_timeout = <span>server</span>[i].fail_timeout;<span>/* 设置备用后端服务器失败时间阈值 */</span>
                backup->peer[n].down = <span>server</span>[i].down;<span>/* 设置备用后端服务器 down 标志位，若该标志位为 1，则不参与策略 */</span>
                n++;
            }
        }

        <span>/*
         * 将备用服务器列表挂载到 ngx_http_upstream_rr_peers_t 结构体中
         * 的成员 next 中；
         */</span>
        peers->next = backup;

        <span>/* 第一种情况到此返回 */</span><span>return</span> NGX_OK;
    }


     <span>/* 第二种情况：若 upstream 机制中没有直接配置后端服务器，则采用默认的方式 proxy_pass 配置后端服务器地址 */</span><span>/* an upstream implicitly defined by proxy_pass, etc. */</span><span>/* 若端口号为 0，则出错返回 */</span><span>if</span> (us->port == <span>0</span>) {
        ngx_log_error(NGX_LOG_EMERG, cf->log, <span>0</span>,
                      <span>"no port in upstream \"%V\" in %s:%ui"</span>,
                      &us->host, us->file_name, us->line);
        <span>return</span> NGX_ERROR;
    }

    <span>/* 初始化 ngx_url_t 结构体所有成员为 0 */</span>
    ngx_memzero(&u, sizeof(ngx_url_t));

    u.host = us->host;
    u.port = us->port;

    <span>/* 解析 IP 地址 */</span><span>if</span> (ngx_inet_resolve_host(cf->pool, &u) != NGX_OK) {
        <span>if</span> (u.err) {
            ngx_log_error(NGX_LOG_EMERG, cf->log, <span>0</span>,
                          <span>"%s in upstream \"%V\" in %s:%ui"</span>,
                          u.err, &us->host, us->file_name, us->line);
        }

        <span>return</span> NGX_ERROR;
    }

    n = u.naddrs;

    <span>/* 分配非备用后端服务器列表的内存空间 */</span>
    peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_rr_peers_t)
                              + sizeof(ngx_http_upstream_rr_peer_t) * (n - <span>1</span>));
    <span>if</span> (peers == NULL) {
        <span>return</span> NGX_ERROR;
    }

    <span>/* 初始化非备用后端服务器列表 */</span>
    peers->single = (n == <span>1</span>);
    peers->number = n;
    peers->weighted = <span>0</span>;
    peers->total_weight = n;
    peers->name = &us->host;

    <span>for</span> (i = <span>0</span>; i < u.naddrs; i++) {
        peers->peer[i].sockaddr = u.addrs[i].sockaddr;
        peers->peer[i].socklen = u.addrs[i].socklen;
        peers->peer[i].name = u.addrs[i].name;
        peers->peer[i].weight = <span>1</span>;
        peers->peer[i].effective_weight = <span>1</span>;
        peers->peer[i].current_weight = <span>0</span>;
        peers->peer[i].max_fails = <span>1</span>;
        peers->peer[i].fail_timeout = <span>10</span>;
    }

    <span>/* 挂载非备用后端服务器列表 */</span>
    us->peer.data = peers;

    <span>/* implicitly defined upstream has no backup servers */</span><span>return</span> NGX_OK;
}</code>

<code><span>/* 当客户端发起请求时，upstream 机制为本轮选择一个后端服务器做初始化工作 */</span>
ngx_int_t
ngx_http_upstream_init_round_robin_peer(ngx_http_request_t <span>*</span>r,
    ngx_http_upstream_srv_conf_t <span>*</span>us)
{
    ngx_uint_t                         n;
    ngx_http_upstream_rr_peer_data_t  <span>*</span>rrp;

    <span>/* 注意：r->upstream->peer 是 ngx_peer_connection_t 结构体类型 */</span><span>/* 获取当前客户端请求中的 ngx_http_upstream_rr_peer_data_t 结构体 */</span>
    rrp <span>=</span> r<span>-></span>upstream<span>-></span>peer<span>.</span><span>data</span>;

    <span>if</span> (rrp <span>==</span><span>NULL</span>) {
        rrp <span>=</span> ngx_palloc(r<span>-></span>pool, sizeof(ngx_http_upstream_rr_peer_data_t));
        <span>if</span> (rrp <span>==</span><span>NULL</span>) {
            <span>return</span> NGX_ERROR;
        }

        r<span>-></span>upstream<span>-></span>peer<span>.</span><span>data</span><span>=</span> rrp;
    }

    <span>/* 获取非备用后端服务器列表 */</span>
    rrp<span>-></span>peers <span>=</span> us<span>-></span>peer<span>.</span><span>data</span>;
    rrp<span>-></span>current <span>=</span><span>0</span>;<span>/* 若采用遍历方式选择后端服务器时，作为起始节点编号 */</span><span>/* 下面是取值 n，若存在备用后端服务器列表，则 n 的值为非备用后端服务器个数 与 备用后端服务器个数 之间的较大者 */</span>    n <span>=</span> rrp<span>-></span>peers<span>-></span>number;

    <span>if</span> (rrp<span>-></span>peers<span>-></span>next <span>&&</span> rrp<span>-></span>peers<span>-></span>next<span>-></span>number <span>></span> n) {
        n <span>=</span> rrp<span>-></span>peers<span>-></span>next<span>-></span>number;
    }

    <span>/* rrp->tried 是一个位图，在本轮选择中，该位图记录各个后端服务器是否被选择过 */</span><span>/*
     * 如果后端服务器数量 n 不大于 32，则只需在一个 int 中即可记录下所有后端服务器状态；
     * 如果后端服务器数量 n 大于 32，则需在内存池中申请内存来存储所有后端服务器的状态；
     */</span><span>if</span> (n <span><=</span><span>8</span><span>*</span> sizeof(uintptr_t)) {
        rrp<span>-></span>tried <span>=</span><span>&</span>rrp<span>-></span><span>data</span>;
        rrp<span>-></span><span>data</span><span>=</span><span>0</span>;

    } <span>else</span> {
        n <span>=</span> (n <span>+</span> (<span>8</span><span>*</span> sizeof(uintptr_t) <span>-</span><span>1</span>)) <span>/</span> (<span>8</span><span>*</span> sizeof(uintptr_t));

        rrp<span>-></span>tried <span>=</span> ngx_pcalloc(r<span>-></span>pool, n <span>*</span> sizeof(uintptr_t));
        <span>if</span> (rrp<span>-></span>tried <span>==</span><span>NULL</span>) {
            <span>return</span> NGX_ERROR;
        }
    }

    <span>/*
     * 设置 ngx_peer_connection_t 结构体中 get 、free 的回调方法；
     * 设置 ngx_peer_connection_t 结构体中 tries 重试连接的次数为非备用后端服务器的个数；
     */</span>
    r<span>-></span>upstream<span>-></span>peer<span>.</span>get <span>=</span> ngx_http_upstream_get_round_robin_peer;
    r<span>-></span>upstream<span>-></span>peer<span>.</span>free <span>=</span> ngx_http_upstream_free_round_robin_peer;
    r<span>-></span>upstream<span>-></span>peer<span>.</span>tries <span>=</span> rrp<span>-></span>peers<span>-></span>number;
<span>#if</span> (NGX_HTTP_SSL)
    r<span>-></span>upstream<span>-></span>peer<span>.</span>set_session <span>=</span>
                               ngx_http_upstream_set_round_robin_peer_session;
    r<span>-></span>upstream<span>-></span>peer<span>.</span>save_session <span>=</span>
                               ngx_http_upstream_save_round_robin_peer_session;
<span>#endif</span><span>return</span> NGX_OK;
}</code>

<code>/* 选择一个后端服务器来处理请求 */
<span>ngx_int_t</span><span>ngx_http_upstream_get_round_robin_peer</span>(ngx_peer_connection_t *pc, void *<span><span>data</span>)</span>
{
    ngx_http_upstream_rr_peer_data_t  *rrp = <span><span>data</span>;</span>    ngx_int_t                      rc;
    ngx_uint_t                     i, n;
    ngx_http_upstream_rr_peer_t   *peer;
    ngx_http_upstream_rr_peers_t  *peers;

    ngx_log_debug1(<span>NGX_LOG_DEBUG_HTTP</span>, pc->log, <span>0</span>,
                   <span>"get rr peer, try: %ui"</span>, pc->tries);

    /* ngx_lock_mutex(rrp->peers->mutex); */

    pc->cached = <span>0</span>;
    pc->connection = <span>NULL</span>;

    /*
     * 检查 ngx_http_upstream_rr_peers_t 结构体中的 single 标志位;
     * 若 single 标志位为 <span>1</span>，表示只有一台非备用后端服务器，
     * 接着检查该非备用后端服务器的 down 标志位，若 down 标志位为 <span>0</span>，则选择该非备用后端服务器来处理请求；
     * 若 down 标志位为 <span>1</span>, 该非备用后端服务器表示不参与策略选择，
     * 则跳至 goto failed 步骤从备用后端服务器列表中选择后端服务器来处理请求；
     */
    <span>if</span> (rrp->peers->single) {
        peer = &rrp->peers->peer[<span>0</span>];

        <span>if</span> (peer->down) {
            goto failed;
        }

    } <span>else</span> {/* 若 single 标志位为 <span>0</span>，表示不止一台非备用后端服务器 */

        /* there are several peers */

        /* 根据非备用后端服务器的权重来选择一台后端服务器处理请求 */
        peer = ngx_http_upstream_get_peer(rrp);

        <span>if</span> (peer == <span>NULL</span>) {
            /*
             * 若从非备用后端服务器列表中没有选择一台合适的后端服务器处理请求，
             * 则 goto failed 从备用后端服务器列表中选择一台后端服务器来处理请求；
             */
            goto failed;
        }

        ngx_log_debug2(<span>NGX_LOG_DEBUG_HTTP</span>, pc->log, <span>0</span>,
                       <span>"get rr peer, current: %ui %i"</span>,
                       rrp->current, peer->current_weight);
    }

    /*
     * 若从非备用后端服务器列表中已经选到了一台合适的后端服务器处理请求;
     * 则获取该后端服务器的地址信息；
     */
    pc->sockaddr = peer->sockaddr;/* 获取被选中的非备用后端服务器的地址 */
    pc->socklen = peer->socklen;/* 获取被选中的非备用后端服务器的地址长度 */
    pc->name = &peer->name;/* 获取被选中的非备用后端服务器的域名 */

    /* ngx_unlock_mutex(rrp->peers->mutex); */

    /*
     * 检查被选中的非备用后端服务器重试连接的次数为 <span>1</span>，且存在备用后端服务器列表，
     * 则将该非备用后端服务器重试连接的次数设置为 备用后端服务器个数加 <span>1</span>；
     * 否则不用重新设置；
     */
    <span>if</span> (pc->tries == <span>1</span> && rrp->peers->next) {
        pc->tries += rrp->peers->next->number;
    }

    /* 到此，表示已经选择到了一台合适的非备用后端服务器来处理请求，则成功返回 */
    return <span>NGX_OK</span>;

<span>failed</span>:
      /*
       * 若从非备用后端服务器列表中没有选择到后端服务器处理请求，
       * 若存在备用后端服务器，则从备用后端服务器列表中选择一台后端服务器来处理请求；
       */

    peers = rrp->peers;

    /* 若存在备用后端服务器，则从备用后端服务器列表中选择一台后端服务器来处理请求；*/
    <span>if</span> (peers->next) {

        /* ngx_unlock_mutex(peers->mutex); */

        ngx_log_debug0(<span>NGX_LOG_DEBUG_HTTP</span>, pc->log, <span>0</span>, <span>"backup servers"</span>);

        /* 获取备用后端服务器列表 */
        rrp->peers = peers->next;
        /* 把后端服务器重试连接的次数 tries 设置为备用后端服务器个数 number */
        pc->tries = rrp->peers->number;

        /* 计算备用后端服务器在位图中的位置 n */
        n = (rrp->peers->number + (<span>8</span> * sizeof(uintptr_t) - <span>1</span>))
                / (<span>8</span> * sizeof(uintptr_t));

        /* 初始化备用后端服务器在位图 rrp->tried[i] 中的值为 <span>0</span> */
        for (i = <span>0</span>; i < n; i++) {
             rrp->tried[i] = <span>0</span>;
        }

        /* 把备用后端服务器列表当前非备用后端服务器列表递归调用 ngx_http_upstream_get_round_robin_peer 选择一台后端服务器 */
        rc = ngx_http_upstream_get_round_robin_peer(pc, rrp);

        /* 若选择成功则返回 */
        <span>if</span> (rc != <span>NGX_BUSY</span>) {
            return rc;
        }

        /* ngx_lock_mutex(peers->mutex); */
    }

    /*
     * 若从备用后端服务器列表中也没有选择到一台后端服务器处理请求，
     * 则重新设置非备用后端服务器连接失败的次数 fails 为 <span>0</span> ,以便重新被选择；
     */
    /* all peers failed, mark them <span>as</span> live for quick recovery */

    for (i = <span>0</span>; i < peers->number; i++) {
        peers->peer[i].fails = <span>0</span>;
    }

    /* ngx_unlock_mutex(peers->mutex); */

    pc->name = peers->name;

    /* 选择失败，则返回 */
    return <span>NGX_BUSY</span>;
}</code>

<code><span>/* 根据后端服务器的权重来选择一台后端服务器处理请求 */</span><span>static</span> ngx_http_upstream_rr_peer_t *
ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp)
{
    time_t                        now;
    uintptr_t                     m;
    ngx_int_t                     total;
    ngx_uint_t                    i, n;
    ngx_http_upstream_rr_peer_t  *peer, *best;

    now = ngx_time();

    best = <span>NULL</span>;
    total = <span>0</span>;

    <span>/* 遍历后端服务器列表 */</span><span>for</span> (i = <span>0</span>; i < rrp->peers->number; i++) {

        <span>/* 计算当前后端服务器在位图中的位置 n */</span>
        n = i / (<span>8</span> * <span>sizeof</span>(uintptr_t));
        m = (uintptr_t) <span>1</span> << i % (<span>8</span> * <span>sizeof</span>(uintptr_t));

        <span>/* 当前后端服务器在位图中已经有记录，则不再次被选择，即 continue 检查下一个后端服务器 */</span><span>if</span> (rrp->tried[n] & m) {
            <span>continue</span>;
        }

        <span>/* 若当前后端服务器在位图中没有记录，则可能被选中，接着计算其权重 */</span>
        peer = &rrp->peers->peer[i];

        <span>/* 检查当前后端服务器的 down 标志位，若为 1 表示不参与策略选择，则 continue 检查下一个后端服务器 */</span><span>if</span> (peer->down) {
            <span>continue</span>;
        }

        <span>/*
         * 当前后端服务器的 down 标志位为 0,接着检查当前后端服务器连接失败的次数是否已经达到 max_fails；
         * 且睡眠的时间还没到 fail_timeout，则当前后端服务器不被选择，continue 检查下一个后端服务器；
         */</span><span>if</span> (peer->max_fails
            && peer->fails >= peer->max_fails
            && now - peer->checked <= peer->fail_timeout)
        {
            <span>continue</span>;
        }

        <span>/* 若当前后端服务器可能被选中，则计算其权重 */</span><span>/*
         * 在上面初始化过程中 current_weight = 0，effective_weight = weight；
         * 此时，设置当前后端服务器的权重 current_weight 的值为原始值加上 effective_weight；
         * 设置总的权重为原始值加上 effective_weight；
         */</span>
        peer->current_weight += peer->effective_weight;
        total += peer->effective_weight;

        <span>/* 服务器正常，调整 effective_weight 的值 */</span><span>if</span> (peer->effective_weight < peer->weight) {
            peer->effective_weight++;
        }

        <span>/* 若当前后端服务器的权重 current_weight 大于目前 best 服务器的权重，则当前后端服务器被选中 */</span><span>if</span> (best == <span>NULL</span> || peer->current_weight > best->current_weight) {
            best = peer;
        }
    }

    <span>if</span> (best == <span>NULL</span>) {
        <span>return</span><span>NULL</span>;
    }

    <span>/* 计算被选中后端服务器在服务器列表中的位置 i */</span>
    i = best - &rrp->peers->peer[<span>0</span>];

    <span>/* 记录被选中后端服务器在 ngx_http_upstream_rr_peer_data_t 结构体 current 成员的值，在释放后端服务器时会用到该值 */</span>
    rrp->current = i;

    <span>/* 计算被选中后端服务器在位图中的位置 */</span>
    n = i / (<span>8</span> * <span>sizeof</span>(uintptr_t));
    m = (uintptr_t) <span>1</span> << i % (<span>8</span> * <span>sizeof</span>(uintptr_t));

    <span>/* 在位图相应的位置记录被选中后端服务器 */</span>
    rrp->tried[n] |= m;

    <span>/* 更新被选中后端服务器的权重 */</span>
    best->current_weight -= total;

    <span>if</span> (now - best->checked > best->fail_timeout) {
        best->checked = now;
    }

    <span>/* 返回被选中的后端服务器 */</span><span>return</span> best;
}</code>

<code>/* 释放后端服务器 */
<span>void</span><span>ngx_http_upstream_free_round_robin_peer</span>(ngx_peer_connection_t *pc, void *<span><span>data</span>,</span>
    ngx_uint_t state)
{
    ngx_http_upstream_rr_peer_data_t  *rrp = <span><span>data</span>;</span>    time_t                       now;
    ngx_http_upstream_rr_peer_t  *peer;

    ngx_log_debug2(<span>NGX_LOG_DEBUG_HTTP</span>, pc->log, <span>0</span>,
                   <span>"free rr peer %ui %ui"</span>, pc->tries, state);

    /* <span>TODO</span>: <span>NGX_PEER_KEEPALIVE</span> */

    /* 若只有一个后端服务器，则设置 ngx_peer_connection_t 结构体成员 tries 为 <span>0</span>，并 return 返回 */
    <span>if</span> (rrp->peers->single) {
        pc->tries = <span>0</span>;
        return;
    }

    /* 若不止一个后端服务器，则执行以下程序 */

    /* 获取已经被选中的后端服务器 */
    peer = &rrp->peers->peer[rrp->current];

    /*
     * 若在本轮被选中的后端服务器在进行连接测试时失败，或者在处理请求过程中失败，
     * 则需要进行重新选择后端服务器；
     */
    <span>if</span> (state & <span>NGX_PEER_FAILED</span>) {
        now = ngx_time();

        /* ngx_lock_mutex(rrp->peers->mutex); */

        peer->fails++;/* 增加当前后端服务器失败的次数 */
        /* 设置当前后端服务器访问的时间 */
        peer->accessed = now;
        peer->checked = now;

        <span>if</span> (peer->max_fails) {
            /* 由于当前后端服务器失败，表示发生异常，此时降低 effective_weight 的值 */
            peer->effective_weight -= peer->weight / peer->max_fails;
        }

        ngx_log_debug2(<span>NGX_LOG_DEBUG_HTTP</span>, pc->log, <span>0</span>,
                       <span>"free rr peer failed: %ui %i"</span>,
                       rrp->current, peer->effective_weight);

        /* 保证 effective_weight 的值不能小于 <span>0</span> */
        <span>if</span> (peer->effective_weight < <span>0</span>) {
            peer->effective_weight = <span>0</span>;
        }

        /* ngx_unlock_mutex(rrp->peers->mutex); */

    } <span>else</span> {/* 若被选中的后端服务器成功处理请求，并返回，则将其 fails 设置为 <span>0</span> */

        /* mark peer live <span>if</span> check passed */

        /* 若 fail_timeout 时间已过，则将其 fails 设置为 <span>0</span> */
        <span>if</span> (peer->accessed < peer->checked) {
            peer->fails = <span>0</span>;
        }
    }

    /* 减少 tries 的值 */
    <span>if</span> (pc->tries) {
        pc->tries<span>--;</span>
    }

    /* ngx_unlock_mutex(rrp->peers->mutex); */
}</code>

<code><span>static</span> ngx_int_t
ngx_http_upstream_init_ip_hash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us)
{
    <span>/* 调用加权轮询策略的初始化函数 */</span><span>if</span> (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) {
        <span>return</span> NGX_ERROR;
    }

    <span>/* 由于 ngx_http_upstream_init_round_robin 方法的选择后端服务器处理客户请求的初始化函数
     * 为 us->peer.init = ngx_http_upstream_init_round_robin_peer;
     */</span><span>/* 重新设置 ngx_http_upstream_peer_t 结构体中 init 的回调方法为  ngx_http_upstream_init_ip_hash_peer */</span>    us->peer.init = ngx_http_upstream_init_ip_hash_peer;

    <span>return</span> NGX_OK;
}</code>

<code>static ngx_int_t
ngx_http_upstream_init_ip_hash_peer(ngx_http_request_t <span>*</span>r,
    ngx_http_upstream_srv_conf_t <span>*</span>us)
{
    struct sockaddr_in                     <span>*</span>sin;
<span>#if</span> (NGX_HAVE_INET6)
    struct sockaddr_in6                    <span>*</span>sin6;
<span>#endif</span>
    ngx_http_upstream_ip_hash_peer_data_t  <span>*</span>iphp;

    <span>/* 分配 ngx_http_upstream_ip_hash_peer_data_t 结构体内存空间 */</span>
    iphp <span>=</span> ngx_palloc(r<span>-></span>pool, sizeof(ngx_http_upstream_ip_hash_peer_data_t));
    <span>if</span> (iphp <span>==</span><span>NULL</span>) {
        <span>return</span> NGX_ERROR;
    }

    r<span>-></span>upstream<span>-></span>peer<span>.</span><span>data</span><span>=</span><span>&</span>iphp<span>-></span>rrp;

    <span>/* 调用加权轮询策略的初始化函数 ngx_http_upstream_init_round_robin_peer */</span><span>if</span> (ngx_http_upstream_init_round_robin_peer(r, us) <span>!=</span> NGX_OK) {
        <span>return</span> NGX_ERROR;
    }

    <span>/* 设置 IP hash 的决策函数  */</span>
    r<span>-></span>upstream<span>-></span>peer<span>.</span>get <span>=</span> ngx_http_upstream_get_ip_hash_peer;

    switch (r<span>-></span>connection<span>-></span>sockaddr<span>-></span>sa_family) {

    <span>/* 保存客户端 IP 地址 */</span><span>/* IPv4 地址 */</span><span>case</span> AF_INET:
        sin <span>=</span> (struct sockaddr_in <span>*</span>) r<span>-></span>connection<span>-></span>sockaddr;
        iphp<span>-></span>addr <span>=</span> (u_char <span>*</span>) <span>&</span>sin<span>-></span>sin_addr<span>.</span>s_addr;
        iphp<span>-></span>addrlen <span>=</span><span>3</span>;
        break;

    <span>/* IPv6 地址 */</span><span>#if</span> (NGX_HAVE_INET6)
    <span>case</span> AF_INET6:
        sin6 <span>=</span> (struct sockaddr_in6 <span>*</span>) r<span>-></span>connection<span>-></span>sockaddr;
        iphp<span>-></span>addr <span>=</span> (u_char <span>*</span>) <span>&</span>sin6<span>-></span>sin6_addr<span>.</span>s6_addr;
        iphp<span>-></span>addrlen <span>=</span><span>16</span>;
        break;
<span>#endif</span><span>/* 非法地址 */</span>
    default:
        iphp<span>-></span>addr <span>=</span> ngx_http_upstream_ip_hash_pseudo_addr;
        iphp<span>-></span>addrlen <span>=</span><span>3</span>;
    }

    <span>/* 初始化 ngx_http_upstream_ip_hash_peer_data_t结构体成员 */</span>
    iphp<span>-></span>hash <span>=</span><span>89</span>;
    iphp<span>-></span>tries <span>=</span><span>0</span>;
    <span>/* 这个是设置为加权轮询策略的决策函数 */</span>
    iphp<span>-></span>get_rr_peer <span>=</span> ngx_http_upstream_get_round_robin_peer;

    <span>return</span> NGX_OK;
}</code>

<code>/* 选择后端服务器处理请求 */
<span>static</span> ngx_int_t
<span>ngx_http_upstream_get_ip_hash_peer</span>(ngx_peer_connection_t *pc, void *<span><span>data</span>)</span>
{
    ngx_http_upstream_ip_hash_peer_data_t  *iphp = <span><span>data</span>;</span>    time_t                        now;
    ngx_int_t                     w;
    uintptr_t                     m;
    ngx_uint_t                    i, n, p, hash;
    ngx_http_upstream_rr_peer_t  *peer;

    ngx_log_debug1(<span>NGX_LOG_DEBUG_HTTP</span>, pc->log, <span>0</span>,
                   <span>"get ip hash peer, try: %ui"</span>, pc->tries);

    /* <span>TODO</span>: cached */

    /* 若重试连接的次数 tries 大于 <span>20</span>，或 只有一台后端服务器，则直接调用加权轮询策略选择当前后端服务器处理请求 */
    <span>if</span> (iphp->tries > <span>20</span> || iphp->rrp.peers->single) {
        return iphp->get_rr_peer(pc, &iphp->rrp);
    }

    now = ngx_time();

    pc->cached = <span>0</span>;
    pc->connection = <span>NULL</span>;

    hash = iphp->hash;

    for ( ;; ) {

        /* 计算 <span>IP</span> 地址的 hash 值 */
        for (i = <span>0</span>; i < (ngx_uint_t) iphp->addrlen; i++) {
            hash = (hash * <span>113</span> + iphp->addr[i]) % <span>6271</span>;/* hash 函数 */
        }

        /* 以下是根据 hash 值选择合适的后端服务器来处理请求 */

        /* 若 ngx_http_upstream_rr_peers_t 结构体中 weighted 标志位为 <span>1</span>，
         * 表示所有后端服务器的总权重 与 后端服务器的数量 相等，
         * 则被选中的后端服务器在后端服务器列表中的位置为 hash 值与后端服务器数量的余数 p；
         */
        <span>if</span> (!iphp->rrp.peers->weighted) {
            p = hash % iphp->rrp.peers->number;

        } <span>else</span> {
            /* 若 ngx_http_upstream_rr_peers_t 结构体中 weighted 标志位为 <span>0</span>，
             * 首先计算 hash 值与后端服务器总权重的余数 w；
             * 将 w 值减去后端服务器的权重，直到有一个后端服务器使 w 值小于 <span>0</span>，
             * 则选中该后端服务器来处理请求，并记录在后端服务器列表中的位置 p；
             */
            w = hash % iphp->rrp.peers->total_weight;

            for (i = <span>0</span>; i < iphp->rrp.peers->number; i++) {
                w -= iphp->rrp.peers->peer[i].weight;
                <span>if</span> (w < <span>0</span>) {
                    break;
                }
            }

            p = i;
        }

        /* 计算被选中后端服务器在位图中的位置 n */
        n = p / (<span>8</span> * sizeof(uintptr_t));
        m = (uintptr_t) <span>1</span> << p % (<span>8</span> * sizeof(uintptr_t));

        /* 若当前被选中的后端服务器已经在位图记录过，则跳至 goto next 执行 */
        <span>if</span> (iphp->rrp.tried[n] & m) {
            goto next;
        }

        ngx_log_debug2(<span>NGX_LOG_DEBUG_HTTP</span>, pc->log, <span>0</span>,
                       <span>"get ip hash peer, hash: %ui %04XA"</span>, p, m);

        /* 获取当前被选中的后端服务器 */
        peer = &iphp->rrp.peers->peer[p];

        /* ngx_lock_mutex(iphp->rrp.peers->mutex); */

        /* 检查当前被选中后端服务器的 down 标志位，若该标志位为<span>1</span>，则跳至 goto next_try 执行 */
        <span>if</span> (peer->down) {
            goto next_try;
        }

        /* 若 down 标志位为 <span>0</span>，接着检查当前被选中后端服务器失败连接次数是否到达 max_fails，
         * 若已经达到 max_fails 次，并且睡眠时间还没到 fail_timeout，则跳至 goto next_try 执行；
         */
        <span>if</span> (peer->max_fails
            && peer->fails >= peer->max_fails
            && now - peer->checked <= peer->fail_timeout)
        {
            goto next_try;
        }

        /* 若不满足以上条件，则表示选择后方服务器成功 */
        break;

    next_try:

        /* 把当前后端服务器记录在位图中 */
        iphp->rrp.tried[n] |= m;

        /* ngx_unlock_mutex(iphp->rrp.peers->mutex); */

        /* 减少当前后端服务器重试连接的次数 */
        pc->tries<span>--;</span>    next:

        /* tries 重试连接的次数加 <span>1</span>，并判断 tries 是否大于阈值 <span>20</span>，若大于，则采用加权轮询策略 */
        <span>if</span> (++iphp->tries >= <span>20</span>) {
            return iphp->get_rr_peer(pc, &iphp->rrp);
        }
    }

    /* 到此已经成功选择了后端服务器来处理请求 */

    /* 记录当前后端服务器在后端服务器列表中的位置，该位置方便释放后端服务器调用 */
    iphp->rrp.current = p;

    /* 记录当前后端服务器的地址信息 */
    pc->sockaddr = peer->sockaddr;
    pc->socklen = peer->socklen;
    pc->name = &peer->name;

    <span>if</span> (now - peer->checked > peer->fail_timeout) {
        peer->checked = now;
    }

    /* ngx_unlock_mutex(iphp->rrp.peers->mutex); */

    /* 把当前后端服务器记录在位图相应的位置 */
    iphp->rrp.tried[n] |= m;
    /* 记录 hash 值 */
    iphp->hash = hash;

    return <span>NGX_OK</span>;
}</code>