iperf3代码分析(3)

iperf3代码分析(3)

对于iperf测试来说,client和都会启动两个主要的定时器,stats timer和reporter timer, 这两个值通过client启动时候的-i参数来配置,且目前stats timer 等于reporter timer,对应代码中是stats_interval和reporter_interval。

            case 'i':
                /* XXX: could potentially want separate stat collection and reporting intervals,
                   but just set them to be the same for now */
                test->stats_interval = test->reporter_interval = atof(optarg);
                if ((test->stats_interval < MIN_INTERVAL || test->stats_interval > MAX_INTERVAL) && test->stats_interval != 0) {
                    i_errno = IEINTERVAL;
                    return -1;
                }
                break;

通过-t配置的参数是测试总的时间,对应代码中是duration。

            case 't':
                test->duration = atoi(optarg);
                if (test->duration > MAX_TIME || test->duration < 0) {
                    i_errno = IEDURATION;
                    return -1;
                }
		duration_flag = 1;
		client_flag = 1;
                break;

上面的定时器和总的测试时长最大最小限制如下,单位为秒:

#define MIN_INTERVAL 0.1
#define MAX_INTERVAL 60.0
#define MAX_TIME 86400

client和server都是在测试过程中的tmr_run触发定时器回调的,stats和reporter定时器到后对应的回调函数分别为:iperf_stats_callback,iperf_reporter_callback。

让我们先看看iperf_stats_callback的实现。

/**
 * Gather statistics during a test.
 * This function works for both the client and server side.
 */
void
iperf_stats_callback(struct iperf_test *test)
{
    struct iperf_stream *sp;
    struct iperf_stream_result *rp = NULL;
    struct iperf_interval_results *irp, temp;
    struct iperf_time temp_time;
    iperf_size_t total_interval_bytes_transferred = 0;
#if defined(HAVE_SCTP_H)
    struct iperf_sctp_info sctp_info;
#endif /* HAVE_SCTP_H */

    temp.omitted = test->omitting;
    temp.rtt = 0;
    temp.rttvar = 0;
    temp.pmtu = 0;
    // 遍历处理本次测试的所有stream
    SLIST_FOREACH(sp, &test->streams, streams) {
        rp = sp->result
        // 根据测试的角色给bytes_transferred赋值,bytes_sent_this_interval和bytes_received_this_interval都是在具体协议的send/recv函数中累加的。
        // 对于一个stream来说,只能是发送或者接收中的一种而不能同时是两者,所以双向流会分别创建两个方向的stream。
	temp.bytes_transferred = sp->sender ? rp->bytes_sent_this_interval : rp->bytes_received_this_interval;

        // 累加本次interval的所有传输字节数,也就是所有的stream的累加结果,这里直接将sent和received的一起加起来,就不用区分角色了,因为每个角色都仅有一个变量有非0值。
        // Total bytes transferred this interval
	total_interval_bytes_transferred += rp->bytes_sent_this_interval + rp->bytes_received_this_interval;

	irp = TAILQ_LAST(&rp->interval_results, irlisthead);
        // 如果irp非空表示已经有测试结果,则取上一次测试结束时间,否则就是第一个测试,取测试开始时间
        /* result->end_time contains timestamp of previous interval */
        if ( irp != NULL ) /* not the 1st interval */
            memcpy(&temp.interval_start_time, &rp->end_time, sizeof(struct iperf_time));
        else /* or use timestamp from beginning */
            memcpy(&temp.interval_start_time, &rp->start_time, sizeof(struct iperf_time));
        // 取当前时间为本次测试的结束时间
        /* now save time of end of this interval */
        iperf_time_now(&rp->end_time);
        memcpy(&temp.interval_end_time, &rp->end_time, sizeof(struct iperf_time));
        // 计算开始到结束时间的差值
        iperf_time_diff(&temp.interval_start_time, &temp.interval_end_time, &temp_time);
        // 将测试结果转换为秒的格式
        temp.interval_duration = iperf_time_in_secs(&temp_time);
	if (test->protocol->id == Ptcp) {
	    if ( has_tcpinfo()) {
		save_tcpinfo(sp, &temp);
		if (test->sender_has_retransmits == 1) {
                    // tcp测试中发送端支持重传机制,根据has_tcpinfo_retransmits的实现,linux都会有重传机制
                    // 通过get_total_retransmits获取tcp协议栈的重传技术
		    long total_retrans = get_total_retransmits(&temp);
                    // 通过总重传减去上次重传,获得本次interval测试的重传统计
		    temp.interval_retrans = total_retrans - rp->stream_prev_total_retrans;
                    // 更新重传统计,更新上次interval的总统计
		    rp->stream_retrans += temp.interval_retrans;
		    rp->stream_prev_total_retrans = total_retrans;

                    // 获取tcp的发送拥塞控制窗口
		    temp.snd_cwnd = get_snd_cwnd(&temp);
		    if (temp.snd_cwnd > rp->stream_max_snd_cwnd) {
			rp->stream_max_snd_cwnd = temp.snd_cwnd;
		    }

                    // 获取tcp的接发送窗口,也就是接收端通告的窗口大小,tcp报文中的窗口大小(需要乘以tcp三次握手中的缩放因子)
		    temp.snd_wnd = get_snd_wnd(&temp);
		    if (temp.snd_wnd > rp->stream_max_snd_wnd) {
			rp->stream_max_snd_wnd = temp.snd_wnd;
		    }

                    // 获取tcp的rtt大小,并计算最大和最小rtt
		    temp.rtt = get_rtt(&temp);
		    if (temp.rtt > rp->stream_max_rtt) {
			rp->stream_max_rtt = temp.rtt;
		    }
		    if (rp->stream_min_rtt == 0 ||
			temp.rtt < rp->stream_min_rtt) {
			rp->stream_min_rtt = temp.rtt;
		    }

                    // 更新总的rtt和rtt统计个数
		    rp->stream_sum_rtt += temp.rtt;
		    rp->stream_count_rtt++;

                    // 分别获取rttvar, pmtu和reoder的信息
		    temp.rttvar = get_rttvar(&temp);
		    temp.pmtu = get_pmtu(&temp);
		    temp.reorder = get_reorder(&temp);
		    rp->stream_reorder = temp.reorder;
		}
	    }
	} else {
            // 获取interval时间内的报文统计,包括包个数,乱序个数,统计错包
	    if (irp == NULL) {
		temp.interval_packet_count = sp->packet_count;
		temp.interval_outoforder_packets = sp->outoforder_packets;
		temp.interval_cnt_error = sp->cnt_error;
	    } else {
		temp.interval_packet_count = sp->packet_count - irp->packet_count;
		temp.interval_outoforder_packets = sp->outoforder_packets - irp->outoforder_packets;
		temp.interval_cnt_error = sp->cnt_error - irp->cnt_error;
	    }
            // 获取包个数,抖动,乱序,错包个数
	    temp.packet_count = sp->packet_count;
	    temp.jitter = sp->jitter;
	    temp.outoforder_packets = sp->outoforder_packets;
	    temp.cnt_error = sp->cnt_error;
	}

#if defined(HAVE_SCTP_H)
	if (test->protocol->id == Psctp) {
            if (iperf_sctp_get_info(sp, &sctp_info) >= 0) {;
                temp.pmtu = sctp_info.pmtu;
                temp.rtt = sctp_info.rtt;
                temp.snd_cwnd = sctp_info.cwnd;
                temp.snd_wnd = sctp_info.wnd;
                if (temp.snd_cwnd > rp->stream_max_snd_cwnd) {
                    rp->stream_max_snd_cwnd = temp.snd_cwnd;
                }
                if (temp.snd_wnd > rp->stream_max_snd_wnd) {
                    rp->stream_max_snd_wnd = temp.snd_wnd;
                }
                if (temp.rtt >= 0) {
                    temp.rtt = sctp_info.rtt;
                    if (temp.rtt > rp->stream_max_rtt) {
                        rp->stream_max_rtt = temp.rtt;
                    }
                    if (rp->stream_min_rtt == 0 ||
                        temp.rtt < rp->stream_min_rtt) {
                        rp->stream_min_rtt = temp.rtt;
                    }
                    rp->stream_sum_rtt += temp.rtt;
                    rp->stream_count_rtt++;
                }
            }
        }
#endif /* HAVE_SCTP_H */

        // add_to_interval_list把本次interval的结果加入到interval list中
        add_to_interval_list(rp, &temp);
        // 清空stream本次interval的sent和recveied统计
        rp->bytes_sent_this_interval = rp->bytes_received_this_interval = 0;
    }

    /* Verify that total server's throughput is not above specified limit */
    if (test->role == 's') {
        // 如果是服务器,检查服务器的吞吐是否超过配置的特定值,这是通过服务器端命令行配置的参数--server-bitrate-limit
        // total_interval_bytes_transferred是本次interval所有stream的收发报文字节数
	iperf_check_total_rate(test, total_interval_bytes_transferred);
    }
}

下面我们看看上面iperf_check_total_rate的实现,也就是如何判断服务器的速率是否超过限制了。

/* Verify that average traffic is not greater than the specified limit */
void
iperf_check_total_rate(struct iperf_test *test, iperf_size_t last_interval_bytes_transferred)
{
    double seconds;
    uint64_t bits_per_second;
    iperf_size_t total_bytes;
    int i;

    if (test->done || test->settings->bitrate_limit == 0)    // Continue only if check should be done
        return;

    // 下面将last_interval_bytes_transferred赋值到每个索引数组中,并计算是否达到统计次数,如果超过才计算服务器的速率是否超过设置的值

    /* Add last interval's transferred bytes to the array */
    if (++test->bitrate_limit_last_interval_index >= test->settings->bitrate_limit_stats_per_interval)
        test->bitrate_limit_last_interval_index = 0;
    test->bitrate_limit_intervals_traffic_bytes[test->bitrate_limit_last_interval_index] = last_interval_bytes_transferred;

    /* Ensure that enough stats periods passed to allow averaging throughput */
    test->bitrate_limit_stats_count += 1;
    if (test->bitrate_limit_stats_count < test->settings->bitrate_limit_stats_per_interval)
        return;

     /* Calculating total bytes traffic to be averaged */
    for (i = 0, total_bytes = 0; i < test->settings->bitrate_limit_stats_per_interval; i++) {
        total_bytes += test->bitrate_limit_intervals_traffic_bytes[i];
    }

    // 将interval的时间乘以每interval的速率,不应该报文数或者字节数么,怎么是时间呢?
    seconds = test->stats_interval * test->settings->bitrate_limit_stats_per_interval;
    // 计算速率,除以8得到Byte的单位
    bits_per_second = total_bytes * 8 / seconds;
    if (test->debug) {
        iperf_printf(test,"Interval %" PRIu64 " - throughput %" PRIu64 " bps (limit %" PRIu64 ")\n", test->bitrate_limit_stats_count, bits_per_second, test->settings->bitrate_limit);
    }

    // 判断速率是否超过限制,如果超过则bitrate_limit_exceeded 置1
    if (bits_per_second  > test->settings->bitrate_limit) {
        if (iperf_get_verbose(test))
            iperf_err(test, "Total throughput of %" PRIu64 " bps exceeded %" PRIu64 " bps limit", bits_per_second, test->settings->bitrate_limit);
	test->bitrate_limit_exceeded = 1;
    }
}

接下来我们看看iperf_reporter_callback的实现。

/**
 * Main report-printing callback.
 * Prints results either during a test (interval report only) or
 * after the entire test has been run (last interval report plus
 * overall summary).
 */
void
iperf_reporter_callback(struct iperf_test *test)
{
    switch (test->state) {
        case TEST_RUNNING:
        case STREAM_RUNNING:
            // 测试中的报告打印,也就是每个interval的结果,打印每个stream
            /* print interval results for each stream */
            iperf_print_intermediate(test);
            break;
        case TEST_END:
        case DISPLAY_RESULTS:
            // 测试完成的结果打印,打印最后一次interval的结果,以及总的结果
            iperf_print_intermediate(test);
            iperf_print_results(test);
            break;
    }

}

上面的reporter_callback中有两个主要的函数:iperf_print_intermediate和iperf_print_results。让我们看看他们的具体实现代码。

/**
 * Print intermediate results during a test (interval report).
 * Uses print_interval_results to print the results for each stream,
 * then prints an interval summary for all streams in this
 * interval.
 */
static void
iperf_print_intermediate(struct iperf_test *test)
{
    struct iperf_stream *sp = NULL;
    struct iperf_interval_results *irp;
    struct iperf_time temp_time;
    cJSON *json_interval;
    cJSON *json_interval_streams;

    int lower_mode, upper_mode;
    int current_mode;
    int discard_json;

    /*
     * Due to timing oddities, there can be cases, especially on the
     * server side, where at the end of a test there is a fairly short
     * interval with no data transferred.  This could caused by
     * the control and data flows sharing the same path in the network,
     * and having the control messages for stopping the test being
     * queued behind the data packets.
     *
     * We'd like to try to omit that last interval when it happens, to
     * avoid cluttering data and output with useless stuff.
     * So we're going to try to ignore very short intervals (less than
     * 10% of the interval time) that have no data.
     */
    int interval_ok = 0;
    // 遍历所有的stream
    SLIST_FOREACH(sp, &test->streams, streams) {
	irp = TAILQ_LAST(&sp->result->interval_results, irlisthead);
	if (irp) {
            // irp非空说明不是第一个测试interval,计算精确的测试时长。
	    iperf_time_diff(&irp->interval_start_time, &irp->interval_end_time, &temp_time);
            // 转换为秒的格式
	    double interval_len = iperf_time_in_secs(&temp_time);
	    if (test->debug) {
		printf("interval_len %f bytes_transferred %" PRIu64 "\n", interval_len, irp->bytes_transferred);
	    }

	    /*
	     * If the interval is at least 10% the normal interval
	     * length, or if there were actual bytes transferred,
	     * then we want to keep this interval.
	     */
            // 如果interval时间大于配置检测的10%或者有报文传送,则认为interval有效
	    if (interval_len >= test->stats_interval * 0.10 ||
		irp->bytes_transferred > 0) {
		interval_ok = 1;
		if (test->debug) {
		    printf("interval forces keep\n");
		}
	    }
	}
    }
    // interval无效则直接返回不做计算
    if (!interval_ok) {
	if (test->debug) {
	    printf("ignoring short interval with no data\n");
	}
	return;
    }

    /*
     * When we use streamed json, we don't actually need to keep the interval
     * results around unless we're the server and the client requested the server output.
     *
     * This avoids unneeded memory build up for long sessions.
     * 
     * The user can still opt in for all measurement data via the --json-stream-full-output option.
     */
    discard_json = test->json_stream == 1 && !test->json_stream_full_output && !(test->role == 's' && test->get_server_output);
    // json输出处理
    if (test->json_output) {
        json_interval = cJSON_CreateObject();
	if (json_interval == NULL)
	    return;
        if (!discard_json)
	    cJSON_AddItemToArray(test->json_intervals, json_interval);
        json_interval_streams = cJSON_CreateArray();
	if (json_interval_streams == NULL)
	    return;
	cJSON_AddItemToObject(json_interval, "streams", json_interval_streams);
    } else {
        json_interval = NULL;
        json_interval_streams = NULL;
    }

    /*
     * We must to sum streams separately.
     * For bidirectional mode we must to display
     * information about sender and receiver streams.
     * For client side we must handle sender streams
     * firstly and receiver streams for server side.
     * The following design allows us to do this.
    // 对于双向模式分别显示发送端和接收端stream信息,对于单向模式则客户端显示发送stream服务器端显示接收stream
    // 这里的lower_mode和upper_mode是为了迭代显示,双向模式区分客户端和服务器端,单向模式则使用mode的值,具体是iperf_mode枚举
    //enum iperf_mode {
    //	SENDER = 1,
    //	RECEIVER = 0,
    //	BIDIRECTIONAL = -1
    //};
    if (test->mode == BIDIRECTIONAL) {
        if (test->role == 'c') {
            lower_mode = -1;
            upper_mode = 0;
        } else {
            lower_mode = 0;
            upper_mode = 1;
        }
    } else {
        lower_mode = test->mode;
        upper_mode = lower_mode;
    }


    for (current_mode = lower_mode; current_mode <= upper_mode; ++current_mode) {
        char ubuf[UNIT_LEN];
        char nbuf[UNIT_LEN];
        char mbuf[UNIT_LEN];
        char zbuf[] = "          ";

        iperf_size_t bytes = 0;
        double bandwidth;
        int64_t retransmits = 0;
        double start_time, end_time;

        int64_t total_packets = 0, lost_packets = 0;
        double avg_jitter = 0.0, lost_percent;
        // 注意stream_must_be_sender的结果,sender是1*1=1, receiver是0*0=0,bidirectional是(-1)*(-1)=1.
        int stream_must_be_sender = current_mode * current_mode;

        char *sum_name;

        /*  Print stream role just for bidirectional mode. */
        // 双向模式的时候显示TX/RX方向,以及client/server角色,单向不显示
        if (test->mode == BIDIRECTIONAL) {
            snprintf(mbuf, sizeof(mbuf), "[%s-%s]", stream_must_be_sender?"TX":"RX", test->role == 'c'?"C":"S");
        } else {
            mbuf[0] = '\0';
            zbuf[0] = '\0';
        }

        SLIST_FOREACH(sp, &test->streams, streams) {
            // 只有发送方向处理
            if (sp->sender == stream_must_be_sender) {
                print_interval_results(test, sp, json_interval_streams);
                /* sum up all streams */
                irp = TAILQ_LAST(&sp->result->interval_results, irlisthead);
                if (irp == NULL) {
                    iperf_err(test,
                            "iperf_print_intermediate error: interval_results is NULL");
                    return;
                }
                // 所有strema累加的发送字节数
                bytes += irp->bytes_transferred;
                if (test->protocol->id == Ptcp) {
                    if (test->sender_has_retransmits == 1) {
                        // tcp累加重传次数
                        retransmits += irp->interval_retrans;
                    }
                } else {
                    // 其他协议累加interval包个数,丢包个数,以及平均jitter
                    total_packets += irp->interval_packet_count;
                    lost_packets += irp->interval_cnt_error;
                    avg_jitter += irp->jitter;
                }
            }
        }

        /* next build string with sum of all streams */
        if (test->num_streams > 1 || test->json_output) {
            /*
             * With BIDIR give a different JSON object name to the one sent/receive sums.
             * The different name is given to the data sent from the server, which is
             * the "reverse" channel.  This makes sure that the name reported on the server
             * and client are compatible, and the names are the same as with non-bidir,
             * except for when reverse is used.
             */
            sum_name = "sum";
            if (test->mode == BIDIRECTIONAL) {
                if ((test->role == 'c' && !stream_must_be_sender) ||
                    (test->role != 'c' && stream_must_be_sender))
                {
                    sum_name = "sum_bidir_reverse";
                }
            }

            sp = SLIST_FIRST(&test->streams); /* reset back to 1st stream */
            /* Only do this of course if there was a first stream */
            if (sp) {
	    irp = TAILQ_LAST(&sp->result->interval_results, irlisthead);    /* use 1st stream for timing info */

            // 计算interval对应的带宽
	    unit_snprintf(ubuf, UNIT_LEN, (double) bytes, 'A');
	    bandwidth = (double) bytes / (double) irp->interval_duration;
	    unit_snprintf(nbuf, UNIT_LEN, bandwidth, test->settings->unit_format);

	    iperf_time_diff(&sp->result->start_time,&irp->interval_start_time, &temp_time);
	    start_time = iperf_time_in_secs(&temp_time);
	    iperf_time_diff(&sp->result->start_time,&irp->interval_end_time, &temp_time);
	    end_time = iperf_time_in_secs(&temp_time);
            // 根据协议的不同显示不同的信息
                if (test->protocol->id == Ptcp || test->protocol->id == Psctp) {
                    if (test->sender_has_retransmits == 1 && stream_must_be_sender) {
                        /* Interval sum, TCP with retransmits. */
                        if (test->json_output)
                            cJSON_AddItemToObject(json_interval, sum_name, iperf_json_printf("start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  retransmits: %d  omitted: %b sender: %b", (double) start_time, (double) end_time, (double) irp->interval_duration, (int64_t) bytes, bandwidth * 8, retransmits, irp->omitted, stream_must_be_sender)); /* XXX irp->omitted or test->omitting? */
                        else
                            iperf_printf(test, report_sum_bw_retrans_format, mbuf, start_time, end_time, ubuf, nbuf, retransmits, irp->omitted?report_omitted:""); /* XXX irp->omitted or test->omitting? */
                    } else {
                        /* Interval sum, TCP without retransmits. */
                        if (test->json_output)
                            cJSON_AddItemToObject(json_interval, sum_name, iperf_json_printf("start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  omitted: %b sender: %b", (double) start_time, (double) end_time, (double) irp->interval_duration, (int64_t) bytes, bandwidth * 8, test->omitting, stream_must_be_sender));
                        else
                            iperf_printf(test, report_sum_bw_format, mbuf, start_time, end_time, ubuf, nbuf, test->omitting?report_omitted:"");
                    }
                } else {
                    /* Interval sum, UDP. */
                    if (stream_must_be_sender) {
                        if (test->json_output)
                            cJSON_AddItemToObject(json_interval, sum_name, iperf_json_printf("start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  packets: %d  omitted: %b sender: %b", (double) start_time, (double) end_time, (double) irp->interval_duration, (int64_t) bytes, bandwidth * 8, (int64_t) total_packets, test->omitting, stream_must_be_sender));
                        else
                            iperf_printf(test, report_sum_bw_udp_sender_format, mbuf, start_time, end_time, ubuf, nbuf, zbuf, total_packets, test->omitting?report_omitted:"");
                    } else {
                        avg_jitter /= test->num_streams;
                        if (total_packets > 0) {
                            lost_percent = 100.0 * lost_packets / total_packets;
                        }
                        else {
                            lost_percent = 0.0;
                        }
                        if (test->json_output)
                            cJSON_AddItemToObject(json_interval, sum_name, iperf_json_printf("start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  jitter_ms: %f  lost_packets: %d  packets: %d  lost_percent: %f  omitted: %b sender: %b", (double) start_time, (double) end_time, (double) irp->interval_duration, (int64_t) bytes, bandwidth * 8, (double) avg_jitter * 1000.0, (int64_t) lost_packets, (int64_t) total_packets, (double) lost_percent, test->omitting, stream_must_be_sender));
                        else
                            iperf_printf(test, report_sum_bw_udp_format, mbuf, start_time, end_time, ubuf, nbuf, avg_jitter * 1000.0, lost_packets, total_packets, lost_percent, test->omitting?report_omitted:"");
                    }
                }
            }
        }
    }

    if (test->json_stream)
        JSONStream_Output(test, "interval", json_interval);
    if (discard_json)
        cJSON_Delete(json_interval);
}

/**
 * Print overall summary statistics at the end of a test.
 */
static void
// 这个函数显示test对应的总体统计结果
iperf_print_results(struct iperf_test *test)
{

    cJSON *json_summary_streams = NULL;

    int lower_mode, upper_mode;
    int current_mode;

    char *sum_sent_name, *sum_received_name, *sum_name;

    int tmp_sender_has_retransmits = test->sender_has_retransmits;

    /* print final summary for all intervals */
    // 显示头信息
    if (test->json_output) {
        json_summary_streams = cJSON_CreateArray();
	if (json_summary_streams == NULL)
	    return;
	cJSON_AddItemToObject(test->json_end, "streams", json_summary_streams);
    } else {
	iperf_printf(test, "%s", report_bw_separator);
	if (test->verbose)
	    iperf_printf(test, "%s", report_summary);
	if (test->protocol->id == Ptcp || test->protocol->id == Psctp) {
	    if (test->sender_has_retransmits || test->other_side_has_retransmits) {
	        if (test->bidirectional)
	            iperf_printf(test, "%s", report_bw_retrans_header_bidir);
	        else
	            iperf_printf(test, "%s", report_bw_retrans_header);
	    }
	    else {
	        if (test->bidirectional)
	            iperf_printf(test, "%s", report_bw_header_bidir);
	        else
	            iperf_printf(test, "%s", report_bw_header);
	    }
	} else {
	    if (test->bidirectional)
	        iperf_printf(test, "%s", report_bw_udp_header_bidir);
	    else
	        iperf_printf(test, "%s", report_bw_udp_header);
	}
    }

    /*
     * We must to sum streams separately.
     * For bidirectional mode we must to display
     * information about sender and receiver streams.
     * For client side we must handle sender streams
     * firstly and receiver streams for server side.
     * The following design allows us to do this.
     */

    if (test->mode == BIDIRECTIONAL) {
        if (test->role == 'c') {
            lower_mode = -1;
            upper_mode = 0;
        } else {
            lower_mode = 0;
            upper_mode = 1;
        }
    } else {
        lower_mode = test->mode;
        upper_mode = lower_mode;
    }


    for (current_mode = lower_mode; current_mode <= upper_mode; ++current_mode) {
        cJSON *json_summary_stream = NULL;
        int64_t total_retransmits = 0;
        int64_t total_packets = 0, lost_packets = 0;
        int64_t sender_packet_count = 0, receiver_packet_count = 0; /* for this stream, this interval */
        int64_t sender_omitted_packet_count = 0, receiver_omitted_packet_count = 0; /* for this stream, this interval */
        int64_t sender_total_packets = 0, receiver_total_packets = 0; /* running total */
        char ubuf[UNIT_LEN];
        char nbuf[UNIT_LEN];
        struct stat sb;
        char sbuf[UNIT_LEN];
        struct iperf_stream *sp = NULL;
        iperf_size_t bytes_sent, total_sent = 0;
        iperf_size_t bytes_received, total_received = 0;
        double start_time, end_time = 0.0, avg_jitter = 0.0, lost_percent = 0.0;
        double sender_time = 0.0, receiver_time = 0.0;
        struct iperf_time temp_time;
        double bandwidth;

        char mbuf[UNIT_LEN];
        // 注意stream_must_be_sender的结果,sender是1*1=1, receiver是0*0=0,bidirectional是(-1)*(-1)=1.
        int stream_must_be_sender = current_mode * current_mode;


        /*  Print stream role just for bidirectional mode. */

        if (test->mode == BIDIRECTIONAL) {
            snprintf(mbuf, sizeof(mbuf), "[%s-%s]", stream_must_be_sender?"TX":"RX", test->role == 'c'?"C":"S");
        } else {
            mbuf[0] = '\0';
        }

        /* Get sender_has_retransmits for each sender side (client and server) */
        if (test->mode == BIDIRECTIONAL && stream_must_be_sender)
            test->sender_has_retransmits = tmp_sender_has_retransmits;
        else if (test->mode == BIDIRECTIONAL && !stream_must_be_sender)
            test->sender_has_retransmits = test->other_side_has_retransmits;

        start_time = 0.;
        sp = SLIST_FIRST(&test->streams);

        /*
         * If there is at least one stream, then figure out the length of time
         * we were running the tests and print out some statistics about
         * the streams.  It's possible to not have any streams at all
         * if the client got interrupted before it got to do anything.
         *
         * Also note that we try to keep separate values for the sender
         * and receiver ending times.  Earlier iperf (3.1 and earlier)
         * servers didn't send that to the clients, so in this case we fall
         * back to using the client's ending timestamp.  The fallback is
         * basically emulating what iperf 3.1 did.
         */

        if (sp) {
        iperf_time_diff(&sp->result->start_time, &sp->result->end_time, &temp_time);
        end_time = iperf_time_in_secs(&temp_time);
        if (sp->sender) {
            sp->result->sender_time = end_time;
            if (sp->result->receiver_time == 0.0) {
                sp->result->receiver_time = sp->result->sender_time;
            }
        }
        else {
            sp->result->receiver_time = end_time;
            if (sp->result->sender_time == 0.0) {
                sp->result->sender_time = sp->result->receiver_time;
            }
        }
        sender_time = sp->result->sender_time;
        receiver_time = sp->result->receiver_time;
        SLIST_FOREACH(sp, &test->streams, streams) {
            if (sp->sender == stream_must_be_sender) {
                if (test->json_output) {
                    json_summary_stream = cJSON_CreateObject();
                    if (json_summary_stream == NULL)
                        return;
                    cJSON_AddItemToArray(json_summary_streams, json_summary_stream);
                }
                // 记录每条流的发送和接收字节数,累加每条发送和接收的字节数
                bytes_sent = sp->result->bytes_sent - sp->result->bytes_sent_omit;
                bytes_received = sp->result->bytes_received;
                total_sent += bytes_sent;
                total_received += bytes_received;

                // 根据发送和接收的角色,记录发送和接收的报文个数
                if (sp->sender) {
                    sender_packet_count = sp->packet_count;
                    sender_omitted_packet_count = sp->omitted_packet_count;
                    receiver_packet_count = sp->peer_packet_count;
                    receiver_omitted_packet_count = sp->peer_omitted_packet_count;
                }
                else {
                    sender_packet_count = sp->peer_packet_count;
                    sender_omitted_packet_count = sp->peer_omitted_packet_count;
                    receiver_packet_count = sp->packet_count;
                    receiver_omitted_packet_count = sp->omitted_packet_count;
                }

                if (test->protocol->id == Ptcp || test->protocol->id == Psctp) {
                    if (test->sender_has_retransmits) {
                        total_retransmits += sp->result->stream_retrans;
                    }
                } else {
                    /*
                     * Running total of the total number of packets.  Use the sender packet count if we
                     * have it, otherwise use the receiver packet count.
                     */
                    // 记录发送和接收的报文个数,这里主要是udp协议
                    int64_t packet_count = sender_packet_count ? sender_packet_count : receiver_packet_count;
                    total_packets += (packet_count - sp->omitted_packet_count);
                    sender_total_packets += (sender_packet_count - sender_omitted_packet_count);
                    receiver_total_packets += (receiver_packet_count - receiver_omitted_packet_count);
                    lost_packets += sp->cnt_error;
                    if (sp->omitted_cnt_error > -1)
                         lost_packets -= sp->omitted_cnt_error;
                    avg_jitter += sp->jitter;
                }
                // 计算发送字节数和发送时间计算平均带宽
                unit_snprintf(ubuf, UNIT_LEN, (double) bytes_sent, 'A');
                if (sender_time > 0.0) {
                    bandwidth = (double) bytes_sent / (double) sender_time;
                }
                else {
                    bandwidth = 0.0;
                }
                unit_snprintf(nbuf, UNIT_LEN, bandwidth, test->settings->unit_format);
                if (test->protocol->id == Ptcp || test->protocol->id == Psctp) {
                    if (test->sender_has_retransmits || test->protocol->id == Psctp) {
                        /* Sender summary, TCP and SCTP with retransmits. */
                        if (test->json_output)
                            cJSON_AddItemToObject(json_summary_stream, report_sender, iperf_json_printf("socket: %d  start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  retransmits: %d  reorder:  %d  max_snd_cwnd:  %d  max_snd_wnd:  %d  max_rtt:  %d  min_rtt:  %d  mean_rtt:  %d sender: %b", (int64_t) sp->socket, (double) start_time, (double) sender_time, (double) sender_time, (int64_t) bytes_sent, bandwidth * 8, (int64_t) sp->result->stream_retrans, (int64_t) sp->result->stream_reorder, (int64_t) sp->result->stream_max_snd_cwnd, (int64_t) sp->result->stream_max_snd_wnd, (int64_t) sp->result->stream_max_rtt, (int64_t) sp->result->stream_min_rtt, (int64_t) ((sp->result->stream_count_rtt == 0) ? 0 : sp->result->stream_sum_rtt / sp->result->stream_count_rtt), stream_must_be_sender));
                        else
                            if (test->role == 's' && !sp->sender) {
                                if (test->verbose)
                                    iperf_printf(test, report_sender_not_available_format, sp->socket);
                            }
                            else {
                                iperf_printf(test, report_bw_retrans_format, sp->socket, mbuf, start_time, sender_time, ubuf, nbuf, sp->result->stream_retrans, report_sender);
                            }
                    } else {
                        /* Sender summary, TCP and SCTP without retransmits. */
                        if (test->json_output)
                            cJSON_AddItemToObject(json_summary_stream, report_sender, iperf_json_printf("socket: %d  start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f sender: %b", (int64_t) sp->socket, (double) start_time, (double) sender_time, (double) sender_time, (int64_t) bytes_sent, bandwidth * 8,  stream_must_be_sender));
                        else
                            if (test->role == 's' && !sp->sender) {
                                if (test->verbose)
                                    iperf_printf(test, report_sender_not_available_format, sp->socket);
                            }
                            else {
                                iperf_printf(test, report_bw_format, sp->socket, mbuf, start_time, sender_time, ubuf, nbuf, report_sender);
                            }
                    }
                } else {
                    /* Sender summary, UDP. */
                    if (sender_packet_count - sender_omitted_packet_count > 0) {
                        lost_percent = 100.0 * (sp->cnt_error - sp->omitted_cnt_error) / (sender_packet_count - sender_omitted_packet_count);
                    }
                    else {
                        lost_percent = 0.0;
                    }
                    if (test->json_output) {
                        /*
                         * For historical reasons, we only emit one JSON
                         * object for the UDP summary, and it contains
                         * information for both the sender and receiver
                         * side.
                         *
                         * The JSON format as currently defined only includes one
                         * value for the number of packets.  We usually want that
                         * to be the sender's value (how many packets were sent
                         * by the sender).  However this value might not be
                         * available on the receiver in certain circumstances
                         * specifically on the server side for a normal test or
                         * the client side for a reverse-mode test.  If this
                         * is the case, then use the receiver's count of packets
                         * instead.
                         */
                        int64_t packet_count = sender_packet_count ? sender_packet_count : receiver_packet_count;
                        cJSON_AddItemToObject(json_summary_stream, "udp", iperf_json_printf("socket: %d  start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  jitter_ms: %f  lost_packets: %d  packets: %d  lost_percent: %f  out_of_order: %d sender: %b", (int64_t) sp->socket, (double) start_time, (double) sender_time, (double) sender_time, (int64_t) bytes_sent, bandwidth * 8, (double) sp->jitter * 1000.0, (int64_t) (sp->cnt_error - sp->omitted_cnt_error), (int64_t) (packet_count - sp->omitted_packet_count), (double) lost_percent, (int64_t) (sp->outoforder_packets - sp->omitted_outoforder_packets), stream_must_be_sender));
                    }
                    else {
                        /*
                         * Due to ordering of messages on the control channel,
                         * the server cannot report on client-side summary
                         * statistics.  If we're the server, omit one set of
                         * summary statistics to avoid giving meaningless
                         * results.
                         */
                        if (test->role == 's' && !sp->sender) {
                            if (test->verbose)
                                iperf_printf(test, report_sender_not_available_format, sp->socket);
                        }
                        else {
                            iperf_printf(test, report_bw_udp_format, sp->socket, mbuf, start_time, sender_time, ubuf, nbuf, 0.0, (int64_t) 0, (sender_packet_count - sender_omitted_packet_count), (double) 0, report_sender);
                        }
                        if ((sp->outoforder_packets - sp->omitted_outoforder_packets) > 0)
                          iperf_printf(test, report_sum_outoforder, mbuf, start_time, sender_time, (sp->outoforder_packets - sp->omitted_outoforder_packets));
                    }
                }

                if (sp->diskfile_fd >= 0) {
                    if (fstat(sp->diskfile_fd, &sb) == 0) {
                        /* In the odd case that it's a zero-sized file, say it was all transferred. */
                        int percent_sent = 100, percent_received = 100;
                        if (sb.st_size > 0) {
                            percent_sent = (int) ( ( (double) bytes_sent / (double) sb.st_size ) * 100.0 );
                            percent_received = (int) ( ( (double) bytes_received / (double) sb.st_size ) * 100.0 );
                        }
                        unit_snprintf(sbuf, UNIT_LEN, (double) sb.st_size, 'A');
                        if (test->json_output)
                            cJSON_AddItemToObject(json_summary_stream, "diskfile", iperf_json_printf("sent: %d  received: %d  size: %d  percent_sent: %d  percent_received: %d  filename: %s", (int64_t) bytes_sent, (int64_t) bytes_received, (int64_t) sb.st_size, (int64_t) percent_sent, (int64_t) percent_received, test->diskfile_name));
                        else
                            if (stream_must_be_sender) {
                                iperf_printf(test, report_diskfile, ubuf, sbuf, percent_sent, test->diskfile_name);
                            }
                            else {
                                unit_snprintf(ubuf, UNIT_LEN, (double) bytes_received, 'A');
                                iperf_printf(test, report_diskfile, ubuf, sbuf, percent_received, test->diskfile_name);
                            }
                    }
                }

                unit_snprintf(ubuf, UNIT_LEN, (double) bytes_received, 'A');
                if (receiver_time > 0) {
                    bandwidth = (double) bytes_received / (double) receiver_time;
                }
                else {
                    bandwidth = 0.0;
                }
                unit_snprintf(nbuf, UNIT_LEN, bandwidth, test->settings->unit_format);
                if (test->protocol->id == Ptcp || test->protocol->id == Psctp) {
                    /* Receiver summary, TCP and SCTP */
                    if (test->json_output)
                        cJSON_AddItemToObject(json_summary_stream, report_receiver, iperf_json_printf("socket: %d  start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f sender: %b", (int64_t) sp->socket, (double) start_time, (double) receiver_time, (double) end_time, (int64_t) bytes_received, bandwidth * 8, stream_must_be_sender));
                    else
                        if (test->role == 's' && sp->sender) {
                            if (test->verbose)
                                iperf_printf(test, report_receiver_not_available_format, sp->socket);
                        }
                        else {
                            iperf_printf(test, report_bw_format, sp->socket, mbuf, start_time, receiver_time, ubuf, nbuf, report_receiver);
                        }
                }
                else {
                    /*
                     * Receiver summary, UDP.  Note that JSON was emitted with
                     * the sender summary, so we only deal with human-readable
                     * data here.
                     */
                    if (! test->json_output) {
                        if (test->omit == 0 && receiver_packet_count > 0) {
                            lost_percent = 100.0 * sp->cnt_error / receiver_packet_count;
                        } else if (receiver_packet_count - receiver_omitted_packet_count > 0 && sp->omitted_cnt_error > -1) {
                            lost_percent = 100.0 * (sp->cnt_error - sp->omitted_cnt_error) / (receiver_packet_count - receiver_omitted_packet_count);
                        }
                        else {
                            lost_percent = 0.0;
                        }

                        if (test->role == 's' && sp->sender) {
                            if (test->verbose)
                                iperf_printf(test, report_receiver_not_available_format, sp->socket);
                        }
                        else {
                            if (test->omit == 0) {
                                iperf_printf(test, report_bw_udp_format, sp->socket, mbuf, start_time, receiver_time, ubuf, nbuf, sp->jitter * 1000.0, sp->cnt_error, receiver_packet_count, lost_percent, report_receiver);
                            } else if (sp->omitted_cnt_error > -1) {
                                iperf_printf(test, report_bw_udp_format, sp->socket, mbuf, start_time, receiver_time, ubuf, nbuf, sp->jitter * 1000.0, (sp->cnt_error - sp->omitted_cnt_error), (receiver_packet_count - receiver_omitted_packet_count), lost_percent, report_receiver);
                            } else {
                                iperf_printf(test, report_bw_udp_format_no_omitted_error, sp->socket, mbuf, start_time, receiver_time, ubuf, nbuf, sp->jitter * 1000.0, (receiver_packet_count - receiver_omitted_packet_count), report_receiver);
                            }
                        }
                    }
                }
            }
        }
        }

        if (test->num_streams > 1 || test->json_output) {
            /*
             * With BIDIR give a different JSON object name to the one sent/receive sums.
             * The different name is given to the data sent from the server, which is
             * the "reverse" channel.  This makes sure that the name reported on the server
             * and client are compatible, and the names are the same as with non-bidir,
             * except for when reverse is used.
             */
            sum_name = "sum";
            sum_sent_name = "sum_sent";
            sum_received_name = "sum_received";
            if (test->mode == BIDIRECTIONAL) {
                if ((test->role == 'c' && !stream_must_be_sender) ||
                    (test->role != 'c' && stream_must_be_sender))
                {
                    sum_name = "sum_bidir_reverse";
                    sum_sent_name = "sum_sent_bidir_reverse";
                    sum_received_name = "sum_received_bidir_reverse";
                }

            }

            unit_snprintf(ubuf, UNIT_LEN, (double) total_sent, 'A');
            /* If no tests were run, arbitrarily set bandwidth to 0. */
            if (sender_time > 0.0) {
                bandwidth = (double) total_sent / (double) sender_time;
            }
            else {
                bandwidth = 0.0;
            }
            unit_snprintf(nbuf, UNIT_LEN, bandwidth, test->settings->unit_format);
            if (test->protocol->id == Ptcp || test->protocol->id == Psctp) {
                if (test->sender_has_retransmits) {
                    /* Summary sum, TCP with retransmits. */
                    if (test->json_output)
                        cJSON_AddItemToObject(test->json_end, sum_sent_name, iperf_json_printf("start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  retransmits: %d sender: %b", (double) start_time, (double) sender_time, (double) sender_time, (int64_t) total_sent, bandwidth * 8, (int64_t) total_retransmits, stream_must_be_sender));
                    else
                        if (test->role == 's' && !stream_must_be_sender) {
                            if (test->verbose)
                                iperf_printf(test, report_sender_not_available_summary_format, "SUM");
                        }
                        else {
                          iperf_printf(test, report_sum_bw_retrans_format, mbuf, start_time, sender_time, ubuf, nbuf, total_retransmits, report_sender);
                        }
                } else {
                    /* Summary sum, TCP without retransmits. */
                    if (test->json_output)
                        cJSON_AddItemToObject(test->json_end, sum_sent_name, iperf_json_printf("start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f sender: %b", (double) start_time, (double) sender_time, (double) sender_time, (int64_t) total_sent, bandwidth * 8, stream_must_be_sender));
                    else
                        if (test->role == 's' && !stream_must_be_sender) {
                            if (test->verbose)
                                iperf_printf(test, report_sender_not_available_summary_format, "SUM");
                        }
                        else {
                            iperf_printf(test, report_sum_bw_format, mbuf, start_time, sender_time, ubuf, nbuf, report_sender);
                        }
                }
                unit_snprintf(ubuf, UNIT_LEN, (double) total_received, 'A');
                /* If no tests were run, set received bandwidth to 0 */
                if (receiver_time > 0.0) {
                    bandwidth = (double) total_received / (double) receiver_time;
                }
                else {
                    bandwidth = 0.0;
                }
                unit_snprintf(nbuf, UNIT_LEN, bandwidth, test->settings->unit_format);
                if (test->json_output)
                    cJSON_AddItemToObject(test->json_end, sum_received_name, iperf_json_printf("start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f sender: %b", (double) start_time, (double) receiver_time, (double) receiver_time, (int64_t) total_received, bandwidth * 8, stream_must_be_sender));
                else
                    if (test->role == 's' && stream_must_be_sender) {
                        if (test->verbose)
                            iperf_printf(test, report_receiver_not_available_summary_format, "SUM");
                    }
                    else {
                        iperf_printf(test, report_sum_bw_format, mbuf, start_time, receiver_time, ubuf, nbuf, report_receiver);
                    }
            } else {
                /* Summary sum, UDP. */
                avg_jitter /= test->num_streams;
                /* If no packets were sent, arbitrarily set loss percentage to 0. */
                if (total_packets > 0) {
                    lost_percent = 100.0 * lost_packets / total_packets;
                }
                else {
                    lost_percent = 0.0;
                }
                if (test->json_output) {
                    /*
                     * Original, summary structure. Using this
                     * structure is not recommended due to
                     * ambiguities between the sender and receiver.
                     */
                    cJSON_AddItemToObject(test->json_end, sum_name, iperf_json_printf("start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  jitter_ms: %f  lost_packets: %d  packets: %d  lost_percent: %f sender: %b", (double) start_time, (double) receiver_time, (double) receiver_time, (int64_t) total_sent, bandwidth * 8, (double) avg_jitter * 1000.0, (int64_t) lost_packets, (int64_t) total_packets, (double) lost_percent, stream_must_be_sender));
                    
                    double sent_bandwidth = sender_time > 0 ? ((double) total_sent * 8 / sender_time) : 0.0;
                    double recv_bandwidth = receiver_time > 0 ? ((double) total_received * 8 / receiver_time) : 0.0;
                    /*
                     * Separate sum_sent and sum_received structures.
                     * Using these structures to get the most complete
                     * information about UDP transfer.
                     */
                    cJSON_AddItemToObject(test->json_end, sum_sent_name, iperf_json_printf("start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  jitter_ms: %f  lost_packets: %d  packets: %d  lost_percent: %f  sender: %b", (double) start_time, (double) sender_time, (double) sender_time, (int64_t) total_sent, sent_bandwidth, (double) 0.0, (int64_t) 0, (int64_t) sender_total_packets, (double) 0.0, 1));
                    cJSON_AddItemToObject(test->json_end, sum_received_name, iperf_json_printf("start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  jitter_ms: %f  lost_packets: %d  packets: %d  lost_percent: %f  sender: %b", (double) start_time, (double) receiver_time, (double) receiver_time, (int64_t) total_received, recv_bandwidth, (double) avg_jitter * 1000.0, (int64_t) lost_packets, (int64_t) receiver_total_packets, (double) lost_percent, 0));
                } else {
                    /*
                     * On the client we have both sender and receiver overall summary
                     * stats.  On the server we have only the side that was on the
                     * server.  Output whatever we have.
                     */
                    if (! (test->role == 's' && !stream_must_be_sender) ) {
                        unit_snprintf(ubuf, UNIT_LEN, (double) total_sent, 'A');
                        iperf_printf(test, report_sum_bw_udp_format, mbuf, start_time, sender_time, ubuf, nbuf, 0.0, (int64_t) 0, sender_total_packets, 0.0, report_sender);
                    }
                    if (! (test->role == 's' && stream_must_be_sender) ) {

                        unit_snprintf(ubuf, UNIT_LEN, (double) total_received, 'A');
                        /* Compute received bandwidth. */
                        if (end_time > 0.0) {
                            bandwidth = (double) total_received / (double) receiver_time;
                        }
                        else {
                            bandwidth = 0.0;
                        }
                        unit_snprintf(nbuf, UNIT_LEN, bandwidth, test->settings->unit_format);
                        iperf_printf(test, report_sum_bw_udp_format, mbuf, start_time, receiver_time, ubuf, nbuf, avg_jitter * 1000.0, lost_packets, receiver_total_packets, lost_percent, report_receiver);
                    }
                }
            }
        }

        if (test->json_output && current_mode == upper_mode) {
            cJSON_AddItemToObject(test->json_end, "cpu_utilization_percent", iperf_json_printf("host_total: %f  host_user: %f  host_system: %f  remote_total: %f  remote_user: %f  remote_system: %f", (double) test->cpu_util[0], (double) test->cpu_util[1], (double) test->cpu_util[2], (double) test->remote_cpu_util[0], (double) test->remote_cpu_util[1], (double) test->remote_cpu_util[2]));
            if (test->protocol->id == Ptcp) {
                char *snd_congestion = NULL, *rcv_congestion = NULL;
                if (stream_must_be_sender) {
                    snd_congestion = test->congestion_used;
                    rcv_congestion = test->remote_congestion_used;
                }
                else {
                    snd_congestion = test->remote_congestion_used;
                    rcv_congestion = test->congestion_used;
                }
                if (snd_congestion) {
                    cJSON_AddStringToObject(test->json_end, "sender_tcp_congestion", snd_congestion);
                }
                if (rcv_congestion) {
                    cJSON_AddStringToObject(test->json_end, "receiver_tcp_congestion", rcv_congestion);
                }
            }
        }
        else {
            if (test->verbose) {
                if (stream_must_be_sender) {
                    if (test->bidirectional) {
                        iperf_printf(test, report_cpu, report_local, stream_must_be_sender?report_sender:report_receiver, test->cpu_util[0], test->cpu_util[1], test->cpu_util[2], report_remote, stream_must_be_sender?report_receiver:report_sender, test->remote_cpu_util[0], test->remote_cpu_util[1], test->remote_cpu_util[2]);
                        iperf_printf(test, report_cpu, report_local, !stream_must_be_sender?report_sender:report_receiver, test->cpu_util[0], test->cpu_util[1], test->cpu_util[2], report_remote, !stream_must_be_sender?report_receiver:report_sender, test->remote_cpu_util[0], test->remote_cpu_util[1], test->remote_cpu_util[2]);
                    } else
                        iperf_printf(test, report_cpu, report_local, stream_must_be_sender?report_sender:report_receiver, test->cpu_util[0], test->cpu_util[1], test->cpu_util[2], report_remote, stream_must_be_sender?report_receiver:report_sender, test->remote_cpu_util[0], test->remote_cpu_util[1], test->remote_cpu_util[2]);
                }
                if (test->protocol->id == Ptcp) {
                    char *snd_congestion = NULL, *rcv_congestion = NULL;
                    if (stream_must_be_sender) {
                        snd_congestion = test->congestion_used;
                        rcv_congestion = test->remote_congestion_used;
                    }
                    else {
                        snd_congestion = test->remote_congestion_used;
                        rcv_congestion = test->congestion_used;
                    }
                    if (snd_congestion) {
                        iperf_printf(test, "snd_tcp_congestion %s\n", snd_congestion);
                    }
                    if (rcv_congestion) {
                        iperf_printf(test, "rcv_tcp_congestion %s\n", rcv_congestion);
                    }
                }
            }

            /* Print server output if we're on the client and it was requested/provided */
            if (test->role == 'c' && iperf_get_test_get_server_output(test) && !test->json_output) {
                if (test->json_server_output) {
		    char *str = cJSON_Print(test->json_server_output);
                    iperf_printf(test, "\nServer JSON output:\n%s\n", str);
		    cJSON_free(str);
                    cJSON_Delete(test->json_server_output);
                    test->json_server_output = NULL;
                }
                if (test->server_output_text) {
                    iperf_printf(test, "\nServer output:\n%s\n", test->server_output_text);
	            free(test->server_output_text);
                    test->server_output_text = NULL;
                }
            }
        }
    }

    /* Set real sender_has_retransmits for current side */
    if (test->mode == BIDIRECTIONAL)
        test->sender_has_retransmits = tmp_sender_has_retransmits;
}

在中会掉用实现打印interval的测量结果,让我们看看具体的实现代码。

/**
 * Print the interval results for one stream.
 * This function needs to know about the overall test so it can determine the
 * context for printing headers, separators, etc.
 */
static void
print_interval_results(struct iperf_test *test, struct iperf_stream *sp, cJSON *json_interval_streams)
{
    char ubuf[UNIT_LEN];
    char nbuf[UNIT_LEN];
    char cbuf[UNIT_LEN];
    char mbuf[UNIT_LEN];
    char zbuf[] = "          ";
    double st = 0., et = 0.;
    struct iperf_time temp_time;
    struct iperf_interval_results *irp = NULL;
    double bandwidth, lost_percent;

    if (test->mode == BIDIRECTIONAL) {
        snprintf(mbuf, sizeof(mbuf), "[%s-%s]", sp->sender?"TX":"RX", test->role == 'c'?"C":"S");
    } else {
        mbuf[0] = '\0';
        zbuf[0] = '\0';
    }

    irp = TAILQ_LAST(&sp->result->interval_results, irlisthead); /* get last entry in linked list */
    if (irp == NULL) {
	iperf_err(test, "print_interval_results error: interval_results is NULL");
        return;
    }
    if (!test->json_output) {
	/* First stream? */
	if (sp == SLIST_FIRST(&test->streams)) {
	    /* It it's the first interval, print the header;
	    ** else if there's more than one stream, print the separator;
	    ** else nothing.
	    */
            // 第一个interval打印头信息,超过一个stream会打印分隔符,否则什么都不打印
	    if (iperf_time_compare(&sp->result->start_time, &irp->interval_start_time) == 0) {
		if (test->protocol->id == Ptcp || test->protocol->id == Psctp) {
		    if (test->sender_has_retransmits == 1) {
		        if (test->bidirectional)
		            iperf_printf(test, "%s", report_bw_retrans_cwnd_header_bidir);
		        else
		            iperf_printf(test, "%s", report_bw_retrans_cwnd_header);
		    }
		    else {
	                if (test->bidirectional)
	                    iperf_printf(test, "%s", report_bw_header_bidir);
	                else
	                    iperf_printf(test, "%s", report_bw_header);
	            }
		} else {
		    if (test->mode == SENDER) {
		        iperf_printf(test, "%s", report_bw_udp_sender_header);
		    } else if (test->mode == RECEIVER){
		        iperf_printf(test, "%s", report_bw_udp_header);
		    } else {
		        /* BIDIRECTIONAL */
		        iperf_printf(test, "%s", report_bw_udp_header_bidir);
		    }
		}
	    } else if (test->num_streams > 1)
		iperf_printf(test, "%s", report_bw_separator);
	}
    }

    // 计算带宽
    unit_snprintf(ubuf, UNIT_LEN, (double) (irp->bytes_transferred), 'A');
    if (irp->interval_duration > 0.0) {
	bandwidth = (double) irp->bytes_transferred / (double) irp->interval_duration;
    }
    else {
	bandwidth = 0.0;
    }
    unit_snprintf(nbuf, UNIT_LEN, bandwidth, test->settings->unit_format);

    iperf_time_diff(&sp->result->start_time, &irp->interval_start_time, &temp_time);
    st = iperf_time_in_secs(&temp_time);
    iperf_time_diff(&sp->result->start_time, &irp->interval_end_time, &temp_time);
    et = iperf_time_in_secs(&temp_time);

    if (test->protocol->id == Ptcp || test->protocol->id == Psctp) {
	if ((test->sender_has_retransmits == 1 || test->protocol->id == Psctp) && sp->sender) {
	    /* Interval, TCP with retransmits. */
	    if (test->json_output)
		cJSON_AddItemToArray(json_interval_streams, iperf_json_printf("socket: %d  start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  retransmits: %d  snd_cwnd:  %d  snd_wnd:  %d  rtt:  %d  rttvar: %d  pmtu: %d  reorder: %d  omitted: %b sender: %b", (int64_t) sp->socket, (double) st, (double) et, (double) irp->interval_duration, (int64_t) irp->bytes_transferred, bandwidth * 8, (int64_t) irp->interval_retrans, (int64_t) irp->snd_cwnd, (int64_t) irp->snd_wnd, (int64_t) irp->rtt, (int64_t) irp->rttvar, (int64_t) irp->pmtu, (int64_t) irp->reorder, irp->omitted, sp->sender));
	    else {
		unit_snprintf(cbuf, UNIT_LEN, irp->snd_cwnd, 'A');
		iperf_printf(test, report_bw_retrans_cwnd_format, sp->socket, mbuf, st, et, ubuf, nbuf, irp->interval_retrans, cbuf, irp->omitted?report_omitted:"");
	    }
	} else {
	    /* Interval, TCP without retransmits. */
	    if (test->json_output)
		cJSON_AddItemToArray(json_interval_streams, iperf_json_printf("socket: %d  start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  omitted: %b sender: %b", (int64_t) sp->socket, (double) st, (double) et, (double) irp->interval_duration, (int64_t) irp->bytes_transferred, bandwidth * 8, irp->omitted, sp->sender));
	    else
		iperf_printf(test, report_bw_format, sp->socket, mbuf, st, et, ubuf, nbuf, irp->omitted?report_omitted:"");
	}
    } else {
	/* Interval, UDP. */
	if (sp->sender) {
	    if (test->json_output)
		cJSON_AddItemToArray(json_interval_streams, iperf_json_printf("socket: %d  start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  packets: %d  omitted: %b sender: %b", (int64_t) sp->socket, (double) st, (double) et, (double) irp->interval_duration, (int64_t) irp->bytes_transferred, bandwidth * 8, (int64_t) irp->interval_packet_count, irp->omitted, sp->sender));
	    else
		iperf_printf(test, report_bw_udp_sender_format, sp->socket, mbuf, st, et, ubuf, nbuf, zbuf, irp->interval_packet_count, irp->omitted?report_omitted:"");
	} else {
	    if (irp->interval_packet_count > 0) {
		lost_percent = 100.0 * irp->interval_cnt_error / irp->interval_packet_count;
	    }
	    else {
		lost_percent = 0.0;
	    }
	    if (test->json_output)
		cJSON_AddItemToArray(json_interval_streams, iperf_json_printf("socket: %d  start: %f  end: %f  seconds: %f  bytes: %d  bits_per_second: %f  jitter_ms: %f  lost_packets: %d  packets: %d  lost_percent: %f  omitted: %b sender: %b", (int64_t) sp->socket, (double) st, (double) et, (double) irp->interval_duration, (int64_t) irp->bytes_transferred, bandwidth * 8, (double) irp->jitter * 1000.0, (int64_t) irp->interval_cnt_error, (int64_t) irp->interval_packet_count, (double) lost_percent, irp->omitted, sp->sender));
	    else
		iperf_printf(test, report_bw_udp_format, sp->socket, mbuf, st, et, ubuf, nbuf, irp->jitter * 1000.0, irp->interval_cnt_error, irp->interval_packet_count, lost_percent, irp->omitted?report_omitted:"");
	}
    }

    if (test->logfile || test->forceflush)
        iflush(test);
}

上面就是iperf中计算统计的主要方法,当然直接阅读这些代码比较枯燥,很难有直观的理解,让我们做个简单的实验看看iperf的测量效果。

我有一个云服务器1.2.3.4 ,出站带宽是3Mbps。我在上面开了一个iperf的server,在家里的电脑测量一下和server之间的带宽,所以家里的电脑是iperf client,服务器是iperf server。

使用命令iperf3.exe -c 1.2.3.4 -u -b 10M,测出的带宽是:

使用命令iperf3.exe -c 1.2.3.4-u -b 10M -R,测出的带宽是:

可以看到,从client到server可以几乎跑满10Mbps带宽,从server到client只有3.72Mbps。这也和云服务商上的带宽说明相符,因为云服务器标称的带宽就是服务器到用户的带宽,而用户到服务器的带宽一般都会大于前者,虽然下面的文档说的是10Mbps,但实际比这个要大,可以达到50Mbps。

Comments are closed.