UPGW关键log设计

rte_log基本流程和功能验证

初始化流程

DPDK代码中广泛使用RTE_INIT宏进行设备驱动或者RTE模块等的初始化工作，其核心是RTE_INIT_PRIO宏，定义在文件rte_common.h中。如下可见，RTE_INIT_PRIO宏的实现，实际为附带GCC编译属性的函数定义。此处用到两个属性，constructor和used。其中后一个used比较简单，向GCC编译器表明此函数的有用性，即使函数没有被任何地方引用，也不要警告。constructor构造属性，类似C++的构造函数，其标注的函数将在主函数（main）之前执行，相关代码位于目标文件的.ctors区（section）。

#define RTE_INIT_PRIO(func, prio) \
static void __attribute__((constructor(RTE_PRIO(prio)), used)) func(void)

另外RTE_PRIO宏指定了constructor的优先级，即函数执行的顺序，值越小优先级越高。目前定义了4个优先级：RTE_PRIORITY_LOG、RTE_PRIORITY_BUS、RTE_PRIORITY_CLASS和RTE_PRIORITY_LAST。

#define RTE_PRIORITY_LOG 101
#define RTE_PRIORITY_BUS 110
#define RTE_PRIORITY_CLASS 120
#define RTE_PRIORITY_LAST 65535

#define RTE_PRIO(prio) \
    RTE_PRIORITY_ ## prio

其中RTE_INIT宏封装了RTE_PRIORITY_LAST优先级的函数。RTE_REGISTER_BUS宏封装了RTE_PRIORITY_BUS优先级的函数。而RTE_REGISTER_CLASS宏封装了RTE_PRIORITY_CLASS优先级的函数。目前DPDK中RTE_PRIORITY_LOG优先级的函数仅有一个rte_log_init，优先级最高，最先初始化（函数所在文件：lib/librte_eal/common/eal_common_log.c）。

RTE_INIT_PRIO(rte_log_init, LOG)
{
    uint32_t i;

    rte_log_set_global_level(RTE_LOG_DEBUG);

    rte_logs.dynamic_types = calloc(RTE_LOGTYPE_FIRST_EXT_ID,
        sizeof(struct rte_log_dynamic_type));
    if (rte_logs.dynamic_types == NULL)
        return;

    /* register legacy log types */
    for (i = 0; i < RTE_DIM(logtype_strings); i++)
        __rte_log_register(logtype_strings[i].logtype,
                logtype_strings[i].log_id);

    rte_logs.dynamic_types_len = RTE_LOGTYPE_FIRST_EXT_ID;
}

rte_logs为全局变量，rte_log_init函数首先调用rte_log_set_global_level函数对全局的一个日志等级进行配置，配置为RTE_LOG_DEBUG。

struct rte_log_dynamic_type {
    const char *name;
    uint32_t loglevel;
};

/** The rte_log structure. */
struct rte_logs {
    uint32_t type;  /**< Bitfield with enabled logs. */
    uint32_t level; /**< Log level. */
    FILE *file;     /**< Output file set by rte_openlog_stream, or NULL. */
    size_t dynamic_types_len;
    struct rte_log_dynamic_type *dynamic_types;
};

/* global log structure */
struct rte_logs rte_logs = {
    .type = ~0,
    .level = RTE_LOG_DEBUG,
    .file = NULL,
};

/* Set global log level */
void
rte_log_set_global_level(uint32_t level)
{
    rte_logs.level = (uint32_t)level;
}

对于各日志模块，再调用__rte_log_register函数逐个进行等级配置，默认为RTE_LOG_INFO，下面是系统已定义的一些日志模块，也可以用户自定义添加，见后续说明，同时各模块的日志等级也能在程序启动时通过命令行“–log-level=log_type:log_level”更改。

static const struct logtype logtype_strings[] = {
    {RTE_LOGTYPE_EAL,        "lib.eal"},
    {RTE_LOGTYPE_MALLOC,     "lib.malloc"},
    {RTE_LOGTYPE_RING,       "lib.ring"},
    {RTE_LOGTYPE_MEMPOOL,    "lib.mempool"},
    {RTE_LOGTYPE_TIMER,      "lib.timer"},
    {RTE_LOGTYPE_PMD,        "pmd"},
    {RTE_LOGTYPE_HASH,       "lib.hash"},
    {RTE_LOGTYPE_LPM,        "lib.lpm"},
    {RTE_LOGTYPE_KNI,        "lib.kni"},
    {RTE_LOGTYPE_ACL,        "lib.acl"},
    {RTE_LOGTYPE_POWER,      "lib.power"},
    {RTE_LOGTYPE_METER,      "lib.meter"},
    {RTE_LOGTYPE_SCHED,      "lib.sched"},
    {RTE_LOGTYPE_PORT,       "lib.port"},
    {RTE_LOGTYPE_TABLE,      "lib.table"},
    {RTE_LOGTYPE_PIPELINE,   "lib.pipeline"},
    {RTE_LOGTYPE_MBUF,       "lib.mbuf"},
    {RTE_LOGTYPE_CRYPTODEV,  "lib.cryptodev"},
    {RTE_LOGTYPE_EFD,        "lib.efd"},
    {RTE_LOGTYPE_EVENTDEV,   "lib.eventdev"},
    {RTE_LOGTYPE_GSO,        "lib.gso"},
    {RTE_LOGTYPE_USER1,      "user1"},
    {RTE_LOGTYPE_USER2,      "user2"},
    {RTE_LOGTYPE_USER3,      "user3"},
    {RTE_LOGTYPE_USER4,      "user4"},
    {RTE_LOGTYPE_USER5,      "user5"},
    {RTE_LOGTYPE_USER6,      "user6"},
    {RTE_LOGTYPE_USER7,      "user7"},
    {RTE_LOGTYPE_USER8,      "user8"}
};

static int
__rte_log_register(const char *name, int id)
{
    char *dup_name = strdup(name);

    if (dup_name == NULL)
        return -ENOMEM;

    rte_logs.dynamic_types[id].name = dup_name;
    rte_logs.dynamic_types[id].loglevel = RTE_LOG_INFO;

    return id;
}

日志等级如下，值越大等级越高，打印的信息越多。

#define RTE_LOG_EMERG    1U  /**< System is unusable.               */
#define RTE_LOG_ALERT    2U  /**< Action must be taken immediately. */
#define RTE_LOG_CRIT     3U  /**< Critical conditions.              */
#define RTE_LOG_ERR      4U  /**< Error conditions.                 */
#define RTE_LOG_WARNING  5U  /**< Warning conditions.               */
#define RTE_LOG_NOTICE   6U  /**< Normal but significant condition. */
#define RTE_LOG_INFO     7U  /**< Informational.                    */
#define RTE_LOG_DEBUG    8U  /**< Debug-level messages.             */

日志输出逻辑

dpdk的各日志模块封装了rte_log函数，rte_log的核心是rte_vlog函数，首先要判断日志等级与全局日志等级的关系，如果大于全局日志等级，则不输出，再判断与该模块定义日志等级的关系，若大于该模块日志等级，同样不输出，在输出的情况下，是输出到rte_logs.file里，即下面的default_log_stream，default_log_stream初始值是在rte_eal_init里面配置的。

#define RTE_LOG(l, t, ...)             \
     rte_log(RTE_LOG_ ## l,            \
         RTE_LOGTYPE_ ## t, # t ": " __VA_ARGS__)

int
rte_log(uint32_t level, uint32_t logtype, const char *format, ...)
{
	va_list ap;
	int ret;

	va_start(ap, format);
	ret = rte_vlog(level, logtype, format, ap);
	va_end(ap);
	return ret;
}

int
rte_vlog(uint32_t level, uint32_t logtype, const char *format, va_list ap)
{
    int ret;
    FILE *f = rte_logs.file;
    if (f == NULL) {
        f = default_log_stream;
        if (f == NULL) {
            /*
             * Grab the current value of stderr here, rather than
             * just initializing default_log_stream to stderr. This
             * ensures that we will always use the current value
             * of stderr, even if the application closes and
             * reopens it.
             */
            f = stderr;
        }
    }

    if (level > rte_logs.level) {
        return 0;
    }

    if (logtype >= rte_logs.dynamic_types_len) {
        return -1;
    }

    if (level > rte_logs.dynamic_types[logtype].loglevel) {
        return 0;
    }

    /* save loglevel and logtype in a global per-lcore variable */
    RTE_PER_LCORE(log_cur_msg).loglevel = level;
    RTE_PER_LCORE(log_cur_msg).logtype = logtype;

    ret = vfprintf(f, format, ap);
    fflush(f);
    return ret;
}

日志输出位置

在rte_eal_log_init里面，log_stream被配置为标准输出stdout，log_stream又被赋值给default_log_stream，这样dpdk的日志默认被输出到stdout上，若想改为其它位置可以在rte_eal_init之后通过rte_openlog_stream重定向到其它文件中。

int
rte_eal_init(int argc, char **argv)
{
    if (rte_eal_log_init(logid, internal_config.syslog_facility) < 0) {
        rte_eal_init_alert("Cannot init logging.");
        rte_errno = ENOMEM;
        rte_atomic32_clear(&run_once);
        return -1;
	}
}

int
rte_eal_log_init(const char *id, int facility)
{
    FILE *log_stream;

    log_stream = fopencookie(NULL, "w+", console_log_func);
    if (log_stream == NULL)
        return -1;

    openlog(id, LOG_NDELAY | LOG_PID, facility);

    eal_log_set_default(log_stream);

    return 0;
}

static cookie_io_functions_t console_log_func = {
    .write = console_log_write,
};

static ssize_t
console_log_write(__attribute__((unused)) void *c, const char *buf, size_t size)
{
    ssize_t ret;

    /* write on stdout */
    ret = fwrite(buf, 1, size, stdout);
    fflush(stdout);

    /* Syslog error levels are from 0 to 7, so subtract 1 to convert */
    syslog(rte_log_cur_msg_loglevel() - 1, "%.*s", (int)size, buf);

    return ret;
}

void
eal_log_set_default(FILE *default_log)
{
    default_log_stream = default_log;

#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
    RTE_LOG(NOTICE, EAL,
        "Debug dataplane logs available - lower performance\n");
#endif
}

修改对应模块日志等级

以l2fwd程序作为验证基础，如下，dpdk程序中已定义该log模块为RTE_LOGTYPE_USER1，可以在上面的模块表找到对应关系{RTE_LOGTYPE_USER1, “user1”}。

#define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1

编译后运行“./build/l2fwd -l 1 -n 4 – -q 1 -p 1”命令启动l2fwd，因为该模块的默认输出的最高日志等级为RTE_LOG_INFO（7），代码中的相关日志也是以INFO的等级输出，可看到屏幕上有下面的打印：

Port statistics ====================================
Statistics for port 0 ------------------------------
Packets sent:                      175
Packets received:                  175
Packets dropped:                     0
Aggregate statistics ===============================
Total packets sent:                175
Total packets received:            175
Total packets dropped:               0
====================================================

Port statistics ====================================
Statistics for port 0 ------------------------------
Packets sent:                      175
Packets received:                  175
Packets dropped:                     0
Aggregate statistics ===============================
Total packets sent:                175
Total packets received:            175
Total packets dropped:               0
====================================================

但当用“./build/l2fwd -l 1 -n 4 –log-level=”user1”:6 – -q 1 -p 1” 命令启动程序后，就看不到日志输出了，因为将该模块输出的最高日志等级改为了RTE_LOG_NOTICE（6），所以INFO等级的日志就不能输出，验证成功。

重定向功能验证

在rte_eal_init后面，添加下列函数，可将日志输出到/var/log/rte.log文件中，验证成功。

FILE *fp = fopen("/var/log/rte.log", "w+");
if (fp) {
    rte_openlog_stream(fp);
}

自定义log_type验证

若新加一个日志模块，需要做以下操作，下列示例也是在l2fwd中完成修改：

定义新的log_type，在rte_eal_init之前完成初始化。

int logtype_l2fwd;

logtype_l2fwd = rte_log_register("l2fwd"); #其实是封装了__rte_log_register

if (logtype_l2fwd >= 0) {
        rte_log_set_level(logtype_l2fwd, RTE_LOG_INFO); #默认最高日志等级为RTE_LOG_INFO
}

封装rte_log函数，预置log类型为logtype_l2fwd，输出该模块的log时直接调用LOG_L2FWD接口。

#define LOG_L2FWD(level, fmt, args...) \
    rte_log(RTE_LOG_ ## level, logtype_l2fwd, "%s(): " fmt, \
                    __func__, ## args)

增加该自定义log后验证了解析–log-level参数和重定向功能，同样ok。

UPGW的log模块设计

如下，设置NES的log模块，初始化日志等级为INFO，eal初始化后将日志重定向到/var/log/rte.log文件中。

int logtype_nes;

logtype_nes = rte_log_register("NES");
    if (logtype_nes >= 0) {
    rte_log_set_level(logtype_nes, RTE_LOG_INFO);
} else {
    NES_LOG("logtype_nes register error\n");
    return NES_FAIL;
}

eal_args = rte_eal_init(argc, argv);

FILE *fp = fopen("/var/log/rte.log", "w+");
if (fp) {
    rte_openlog_stream(fp);
}

设计NES_LOG的接口如下，每条日志的通用字段是日志等级和打印这条日志的文件、函数和行数，其它为可选参数。

#define NES_LOG(level, fmt, args...) \
    rte_log(RTE_LOG_ ## level, logtype_nes, "["#level"] "fmt" %s -- %s() -- %d\n", \
        ## args, __FILE__, __func__, __LINE__)

LOG具体实现

目前日志的实现思路是打印一条流从进入到送出的整个流程，细化下来就是这条流在经过recv模块、scatter、flow和send模块时，都留下相关的日志输出，为跟踪代码流程和排错提供参考。

recv模块日志

在nes_dev_t结构体中增加port_rx变量，标识收包index，收到报文后，将更新后的值赋予mbuf中的udata64（用户数据）变量，这样该端口收到的每个mbuf都是唯一标识的，它在传递的过程也不会改变，日志中记录该线程为nes_io、模块为recv，同时记录端口名字和index。

typedef struct nes_dev_s {
    uint64_t port_rx; /* 增加该变量记录端口的收包index */
} nes_dev_t;

self->port_rx++;
self->rx_pkts[i]->udata64 = self->port_rx; /* 在mbuf中标识该报文 */

NES_LOG(DEBUG, "thread:nes_io, module:recv, portname:%s, index:%llu", self->name, self->rx_pkts[i]->udata64);

scatter模块日志

scatter根据收包的端口不通，流程也会不同。

upstream口

判断为NO ETHER_TYPE_IPv4类型的报文，直接送入engress的队列，不再经过flow模块，日志内容如下，记录线程为nes_io、模块为scatter、方向为FROM UPSTR、协议类型为NO ETHER_TYPE_IPv4，同时记录端口名字、index和将要发往的ring。
```
NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, traffic-direction:FROM UPSTR, proto-type:NO ETHER_TYPE_IPv4, to_ring:%s", self->name, self->rx_pkts[i]->udata64, ring->ring->name);
```

判断为SCTP类型的报文，送往NIS_PORT_UPSTR_SCTP队列，日志如下，这里因为已经解析到了ip地址，所以将源目ip也记录了下来。

NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM UPSTR, proto-type:SCTP, to_ring:%s", self->name, self->rx_pkts[i]->udata64, src_ip, dst_ip, ring->ring->name);

判断为IP类型的报文，送往NTS_PORT_UPSTR_IP队列，日志如下。

NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:%s", self->name, self->rx_pkts[i]->udata64, src_ip, dst_ip, ring->ring->name);

判断是LTE WITHOUT UDP类型的报文，直接送入engress的队列，不再经过flow模块，日志如下。

NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM UPSTR, proto-type:LTE WITHOUT UDP, to_ring:%s", self->name, self->rx_pkts[i]->udata64, src_ip, dst_ip, ring->ring->name);

判断是GTPU类型的报文，送往NTS_PORT_UPSTR_GTPU队列，日志如下。

NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM UPSTR, proto-type:GTPU, to_ring:%s", self->name, self->rx_pkts[i]->udata64, src_ip, dst_ip, ring->ring->name);

判断是GTPC 类型的报文，送往NIS_PORT_UPSTR_GTPC 队列，日志如下。

NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM UPSTR, proto-type:GTPC, to_ring:%s", self->name, self->rx_pkts[i]->udata64, src_ip, dst_ip, ring->ring->name);

判断是LTE WITH UDP类型的报文，送往engress队列，不再经过flow模块，日志如下。

NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM UPSTR, proto-type:LTE WITH UDP, to_ring:%s", self->name, self->rx_pkts[i]->udata64, src_ip, dst_ip, ring->ring->name);

downstream口

与上文报文分类一致，只是报文方向改为了FROM DWSTR，日志不再赘述。

lbp口

报文进入NTS_LBP_ANY队列，在此模块没有解析ip，在flow模块进行了解析，若enq_burst一次没有发送完成，记录上面的日志，module为scatter less，若一次发送完成记录下面的日志，module为scatter，如下所示。

NES_LOG(DEBUG, "thread:nes_io, module:scatter less, portname:%s, index:%llu, traffic-direction:FROM LBP, to_ring:%s", self->name, self->rx_pkts[j]->udata64, rx_ring->ring->name);

NES_LOG(DEBUG, "thread:nes_io, module:scatter, portname:%s, index:%llu, traffic-direction:FROM LBP, to_ring:%s", self->name, self->rx_pkts[j]->udata64, rx_ring->ring->name);

flow模块日志

处理NTS_PORT_UPSTR_IP队列报文

判断是否命中DNS规则，若命中日志如下。

NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:FROM UPSTR TO DNS, proto-type:IP, to_ring:%s\n", devices_tab[mbufs[i]->port].device->name, mbufs[i]->udata64, ring->ring->name);

判断是否命中LBP规则，这条日志不能区分UPSTR和DWSTR，IP和GTPU，可与前面日志联系进行区分，若命中日志如下。

NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:TO LBP, to_ring:%s\n", devices_tab[mbuf->port].device->name, mbuf->udata64, entry->dst_ring->ring->name);

其它情况往engress口转发。

NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:%s\n", devices_tab[mbufs[i]->port].device->name, mbufs[i]->udata64, egress_ring->ring->name);

处理NTS_PORT_UPSTR_GTPU队列报文

判断是否有NONE_N3_N9_EXTENSION_FORWARD标志，若有往engress口转发，日志如下：

NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:NONE_N3_N9, to_ring:%s\n", devices_tab[mbufs[i]->port].device->name, mbufs[i]->udata64, egress_ring->ring->name);

判断是否命中DNS规则，若命中日志如下。

NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:FROM UPSTR TO DNS, proto-type:GTPU, to_ring:%s\n", devices_tab[mbufs[i]->port].device->name, mbufs[i]->udata64, ring->ring->name);

判断是否命中LBP规则，若命中日志如下。

NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:TO LBP, to_ring:%s\n", devices_tab[mbuf->port].device->name, mbuf->udata64, entry->dst_ring->ring->name);

其它情况往engress口转发。

NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:GTPU, to_ring:%s\n", devices_tab[mbufs[i]->port].device->name, mbufs[i]->udata64, egress_ring->ring->name);

处理NTS_PORT_DWSTR_IP队列报文

与上面处理NTS_PORT_UPSTR_IP队列报文类型，只是不再命中DNS规则，同时报文方向改为了FROM DWSTR。
处理NTS_PORT_DWSTR_GTPU队列报文

与上面处理NTS_PORT_UPSTR_GTPU队列报文类型，只是不再命中DNS规则，同时报文方向改为了FROM DWSTR。

处理NTS_LBP_ANY队列报文

若是arp request报文，修改为arp reply报文后发送给源端口，日志如下。

NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM LBP TO LBP, proto-type:ARP REPLY, to_ring:%s\n", devices_tab[mbuf->port].device->name, mbuf->udata64, src_ip, dst_ip, dst_ring->ring->name);

若是ip报文，根据命中learning表项的情况往对应的engress口发出，日志如下。

NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM LBP TO ENGRESS, proto-type:IP, to_ring:%s\n", devices_tab[mbufs[idx]->port].device->name, mbufs[idx]->udata64, src_ip, dst_ip, egress_ring->ring->name);

若是gtpu报文，根据命中learning表项的情况往对应的engress口发出，日志如下。

NES_LOG(DEBUG, "thread:nts_io, module:flow, portname:%s, index:%llu, srcip:%s, dstip:%s, traffic-direction:FROM LBP TO ENGRESS, proto-type:GTPU, to_ring:%s\n", devices_tab[mbufs[idx]->port].device->name, mbufs[idx]->udata64, src_ip, dst_ip, egress_ring->ring->name);

send模块日志

send

self->tx_buffer中还保存有未发完的报文，记录下面的日志，from portname代表从哪个端口进入的报文，from ring代表从什么队列取到的包，可以和前面的日志进行联系，再加上index就可以一一对应。
```
NES_LOG(DEBUG, "thread:nes_io, module:send remaining, from portname:%s, to portname:%s, index:%llu, from ring:%s", devices_tab[self->tx_buffer[j]->port].device->name, self->name, self->tx_buffer[j]->udata64, tx_ring->ring->name);
```

tx_burst未一次发送完成记录下面的日志。

NES_LOG(DEBUG, "thread:nes_io, module:send less, from portname:%s, to portname:%s, index:%llu, from ring:%s", devices_tab[buf[j]->port].device->name, self->name, buf[j]->udata64, tx_ring->ring->name);

tx_burst发送完成记录下面的日志。

NES_LOG(DEBUG, "thread:nes_io, module:send, from portname:%s, to portname:%s,  index:%llu, from ring:%s", devices_tab[buf[j]->port].device->name, self->name, buf[j]->udata64, tx_ring->ring->name);

send mtu

与上面基本一致，只是module:send xxx改为了module:send mtu xxx，不再重复记录。

UPGW日志模块测试

逻辑拓扑

如下所示，nr0和up0直连，nr1和up1直连，upf0和down0直连，upf1和down1直连，dn0和lbp0直连，dn1和lbp1直连。

逻辑拓扑

配置文件

[PORT0]
name = 0000:00:04.0
description = 82540EM Gigabit Ethernet Controller
pci-address = 0000:00:04.0
traffic-type = IP
traffic-direction = upstream
egress-port = 1

[PORT1]
name = 0000:00:05.0
description = 82540EM Gigabit Ethernet Controller
pci-address = 0000:00:05.0
traffic-type = IP
traffic-direction = downstream
egress-port = 0

[PORT2]
name = 0000:00:06.0
description = 82540EM Gigabit Ethernet Controller
pci-address = 0000:00:06.0
traffic-type = IP
traffic-direction = lbp
egress-port = 0
lbp-mac = fa:16:3e:c0:70:fa

[PORT3]
name = 0000:00:0e.0
description = 82540EM Gigabit Ethernet Controller
pci-address = 0000:00:0e.0
traffic-type = IP
traffic-direction = upstream
egress-port = 4

[PORT4]
name = 0000:00:0f.0
description = 82540EM Gigabit Ethernet Controller
pci-address = 0000:00:0f.0
traffic-type = IP
traffic-direction = downstream
egress-port = 3

[PORT5]
name = 0000:00:10.0
description = 82540EM Gigabit Ethernet Controller
pci-address = 0000:00:10.0
traffic-type = IP
traffic-direction = lbp
egress-port = 3
lbp-mac = fa:16:3e:a7:02:cc

[VM common]
max = 32
number = 2
vhost-dev = /var/lib/appliance/nts/qemu/usvhost-1

[NES_SERVER]
;ctrl_socket = /var/lib/appliance/nts/control-socket
;ctrl_ip = 127.0.0.1
ctrl_ip = 0.0.0.0
ctrl_port = 8080

[KNI]
max = 32

[REDIS_SERVER]
host = 127.0.0.1
port = 6379

测试项1-nr0向upf0进行ping包

截取日志，如下所示，IP包为四条log记录，ARP包为三条log记录。

[DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:1 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:1, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418
[DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:1, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:PORT_1_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_upstream_ip() -- 816
[DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:1, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

[DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:1 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:1, srcip:192.168.30.116, dstip:192.168.20.74, traffic-direction:FROM DWSTR, proto-type:IP, to_ring:NTS_DWSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 422
[DEBUG] thread:nts_io, module:flow, portname:0000:00:05.0, index:1, traffic-direction:FROM DWSTR TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_downstream_ip() -- 874
[DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:1, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

[DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:2 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:2, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418
[DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:2, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267
[DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:2, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:PORT_1_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_upstream_ip() -- 816

[DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:2 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:2, srcip:192.168.30.116, dstip:192.168.20.74, traffic-direction:FROM DWSTR, proto-type:IP, to_ring:NTS_DWSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 422
[DEBUG] thread:nts_io, module:flow, portname:0000:00:05.0, index:2, traffic-direction:FROM DWSTR TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_downstream_ip() -- 874
[DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:2, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

[DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:3 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:3, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418
[DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:3, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267
[DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:3, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:PORT_1_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_upstream_ip() -- 816

[DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:3 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:3, srcip:192.168.30.116, dstip:192.168.20.74, traffic-direction:FROM DWSTR, proto-type:IP, to_ring:NTS_DWSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 422
[DEBUG] thread:nts_io, module:flow, portname:0000:00:05.0, index:3, traffic-direction:FROM DWSTR TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_downstream_ip() -- 874
[DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:3, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

[DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:4 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:4, traffic-direction:FROM DWSTR, proto-type:NO ETHER_TYPE_IPv4, to_ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 219
[DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:4, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

[DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:4 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:4, traffic-direction:FROM UPSTR, proto-type:NO ETHER_TYPE_IPv4, to_ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 217
[DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:4, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

测试项2-upf0向nr0进行ping包

截取日志，如下所示，IP包为四条log记录，ARP包为三条log记录。

[DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:1 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:1, srcip:192.168.30.116, dstip:192.168.20.74, traffic-direction:FROM DWSTR, proto-type:IP, to_ring:NTS_DWSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 422
[DEBUG] thread:nts_io, module:flow, portname:0000:00:05.0, index:1, traffic-direction:FROM DWSTR TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_downstream_ip() -- 874
[DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:1, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

[DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:1 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:1, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418
[DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:1, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:PORT_1_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_upstream_ip() -- 816
[DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:1, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

[DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:2 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:2, srcip:192.168.30.116, dstip:192.168.20.74, traffic-direction:FROM DWSTR, proto-type:IP, to_ring:NTS_DWSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 422
[DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:2, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267
[DEBUG] thread:nts_io, module:flow, portname:0000:00:05.0, index:2, traffic-direction:FROM DWSTR TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_downstream_ip() -- 874

[DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:2 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:2, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418
[DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:2, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:PORT_1_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_upstream_ip() -- 816
[DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:2, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

[DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:3 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:3, srcip:192.168.30.116, dstip:192.168.20.74, traffic-direction:FROM DWSTR, proto-type:IP, to_ring:NTS_DWSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 422
[DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:3, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267
[DEBUG] thread:nts_io, module:flow, portname:0000:00:05.0, index:3, traffic-direction:FROM DWSTR TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_downstream_ip() -- 874

[DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:3 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:3, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418
[DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:3, traffic-direction:FROM UPSTR TO ENGRESS, proto-type:IP, to_ring:PORT_1_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_upstream_ip() -- 816
[DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:3, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

[DEBUG] thread:nes_io, module:recv, portname:0000:00:05.0, index:4 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:05.0, index:4, traffic-direction:FROM DWSTR, proto-type:NO ETHER_TYPE_IPv4, to_ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 219
[DEBUG] thread:nes_io, module:send, from portname:0000:00:05.0, to portname:0000:00:04.0, index:4, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

[DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:4 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:4, traffic-direction:FROM UPSTR, proto-type:NO ETHER_TYPE_IPv4, to_ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 217
[DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:05.0, index:4, from ring:PORT_1_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

测试项3-配置route后nr0向upf0进行ping包

route规则为：

route add fa:16:3e:c0:70:fa ace4d210-3d92-422b-83d7-11a0de50fea5  prio:98,srv_ip:192.168.30.116/32,encap_proto:noencap,dst_lbp_port:2

截取日志，如下所示，进入up0的IP包被卸载到lbp0。

[DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:1 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:1, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418
[DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:1, traffic-direction:TO LBP, to_ring:PORT_2_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_packet_edit_enq() -- 435
[DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:06.0, index:1, from ring:PORT_2_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

[DEBUG] thread:nes_io, module:recv, portname:0000:00:04.0, index:2 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:04.0, index:2, srcip:192.168.20.74, dstip:192.168.30.116, traffic-direction:FROM UPSTR, proto-type:IP, to_ring:NTS_UPSTR_IP /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_packets() -- 418
[DEBUG] thread:nes_io, module:send, from portname:0000:00:04.0, to portname:0000:00:06.0, index:2, from ring:PORT_2_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267
[DEBUG] thread:nts_io, module:flow, portname:0000:00:04.0, index:2, traffic-direction:TO LBP, to_ring:PORT_2_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_packet_edit_enq() -- 435

测试项4-配置route后dn0向nr0进行ping包

截取日志，如下所示，首先有个ARP代答的记录，然后lbp0的流量被送到了up0。

[DEBUG] thread:nes_io, module:recv, portname:0000:00:06.0, index:12 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:06.0, index:12, traffic-direction:FROM LBP, to_ring:NTS_LBP_ANY /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_lbp() -- 504
[DEBUG] thread:nts_io, module:flow, portname:0000:00:06.0, index:12, srcip:192.168.40.84, dstip:192.168.20.74, traffic-direction:FROM LBP, proto-type:ARP REQUEST
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_vm() -- 1209
[DEBUG] thread:nts_io, module:flow, portname:0000:00:06.0, index:12, srcip:192.168.20.74, dstip:192.168.40.84, traffic-direction:FROM LBP TO LBP, proto-type:ARP REPLY, to_ring:PORT_2_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_edge_arp_edit() -- 1110
[DEBUG] thread:nes_io, module:send, from portname:0000:00:06.0, to portname:0000:00:06.0, index:12, from ring:PORT_2_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267

[DEBUG] thread:nes_io, module:recv, portname:0000: 00:06.0, index:13 /opt/upgw/daemon/io/nes_dev_eth.c -- recv_eth() -- 211
[DEBUG] thread:nes_io, module:scatter, portname:0000:00:06.0, index:13, traffic-direction:FROM LBP, to_ring:NTS_LBP_ANY /opt/upgw/daemon/io/nes_dev_port.c -- scatter_eth_lbp() -- 504
[DEBUG] thread:nts_io, module:flow, portname:0000:00:06.0, index:13, srcip:192.168.40.84, dstip:192.168.20.74, traffic-direction:FROM LBP TO ENGRESS, proto-type:IP, to_ring:PORT_0_IO_ANY
 /opt/upgw/daemon/nts/nts_edit.c -- nts_flow_vm() -- 1242
[DEBUG] thread:nes_io, module:send, from portname:0000:00:06.0, to portname:0000:00:04.0, index:13, from ring:PORT_0_IO_ANY /opt/upgw/daemon/io/nes_dev_eth.c -- send_eth() -- 267