Netlink Performance 測試

這篇文章的由來在於客戶說的一句話：「Netlink 的效能似乎不太好」客戶口中的不太好指的是 Throughput 只有 10Mbps。本來嘛，我想之後才處理這件事情，但結果我的一個同事 York 抱著追根究底的精神進行了下面的實驗。首先，他用一支 user space 的程式來產生封包到 kernel space 的模組，模組收到以後就將封包打回 user space。簡單來講究是一個 echo 的行為。下面會列出這兩隻程式：

實驗平台：某平台
Kernel 版本：3.10

User Space：

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 

#define NETLINK_TEST 18
//#define MAX_PAYLOAD 1024
//#define MAX_PAYLOAD 2048
//#define MAX_PAYLOAD 4096
#define MAX_PAYLOAD 8192
#define MESSAGE_COUNT 1048576

struct sockaddr_nl src_addr, dest_addr;
struct msghdr msg;
struct nlmsghdr *nlh = NULL;
struct iovec iov;
int sock_fd;

void main()
{
    int _i = 0;

    time_t startTime = 0;
    time_t endTime = 0;

    sock_fd = socket(PF_NETLINK, SOCK_RAW, NETLINK_TEST);

    memset(&src_addr, 0, sizeof(src_addr));
    src_addr.nl_family = AF_NETLINK;
    src_addr.nl_pid = getpid();
    src_addr.nl_groups = 0;

    bind(sock_fd, (struct sockaddr*)&src_addr, sizeof(src_addr));

    memset(&dest_addr, 0, sizeof(dest_addr));
    dest_addr.nl_family = AF_NETLINK;
    dest_addr.nl_pid = 0;    // send to kernel
    dest_addr.nl_groups = 0;

    startTime = time(NULL);
    printf("start time: %s\n", ctime(&startTime));
    for (_i = 0; _i < MESSAGE_COUNT; _i++)
    {
        nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(MAX_PAYLOAD));
        memset(nlh, 0, NLMSG_SPACE(MAX_PAYLOAD));

        nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD);
        nlh->nlmsg_pid = getpid();
        nlh->nlmsg_flags = 0;

        iov.iov_base = (void *)nlh;
        iov.iov_len = nlh->nlmsg_len;
        msg.msg_name = (void *)&dest_addr;
        msg.msg_namelen = sizeof(dest_addr);
        msg.msg_iov = &iov;
        msg.msg_iovlen = 1;


        strcpy(NLMSG_DATA(nlh), "Hello Kernel");
        sendmsg(sock_fd, &msg, 0);  // send message to kernel
        memset(nlh, 0, NLMSG_SPACE(strlen("Hello Kernel")));
        recvmsg(sock_fd, &msg, 0);

        free(nlh);
    }

    endTime = time(NULL);
    printf("end time: %s\n", ctime(&endTime));

    printf("Avg speed is %lf Byte/s\n", (double)((unsigned long long)MESSAGE_COUNT * (unsigned long long)MAX_PAYLOAD) / difftime(endTime, startTime));

    close(sock_fd);
}

Kernel Space：

/* 
* Dist/Platform Kernel Version:
*     Ubuntu 14.04(Linux Mint 17): 3.13
*     CentOS 7: 3.10
*     Cavium AC: 3.10
*     Ubuntu 12.04(Linux Mint 13): 3.2
*/
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 

#define NETLINK_TEST 18
//#define MAX_PAYLOAD 1024
//#define MAX_PAYLOAD 2048
//#define MAX_PAYLOAD 4096
#define MAX_PAYLOAD 8192

MODULE_LICENSE("GPL");

static struct sock *nl_sk = NULL;

int netlinkSayHello(int pid)
{
    struct sk_buff *skb;
    struct nlmsghdr *nlh;

    skb = nlmsg_new(MAX_PAYLOAD, GFP_ATOMIC);
    if (skb == NULL)
    {
        printk(KERN_ERR "Failed to alloc skb\n");
        return 0;
    }

    // put into skb
    nlh = nlmsg_put(skb, 0, 0, 0, MAX_PAYLOAD, 0);

    // below line is meaningless
    memcpy(NLMSG_DATA(nlh), "Hello Client", sizeof("Hello Client"));

    if (netlink_unicast(nl_sk, skb, pid, 0) < 0)
    {
        printk(KERN_ERR"Failed to unicast skb\n");
        return 0;
    }

    return 1;
}

static void nl_data_ready(struct sk_buff *skb)
{
    struct nlmsghdr *nlh = NULL;

    if (skb == NULL)
    {
        printk(KERN_INFO"skb is NULL\n");
        return;
    }

    nlh = (struct nlmsghdr *)skb->data;

    netlinkSayHello(nlh->nlmsg_pid);
}

static void netlink_test(void)
{
/* for Cavium AC/CentOS 7/Ubuntu 14.04/Linux Mint 17 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)
    struct netlink_kernel_cfg cfg = {
        .input = nl_data_ready,
    };
    nl_sk = netlink_kernel_create(&init_net, NETLINK_TEST, &cfg);

/* for Ubuntu 12.04/Linux Mint 13 */
//#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0)
#else
    nl_sk = netlink_kernel_create(&init_net,
                                  NETLINK_TEST,
                                  0,
                                  nl_data_ready,
                                  NULL,
                                  THIS_MODULE);
#endif

    if (nl_sk == NULL)
    {
        printk(KERN_ERR "Failed to create netlink socket\n");
    }
}

static int __init my_module_init(void)
{
    printk(KERN_INFO "Initializing Netlink Socket\n");
    netlink_test();

    return 0;
}

static void __exit my_module_exit(void)
{
    netlink_kernel_release(nl_sk);
    printk(KERN_INFO "Goodbye\n");
}

module_init(my_module_init);
module_exit(my_module_exit);

實驗的結果如下：

1KB * 1048576 (1G)
Avg speed is 67108864.000000 Byte/s = 67MBps
2KB * 1048576 (2G)
Avg speed is 89478485.333333 Byte/s = 89MBps
4KB * 1048576 (4G)
Avg speed is 226050910.315789 Byte/s = 226MBps
8KB * 1048576 (8G)
Avg speed is 268435456.000000 Byte/s = 268MBps

看起來比想像中好很多呢~