问题现象

继上一次【麒麟系统下Conntrack工具删除命令无法使用问题】[1]之后，麒麟SP03版本发布的内核4.19.90-52.42，又发现存在ping和发送UDP大包无响应问题。

[root@node1 ~]# uname -a
Linux node1 4.19.90-52.42.v2207.ky10.aarch64 #4 SMP Tue Oct 8 16:54:49 CST 2024 aarch64 aarch64 aarch64 GNU/Linux

复现方法

ping测试：

# 使用默认ping命令正常：
[root@node1 ~]# kubectl exec -it test-75f9dc9f95-gx6q6 bash
[root@test-75f9dc9f95-gx6q6 /]# ping 192.168.1.5
PING 192.168.1.5 (192.168.1.5) 56(84) bytes of data.
64 bytes from 192.168.1.5: icmp_seq=1 ttl=63 time=2.64 ms
64 bytes from 192.168.1.5: icmp_seq=2 ttl=63 time=0.567 ms
64 bytes from 192.168.1.5: icmp_seq=3 ttl=63 time=0.370 ms

# 使用ping命令自定义发送报文大小时，如果设置报文大小为4096，则无法ping通：
[root@test-75f9dc9f95-gx6q6 /]# ping 192.168.1.5 -s 4096
PING 192.168.1.5 (192.168.1.5) 4096(4124) bytes of data.
^C

抓包结果：

[root@node1 ~]# tcpdump -n -i any icmp and host 192.168.1.5
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on any, link-type LINUX_SLL (Linux cooked v1), capture size 262144 bytes
21:06:41.698485 IP 10.10.133.203 > 192.168.1.5: ICMP echo request, id 27, seq 1, length 1480
21:06:41.698530 IP 10.10.133.203 > 192.168.1.5: ip-proto-1
21:06:41.698536 IP 10.10.133.203 > 192.168.1.5: ip-proto-1
21:06:41.698644 IP 192.168.1.3 > 192.168.1.5: ICMP echo request, id 39858, seq 1, length 1480
21:06:41.698671 IP 192.168.1.3 > 192.168.1.5: ip-proto-1
21:06:41.698677 IP 192.168.1.3 > 192.168.1.5: ip-proto-1
21:06:41.700575 IP 192.168.1.5 > 192.168.1.3: ICMP echo reply, id 39858, seq 1, length 1480
21:06:41.700618 IP 192.168.1.5 > 192.168.1.3: ip-proto-1
21:06:41.700624 IP 192.168.1.5 > 192.168.1.3: ip-proto-1
21:06:41.700680 IP 192.168.1.5 > 10.10.133.203: ICMP echo reply, id 27, seq 1, length 1480
21:06:41.700689 IP 192.168.1.5 > 10.10.133.203: ip-proto-1
21:06:41.700692 IP 192.168.1.5 > 10.10.133.203: ip-proto-1
21:07:11.723710 IP 10.10.133.203 > 192.168.1.5: ICMP ip reassembly time exceeded, length 556

从抓包结果看，数据包在IP分片的重组过程中出现了超时。这意味着部分数据包的分片未能在允许的时间内成功重组，从而导致超时并发送该ICMP错误消息以通知发送方。

编写测试代码udp_client.cpp，验证UDP包：

#include <iostream>
#include <cstring>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <unistd.h>

void udp_client(const std::string &server_ip, int server_port, int packet_size) {
    int sock = socket(AF_INET, SOCK_DGRAM, 0);
    if (sock < 0) {
        std::cerr << "Socket creation failed!" << std::endl;
        return;
    }

    struct sockaddr_in server_address;
    memset(&server_address, 0, sizeof(server_address));
    server_address.sin_family = AF_INET;
    server_address.sin_port = htons(server_port);

    if (inet_pton(AF_INET, server_ip.c_str(), &server_address.sin_addr) <= 0) {
        std::cerr << "Invalid address!" << std::endl;
        close(sock);
        return;
    }

    // Create the message to be sent.
    char *message = new char[packet_size];
    memset(message, 'X', packet_size);

    // Send the entire packet at once.
    int sent_bytes = sendto(sock, message, packet_size, 0, (struct sockaddr *)&server_address, sizeof(server_address));
    if (sent_bytes < 0) {
        std::cerr << "Send failed!" << std::endl;
    } else {
        std::cout << "Sent packet of size " << sent_bytes << std::endl;
    }

    char buffer[packet_size];
    socklen_t server_address_len = sizeof(server_address);
    int received_bytes = recvfrom(sock, buffer, sizeof(buffer), 0, (struct sockaddr *)&server_address, &server_address_len);
    if (received_bytes < 0) {
        std::cerr << "Receive failed!" << std::endl;
    } else {
        std::cout << "Received echo of size " << received_bytes << ": " << std::string(buffer, received_bytes) << std::endl;
    }

    delete[] message;
    close(sock);
}

int main(int argc, char *argv[]) {
    if (argc != 4) {
        std::cerr << "Usage: " << argv[0] << " <server_ip> <server_port> <packet_size>" << std::endl;
        return 1;
    }

    std::string server_ip = argv[1];
    int server_port = std::stoi(argv[2]);
    int packet_size = std::stoi(argv[3]);

    // It is generally advisable to keep packet_size within 1472 bytes for ethernet to avoid fragmentation.
    if (packet_size > 1472) {
        std::cerr << "Warning: packet_size exceeds typical MTU limits. This might lead to fragmentation." << std::endl;
    }

    udp_client(server_ip, server_port, packet_size);
    return 0;
}

UDP Server端（为了跟问题环境保持一致，用java实现服务端）：

import java.net.*;

public class UDPServer {
    public static void main(String[] args) {
        int port = 12345;

        try (DatagramSocket socket = new DatagramSocket(port)) {
            System.out.println("Server listening on port " + port);

            byte[] buffer = new byte[1024];
            DatagramPacket packet = new DatagramPacket(buffer, buffer.length);

            while (true) {
                socket.receive(packet);
                String received = new String(packet.getData(), 0, packet.getLength());
                System.out.println("Received: " + received);

                // Echo back to client
                InetAddress clientAddress = packet.getAddress();
                int clientPort = packet.getPort();
                DatagramPacket echoPacket = new DatagramPacket(buffer, packet.getLength(), clientAddress, clientPort);
                socket.send(echoPacket);
            }
        } catch (Exception e) {
            System.out.println("Server error: " + e.getMessage());
        }
    }
}

测试方法：服务端，进入业务容器内，启动服务端监听：

[root@node1 ~]# kubectl exec -it test-75f9dc9f95-gx6q6 bash
[root@test-75f9dc9f95-gx6q6 /]# javac UDPServer.java
[root@test-75f9dc9f95-gx6q6 /]# java UDPServer
Server listening on port 12345

客户端，编译代码，分别发送1024，2048，4096，65535大小的包验证：

[root@node4 ~]# g++ -std=c++11 -o udp_client udp_client.cpp
[root@node4 ~]# ./udp_client 192.168.1.3 12345 1024
Sent packet of size 1024
Received echo of size 1024: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
[root@node4 ~]#
[root@node4 ~]# ./udp_client 192.168.1.3 12345 2048
Warning: packet_size exceeds typical MTU limits. This might lead to fragmentation.
Sent packet of size 2048
Received echo of size 1024: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
[root@node4 ~]#
[root@node4 ~]#
[root@node4 ~]# ./udp_client 192.168.1.3 12345 4096
Warning: packet_size exceeds typical MTU limits. This might lead to fragmentation.
Sent packet of size 4096
^C
[root@node4 ~]# ./udp_client 192.168.1.3 12345 65535
Warning: packet_size exceeds typical MTU limits. This might lead to fragmentation.
Send failed!
^C

抓包结果：

[root@node1 ~]# tcpdump -n -i any udp port 12345
19:57:52.147198 IP 192.168.1.5.57939 > 192.168.1.3.italk: UDP, length 1024
19:57:52.147385 IP 192.168.1.3.15642 > 10.10.133.203.italk: UDP, length 1024
19:57:52.148085 IP 10.10.133.203.italk > 192.168.1.3.15642: UDP, length 1024
19:57:52.148178 IP 192.168.1.3.italk > 192.168.1.5.57939: UDP, length 1024

19:58:01.952797 IP 192.168.1.5.53262 > 192.168.1.3.italk: UDP, bad length 2048 > 1472
19:58:01.952938 IP 192.168.1.3.13362 > 10.10.133.203.italk: UDP, bad length 2048 > 1472
19:58:01.953399 IP 10.10.133.203.italk > 192.168.1.3.13362: UDP, length 1024
19:58:01.953435 IP 192.168.1.3.italk > 192.168.1.5.53262: UDP, length 1024

19:58:22.270227 IP 192.168.1.5.50365 > 192.168.1.3.italk: UDP, bad length 4096 > 1472
19:58:22.270459 IP 192.168.1.3.ndm-server > 10.10.133.203.italk: UDP, bad length 4096 > 1472

# 之后无抓包结果

对比麒麟SP03的原生内核4.19.90-52.22，相同的测试方法，ping和UDP请求都可以正常响应，说明是内核版本升级引起的问题。

解决方案

麒麟官方确认存在问题，计划在4.19.90-52.43内核版本里解决。另外，麒麟SP03-2403版本的4.19.90-89.17内核同样存在问题，计划在4.19.90-89.18内核版本里解决。

另外，对于业务侧发送UDP大包场景，建议主动做分片，相关资料说明如下：

请注意，UDP本身不提供任何分片机制，它依赖于IP层的分片。发送的数据包大小不应大于网络的MTU（通常是1500字节）；否则，IP层会对其进行分片，如果发送的数据超过MTU，很有可能由于网络限制导致分片包被丢弃。因此，最佳实践是在应用层中将包大小控制在安全范围内，而不是通过UDP发送单个大包。

参考资料

https://lyyao09.github.io/2024/09/08/k8s/K8S%E9%97%AE%E9%A2%98%E6%8E%92%E6%9F%A5-%E9%BA%92%E9%BA%9F%E7%B3%BB%E7%BB%9F%E4%B8%8BConntrack%E5%B7%A5%E5%85%B7%E5%88%A0%E9%99%A4%E5%91%BD%E4%BB%A4%E6%97%A0%E6%B3%95%E4%BD%BF%E7%94%A8%E9%97%AE%E9%A2%98/