Hello,
I face a problem due to binary compatibility of XFRM structures.
I use XFRM interface to create IPsec SAs. My user application is compiled as 32bit binary.
It works properly when running against 32bit kernel. However when I've run it on 64bit kernel, netlink returns an error (EINVAL).
I use Fedora for both 32 and 64-bit environment.
I'm aware that it is due to incompatibility (different alignment and padding) of XFRM structures passed between user space and kernel.
My question is: shall not the structures be designed to be the same in 32 and 64bit environment?
I think a padding should be added to meet the more strict alignment rules (i.e. 64bit environment).
I used pahole tool (available in dwarves package) to display structure alignment (see attachment).
pahole_32.txt - structures from 32bit binary
pahole_64.txt - structures from 64bit binary
What causes my trouble is "xfrm_usersa_info" which is padded with 7 bytes at the end in 64-bit,
so that the whole structure is 224 bytes in 64bit application against 220 bytes in 32bit (just 3-byte padding).
An explicit 7-byte padding in the structure would cure the case, IMHO.
Thanks for your comments.
Jiri Klimes
struct xfrm_userpolicy_id {
struct xfrm_selector sel; /* 0 56 */
__u32 index; /* 56 4 */
__u8 dir; /* 60 1 */
/* size: 64, cachelines: 1 */
/* padding: 3 */
}; /* definitions: 1 */
struct xfrm_userpolicy_info {
struct xfrm_selector sel; /* 0 56 */
struct xfrm_lifetime_cfg lft; /* 56 64 */
/* --- cacheline 1 boundary (64 bytes) was 56 bytes ago --- */
struct xfrm_lifetime_cur curlft; /* 120 32 */
/* --- cacheline 2 boundary (128 bytes) was 24 bytes ago --- */
__u32 priority; /* 152 4 */
__u32 index; /* 156 4 */
__u8 dir; /* 160 1 */
__u8 action; /* 161 1 */
__u8 flags; /* 162 1 */
__u8 share; /* 163 1 */
/* size: 164, cachelines: 3 */
/* last cacheline: 36 bytes */
}; /* definitions: 1 */
struct xfrm_usersa_id {
xfrm_address_t daddr; /* 0 16 */
__be32 spi; /* 16 4 */
__u16 family; /* 20 2 */
__u8 proto; /* 22 1 */
/* size: 24, cachelines: 1 */
/* padding: 1 */
/* last cacheline: 24 bytes */
}; /* definitions: 1 */
struct xfrm_usersa_info {
struct xfrm_selector sel; /* 0 56 */
struct xfrm_id id; /* 56 24 */
/* XXX last struct has 3 bytes of padding */
/* --- cacheline 1 boundary (64 bytes) was 16 bytes ago --- */
xfrm_address_t saddr; /* 80 16 */
struct xfrm_lifetime_cfg lft; /* 96 64 */
/* --- cacheline 2 boundary (128 bytes) was 32 bytes ago --- */
struct xfrm_lifetime_cur curlft; /* 160 32 */
/* --- cacheline 3 boundary (192 bytes) --- */
struct xfrm_stats stats; /* 192 12 */
__u32 seq; /* 204 4 */
__u32 reqid; /* 208 4 */
__u16 family; /* 212 2 */
__u8 mode; /* 214 1 */
__u8 replay_window; /* 215 1 */
__u8 flags; /* 216 1 */
/* size: 220, cachelines: 4 */
/* padding: 3 */
/* paddings: 1, sum paddings: 3 */
/* last cacheline: 28 bytes */
}; /* definitions: 1 */
struct xfrm_encap_tmpl {
__u16 encap_type; /* 0 2 */
__be16 encap_sport; /* 2 2 */
__be16 encap_dport; /* 4 2 */
/* XXX 2 bytes hole, try to pack */
xfrm_address_t encap_oa; /* 8 16 */
/* size: 24, cachelines: 1 */
/* sum members: 22, holes: 1, sum holes: 2 */
/* last cacheline: 24 bytes */
}; /* definitions: 1 */
struct xfrm_user_tmpl {
struct xfrm_id id; /* 0 24 */
/* XXX last struct has 3 bytes of padding */
__u16 family; /* 24 2 */
/* XXX 2 bytes hole, try to pack */
xfrm_address_t saddr; /* 28 16 */
__u32 reqid; /* 44 4 */
__u8 mode; /* 48 1 */
__u8 share; /* 49 1 */
__u8 optional; /* 50 1 */
/* XXX 1 byte hole, try to pack */
__u32 aalgos; /* 52 4 */
__u32 ealgos; /* 56 4 */
__u32 calgos; /* 60 4 */
/* --- cacheline 1 boundary (64 bytes) --- */
/* size: 64, cachelines: 1 */
/* sum members: 61, holes: 2, sum holes: 3 */
/* paddings: 1, sum paddings: 3 */
}; /* definitions: 1 */
struct xfrm_stats {
__u32 replay_window; /* 0 4 */
__u32 replay; /* 4 4 */
__u32 integrity_failed; /* 8 4 */
/* size: 12, cachelines: 1 */
/* last cacheline: 12 bytes */
}; /* definitions: 1 */
struct xfrm_algo {
char alg_name[64]; /* 0 64 */
/* --- cacheline 1 boundary (64 bytes) --- */
unsigned int alg_key_len; /* 64 4 */
char alg_key[0]; /* 68 0 */
/* size: 68, cachelines: 2 */
/* last cacheline: 4 bytes */
}; /* definitions: 1 */
struct xfrm_lifetime_cur {
__u64 bytes; /* 0 8 */
__u64 packets; /* 8 8 */
__u64 add_time; /* 16 8 */
__u64 use_time; /* 24 8 */
/* size: 32, cachelines: 1 */
/* last cacheline: 32 bytes */
}; /* definitions: 1 */
struct xfrm_lifetime_cfg {
__u64 soft_byte_limit; /* 0 8 */
__u64 hard_byte_limit; /* 8 8 */
__u64 soft_packet_limit; /* 16 8 */
__u64 hard_packet_limit; /* 24 8 */
__u64 soft_add_expires_seconds; /* 32 8 */
__u64 hard_add_expires_seconds; /* 40 8 */
__u64 soft_use_expires_seconds; /* 48 8 */
__u64 hard_use_expires_seconds; /* 56 8 */
/* --- cacheline 1 boundary (64 bytes) --- */
/* size: 64, cachelines: 1 */
}; /* definitions: 1 */
struct xfrm_selector {
xfrm_address_t daddr; /* 0 16 */
xfrm_address_t saddr; /* 16 16 */
__be16 dport; /* 32 2 */
__be16 dport_mask; /* 34 2 */
__be16 sport; /* 36 2 */
__be16 sport_mask; /* 38 2 */
__u16 family; /* 40 2 */
__u8 prefixlen_d; /* 42 1 */
__u8 prefixlen_s; /* 43 1 */
__u8 proto; /* 44 1 */
/* XXX 3 bytes hole, try to pack */
int ifindex; /* 48 4 */
uid_t user; /* 52 4 */
/* size: 56, cachelines: 1 */
/* sum members: 53, holes: 1, sum holes: 3 */
/* last cacheline: 56 bytes */
}; /* definitions: 1 */
struct xfrm_id {
xfrm_address_t daddr; /* 0 16 */
__be32 spi; /* 16 4 */
__u8 proto; /* 20 1 */
/* size: 24, cachelines: 1 */
/* padding: 3 */
/* last cacheline: 24 bytes */
}; /* definitions: 1 */
struct rtattr {
short unsigned int rta_len; /* 0 2 */
short unsigned int rta_type; /* 2 2 */
/* size: 4, cachelines: 1 */
/* last cacheline: 4 bytes */
}; /* definitions: 1 */
struct nlmsgerr {
int error; /* 0 4 */
struct nlmsghdr msg; /* 4 16 */
/* size: 20, cachelines: 1 */
/* last cacheline: 20 bytes */
}; /* definitions: 1 */
struct nlmsghdr {
__u32 nlmsg_len; /* 0 4 */
__u16 nlmsg_type; /* 4 2 */
__u16 nlmsg_flags; /* 6 2 */
__u32 nlmsg_seq; /* 8 4 */
__u32 nlmsg_pid; /* 12 4 */
/* size: 16, cachelines: 1 */
/* last cacheline: 16 bytes */
}; /* definitions: 1 */
struct sockaddr_nl {
sa_family_t nl_family; /* 0 2 */
short unsigned int nl_pad; /* 2 2 */
__u32 nl_pid; /* 4 4 */
__u32 nl_groups; /* 8 4 */
/* size: 12, cachelines: 1 */
/* last cacheline: 12 bytes */
}; /* definitions: 1 */
struct sockaddr_in6 {
sa_family_t sin6_family; /* 0 2 */
in_port_t sin6_port; /* 2 2 */
uint32_t sin6_flowinfo; /* 4 4 */
struct in6_addr sin6_addr; /* 8 16 */
uint32_t sin6_scope_id; /* 24 4 */
/* size: 28, cachelines: 1 */
/* last cacheline: 28 bytes */
}; /* definitions: 1 */
struct sockaddr_in {
sa_family_t sin_family; /* 0 2 */
in_port_t sin_port; /* 2 2 */
struct in_addr sin_addr; /* 4 4 */
unsigned char sin_zero[8]; /* 8 8 */
/* size: 16, cachelines: 1 */
/* last cacheline: 16 bytes */
}; /* definitions: 1 */
struct in6_addr {
union {
uint8_t u6_addr8[16]; /* 16 */
uint16_t u6_addr16[8]; /* 16 */
uint32_t u6_addr32[4]; /* 16 */
} in6_u; /* 0 16 */
/* size: 16, cachelines: 1 */
/* last cacheline: 16 bytes */
}; /* definitions: 1 */
struct in_addr {
in_addr_t s_addr; /* 0 4 */
/* size: 4, cachelines: 1 */
/* last cacheline: 4 bytes */
}; /* definitions: 1 */
struct msghdr {
void * msg_name; /* 0 4 */
socklen_t msg_namelen; /* 4 4 */
struct iovec * msg_iov; /* 8 4 */
size_t msg_iovlen; /* 12 4 */
void * msg_control; /* 16 4 */
size_t msg_controllen; /* 20 4 */
int msg_flags; /* 24 4 */
/* size: 28, cachelines: 1 */
/* last cacheline: 28 bytes */
}; /* definitions: 1 */
struct sockaddr {
sa_family_t sa_family; /* 0 2 */
char sa_data[14]; /* 2 14 */
/* size: 16, cachelines: 1 */
/* last cacheline: 16 bytes */
}; /* definitions: 1 */
struct iovec {
void * iov_base; /* 0 4 */
size_t iov_len; /* 4 4 */
/* size: 8, cachelines: 1 */
/* last cacheline: 8 bytes */
}; /* definitions: 1 */
struct _IO_marker {
struct _IO_marker * _next; /* 0 4 */
struct _IO_FILE * _sbuf; /* 4 4 */
int _pos; /* 8 4 */
/* size: 12, cachelines: 1 */
/* last cacheline: 12 bytes */
}; /* definitions: 1 */
struct _IO_FILE {
int _flags; /* 0 4 */
char * _IO_read_ptr; /* 4 4 */
char * _IO_read_end; /* 8 4 */
char * _IO_read_base; /* 12 4 */
char * _IO_write_base; /* 16 4 */
char * _IO_write_ptr; /* 20 4 */
char * _IO_write_end; /* 24 4 */
char * _IO_buf_base; /* 28 4 */
char * _IO_buf_end; /* 32 4 */
char * _IO_save_base; /* 36 4 */
char * _IO_backup_base; /* 40 4 */
char * _IO_save_end; /* 44 4 */
struct _IO_marker * _markers; /* 48 4 */
struct _IO_FILE * _chain; /* 52 4 */
int _fileno; /* 56 4 */
int _flags2; /* 60 4 */
/* --- cacheline 1 boundary (64 bytes) --- */
__off_t _old_offset; /* 64 4 */
short unsigned int _cur_column; /* 68 2 */
signed char _vtable_offset; /* 70 1 */
char _shortbuf[1]; /* 71 1 */
_IO_lock_t * _lock; /* 72 4 */
__off64_t _offset; /* 76 8 */
void * __pad1; /* 84 4 */
void * __pad2; /* 88 4 */
void * __pad3; /* 92 4 */
void * __pad4; /* 96 4 */
size_t __pad5; /* 100 4 */
int _mode; /* 104 4 */
char _unused2[40]; /* 108 40 */
/* --- cacheline 2 boundary (128 bytes) was 20 bytes ago --- */
/* size: 148, cachelines: 3 */
/* last cacheline: 20 bytes */
}; /* definitions: 1 */
struct xfrm_userpolicy_id {
struct xfrm_selector sel; /* 0 56 */
__u32 index; /* 56 4 */
__u8 dir; /* 60 1 */
/* size: 64, cachelines: 1 */
/* padding: 3 */
}; /* definitions: 1 */
struct xfrm_userpolicy_info {
struct xfrm_selector sel; /* 0 56 */
struct xfrm_lifetime_cfg lft; /* 56 64 */
/* --- cacheline 1 boundary (64 bytes) was 56 bytes ago --- */
struct xfrm_lifetime_cur curlft; /* 120 32 */
/* --- cacheline 2 boundary (128 bytes) was 24 bytes ago --- */
__u32 priority; /* 152 4 */
__u32 index; /* 156 4 */
__u8 dir; /* 160 1 */
__u8 action; /* 161 1 */
__u8 flags; /* 162 1 */
__u8 share; /* 163 1 */
/* size: 168, cachelines: 3 */
/* padding: 4 */
/* last cacheline: 40 bytes */
}; /* definitions: 1 */
struct xfrm_usersa_id {
xfrm_address_t daddr; /* 0 16 */
__be32 spi; /* 16 4 */
__u16 family; /* 20 2 */
__u8 proto; /* 22 1 */
/* size: 24, cachelines: 1 */
/* padding: 1 */
/* last cacheline: 24 bytes */
}; /* definitions: 1 */
struct xfrm_usersa_info {
struct xfrm_selector sel; /* 0 56 */
struct xfrm_id id; /* 56 24 */
/* XXX last struct has 3 bytes of padding */
/* --- cacheline 1 boundary (64 bytes) was 16 bytes ago --- */
xfrm_address_t saddr; /* 80 16 */
struct xfrm_lifetime_cfg lft; /* 96 64 */
/* --- cacheline 2 boundary (128 bytes) was 32 bytes ago --- */
struct xfrm_lifetime_cur curlft; /* 160 32 */
/* --- cacheline 3 boundary (192 bytes) --- */
struct xfrm_stats stats; /* 192 12 */
__u32 seq; /* 204 4 */
__u32 reqid; /* 208 4 */
__u16 family; /* 212 2 */
__u8 mode; /* 214 1 */
__u8 replay_window; /* 215 1 */
__u8 flags; /* 216 1 */
/* size: 224, cachelines: 4 */
/* padding: 7 */
/* paddings: 1, sum paddings: 3 */
/* last cacheline: 32 bytes */
}; /* definitions: 1 */
struct xfrm_encap_tmpl {
__u16 encap_type; /* 0 2 */
__be16 encap_sport; /* 2 2 */
__be16 encap_dport; /* 4 2 */
/* XXX 2 bytes hole, try to pack */
xfrm_address_t encap_oa; /* 8 16 */
/* size: 24, cachelines: 1 */
/* sum members: 22, holes: 1, sum holes: 2 */
/* last cacheline: 24 bytes */
}; /* definitions: 1 */
struct xfrm_user_tmpl {
struct xfrm_id id; /* 0 24 */
/* XXX last struct has 3 bytes of padding */
__u16 family; /* 24 2 */
/* XXX 2 bytes hole, try to pack */
xfrm_address_t saddr; /* 28 16 */
__u32 reqid; /* 44 4 */
__u8 mode; /* 48 1 */
__u8 share; /* 49 1 */
__u8 optional; /* 50 1 */
/* XXX 1 byte hole, try to pack */
__u32 aalgos; /* 52 4 */
__u32 ealgos; /* 56 4 */
__u32 calgos; /* 60 4 */
/* --- cacheline 1 boundary (64 bytes) --- */
/* size: 64, cachelines: 1 */
/* sum members: 61, holes: 2, sum holes: 3 */
/* paddings: 1, sum paddings: 3 */
}; /* definitions: 1 */
struct xfrm_stats {
__u32 replay_window; /* 0 4 */
__u32 replay; /* 4 4 */
__u32 integrity_failed; /* 8 4 */
/* size: 12, cachelines: 1 */
/* last cacheline: 12 bytes */
}; /* definitions: 1 */
struct xfrm_algo {
char alg_name[64]; /* 0 64 */
/* --- cacheline 1 boundary (64 bytes) --- */
unsigned int alg_key_len; /* 64 4 */
char alg_key[0]; /* 68 0 */
/* size: 68, cachelines: 2 */
/* last cacheline: 4 bytes */
}; /* definitions: 1 */
struct xfrm_lifetime_cur {
__u64 bytes; /* 0 8 */
__u64 packets; /* 8 8 */
__u64 add_time; /* 16 8 */
__u64 use_time; /* 24 8 */
/* size: 32, cachelines: 1 */
/* last cacheline: 32 bytes */
}; /* definitions: 1 */
struct xfrm_lifetime_cfg {
__u64 soft_byte_limit; /* 0 8 */
__u64 hard_byte_limit; /* 8 8 */
__u64 soft_packet_limit; /* 16 8 */
__u64 hard_packet_limit; /* 24 8 */
__u64 soft_add_expires_seconds; /* 32 8 */
__u64 hard_add_expires_seconds; /* 40 8 */
__u64 soft_use_expires_seconds; /* 48 8 */
__u64 hard_use_expires_seconds; /* 56 8 */
/* --- cacheline 1 boundary (64 bytes) --- */
/* size: 64, cachelines: 1 */
}; /* definitions: 1 */
struct xfrm_selector {
xfrm_address_t daddr; /* 0 16 */
xfrm_address_t saddr; /* 16 16 */
__be16 dport; /* 32 2 */
__be16 dport_mask; /* 34 2 */
__be16 sport; /* 36 2 */
__be16 sport_mask; /* 38 2 */
__u16 family; /* 40 2 */
__u8 prefixlen_d; /* 42 1 */
__u8 prefixlen_s; /* 43 1 */
__u8 proto; /* 44 1 */
/* XXX 3 bytes hole, try to pack */
int ifindex; /* 48 4 */
uid_t user; /* 52 4 */
/* size: 56, cachelines: 1 */
/* sum members: 53, holes: 1, sum holes: 3 */
/* last cacheline: 56 bytes */
}; /* definitions: 1 */
struct xfrm_id {
xfrm_address_t daddr; /* 0 16 */
__be32 spi; /* 16 4 */
__u8 proto; /* 20 1 */
/* size: 24, cachelines: 1 */
/* padding: 3 */
/* last cacheline: 24 bytes */
}; /* definitions: 1 */
struct rtattr {
short unsigned int rta_len; /* 0 2 */
short unsigned int rta_type; /* 2 2 */
/* size: 4, cachelines: 1 */
/* last cacheline: 4 bytes */
}; /* definitions: 1 */
struct nlmsgerr {
int error; /* 0 4 */
struct nlmsghdr msg; /* 4 16 */
/* size: 20, cachelines: 1 */
/* last cacheline: 20 bytes */
}; /* definitions: 1 */
struct nlmsghdr {
__u32 nlmsg_len; /* 0 4 */
__u16 nlmsg_type; /* 4 2 */
__u16 nlmsg_flags; /* 6 2 */
__u32 nlmsg_seq; /* 8 4 */
__u32 nlmsg_pid; /* 12 4 */
/* size: 16, cachelines: 1 */
/* last cacheline: 16 bytes */
}; /* definitions: 1 */
struct sockaddr_nl {
sa_family_t nl_family; /* 0 2 */
short unsigned int nl_pad; /* 2 2 */
__u32 nl_pid; /* 4 4 */
__u32 nl_groups; /* 8 4 */
/* size: 12, cachelines: 1 */
/* last cacheline: 12 bytes */
}; /* definitions: 1 */
struct sockaddr_in6 {
sa_family_t sin6_family; /* 0 2 */
in_port_t sin6_port; /* 2 2 */
uint32_t sin6_flowinfo; /* 4 4 */
struct in6_addr sin6_addr; /* 8 16 */
uint32_t sin6_scope_id; /* 24 4 */
/* size: 28, cachelines: 1 */
/* last cacheline: 28 bytes */
}; /* definitions: 1 */
struct sockaddr_in {
sa_family_t sin_family; /* 0 2 */
in_port_t sin_port; /* 2 2 */
struct in_addr sin_addr; /* 4 4 */
unsigned char sin_zero[8]; /* 8 8 */
/* size: 16, cachelines: 1 */
/* last cacheline: 16 bytes */
}; /* definitions: 1 */
struct in6_addr {
union {
uint8_t __u6_addr8[16]; /* 16 */
uint16_t __u6_addr16[8]; /* 16 */
uint32_t __u6_addr32[4]; /* 16 */
} __in6_u; /* 0 16 */
/* size: 16, cachelines: 1 */
/* last cacheline: 16 bytes */
}; /* definitions: 1 */
struct in_addr {
in_addr_t s_addr; /* 0 4 */
/* size: 4, cachelines: 1 */
/* last cacheline: 4 bytes */
}; /* definitions: 1 */
struct msghdr {
void * msg_name; /* 0 8 */
socklen_t msg_namelen; /* 8 4 */
/* XXX 4 bytes hole, try to pack */
struct iovec * msg_iov; /* 16 8 */
size_t msg_iovlen; /* 24 8 */
void * msg_control; /* 32 8 */
size_t msg_controllen; /* 40 8 */
int msg_flags; /* 48 4 */
/* size: 56, cachelines: 1 */
/* sum members: 48, holes: 1, sum holes: 4 */
/* padding: 4 */
/* last cacheline: 56 bytes */
}; /* definitions: 1 */
struct sockaddr {
sa_family_t sa_family; /* 0 2 */
char sa_data[14]; /* 2 14 */
/* size: 16, cachelines: 1 */
/* last cacheline: 16 bytes */
}; /* definitions: 1 */
struct iovec {
void * iov_base; /* 0 8 */
size_t iov_len; /* 8 8 */
/* size: 16, cachelines: 1 */
/* last cacheline: 16 bytes */
}; /* definitions: 1 */
struct _IO_marker {
struct _IO_marker * _next; /* 0 8 */
struct _IO_FILE * _sbuf; /* 8 8 */
int _pos; /* 16 4 */
/* size: 24, cachelines: 1 */
/* padding: 4 */
/* last cacheline: 24 bytes */
}; /* definitions: 1 */
struct _IO_FILE {
int _flags; /* 0 4 */
/* XXX 4 bytes hole, try to pack */
char * _IO_read_ptr; /* 8 8 */
char * _IO_read_end; /* 16 8 */
char * _IO_read_base; /* 24 8 */
char * _IO_write_base; /* 32 8 */
char * _IO_write_ptr; /* 40 8 */
char * _IO_write_end; /* 48 8 */
char * _IO_buf_base; /* 56 8 */
/* --- cacheline 1 boundary (64 bytes) --- */
char * _IO_buf_end; /* 64 8 */
char * _IO_save_base; /* 72 8 */
char * _IO_backup_base; /* 80 8 */
char * _IO_save_end; /* 88 8 */
struct _IO_marker * _markers; /* 96 8 */
struct _IO_FILE * _chain; /* 104 8 */
int _fileno; /* 112 4 */
int _flags2; /* 116 4 */
__off_t _old_offset; /* 120 8 */
/* --- cacheline 2 boundary (128 bytes) --- */
short unsigned int _cur_column; /* 128 2 */
signed char _vtable_offset; /* 130 1 */
char _shortbuf[1]; /* 131 1 */
/* XXX 4 bytes hole, try to pack */
_IO_lock_t * _lock; /* 136 8 */
__off64_t _offset; /* 144 8 */
void * __pad1; /* 152 8 */
void * __pad2; /* 160 8 */
void * __pad3; /* 168 8 */
void * __pad4; /* 176 8 */
size_t __pad5; /* 184 8 */
/* --- cacheline 3 boundary (192 bytes) --- */
int _mode; /* 192 4 */
char _unused2[20]; /* 196 20 */
/* size: 216, cachelines: 4 */
/* sum members: 208, holes: 2, sum holes: 8 */
/* last cacheline: 24 bytes */
}; /* definitions: 1 */
