[PATCH net] xfrm: fix integer overflow in xfrm_replay_state_esn_len()

[Dan Carpenter <dan.carpenter@xxxxxxxxxx>]

The problem is that "replay_esn->bmp_len" comes from the user and it's
a u32.  The xfrm_replay_state_esn_len() function also returns a u32.
So if we choose a ->bmp_len which very high then the total will be
more than UINT_MAX and value will be truncated when we return.  The
returned value will be smaller than expected causing problems in the
caller.

To fix this:
1) Use size_add() and size_mul().  This change is necessary for 32bit
   systems.
2) Change the type of xfrm_replay_state_esn_len() and related variables
   from u32 to size_t.
3) Remove the casts to (int).  The size should never be negative.
   Generally, values which come from size_add/mul() should stay as type
   size_t and not be truncated to user fewer than all the bytes in a
   unsigned long.

Cc: stable@xxxxxxxxxxxxxxx
Fixes: 9736acf395d3 ("xfrm: Add basic infrastructure to support IPsec extended sequence numbers")
Signed-off-by: Dan Carpenter <dan.carpenter@xxxxxxxxxx>
---
The one caller that I didn't modify was xfrm_sa_len().  That's a bit
complicated and also I'm kind of hoping that we don't handle user
controlled data in that function?  The place where we definitely are
handling user data is in xfrm_alloc_replay_state_esn() and this patch
fixes that.

 include/net/xfrm.h   |  4 ++--
 net/xfrm/xfrm_user.c | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index ed4b83696c77..0a42614d7840 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -1981,9 +1981,9 @@ static inline unsigned int xfrm_alg_auth_len(const struct xfrm_algo_auth *alg)
 	return sizeof(*alg) + ((alg->alg_key_len + 7) / 8);
 }
 
-static inline unsigned int xfrm_replay_state_esn_len(struct xfrm_replay_state_esn *replay_esn)
+static inline size_t xfrm_replay_state_esn_len(struct xfrm_replay_state_esn *replay_esn)
 {
-	return sizeof(*replay_esn) + replay_esn->bmp_len * sizeof(__u32);
+	return size_add(sizeof(*replay_esn), size_mul(replay_esn->bmp_len, sizeof(__u32)));
 }
 
 #ifdef CONFIG_XFRM_MIGRATE
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
index 08c6d6f0179f..4bfa72547dab 100644
--- a/net/xfrm/xfrm_user.c
+++ b/net/xfrm/xfrm_user.c
@@ -151,7 +151,7 @@ static inline int verify_replay(struct xfrm_usersa_info *p,
 		return -EINVAL;
 	}
 
-	if (nla_len(rt) < (int)xfrm_replay_state_esn_len(rs) &&
+	if (nla_len(rt) < xfrm_replay_state_esn_len(rs) &&
 	    nla_len(rt) != sizeof(*rs)) {
 		NL_SET_ERR_MSG(extack, "ESN attribute is too short to fit the full bitmap length");
 		return -EINVAL;
@@ -681,7 +681,7 @@ static inline int xfrm_replay_verify_len(struct xfrm_replay_state_esn *replay_es
 					 struct netlink_ext_ack *extack)
 {
 	struct xfrm_replay_state_esn *up;
-	unsigned int ulen;
+	size_t ulen;
 
 	if (!replay_esn || !rp)
 		return 0;
@@ -691,7 +691,7 @@ static inline int xfrm_replay_verify_len(struct xfrm_replay_state_esn *replay_es
 
 	/* Check the overall length and the internal bitmap length to avoid
 	 * potential overflow. */
-	if (nla_len(rp) < (int)ulen) {
+	if (nla_len(rp) < ulen) {
 		NL_SET_ERR_MSG(extack, "ESN attribute is too short");
 		return -EINVAL;
 	}
@@ -719,14 +719,14 @@ static int xfrm_alloc_replay_state_esn(struct xfrm_replay_state_esn **replay_esn
 				       struct nlattr *rta)
 {
 	struct xfrm_replay_state_esn *p, *pp, *up;
-	unsigned int klen, ulen;
+	size_t klen, ulen;
 
 	if (!rta)
 		return 0;
 
 	up = nla_data(rta);
 	klen = xfrm_replay_state_esn_len(up);
-	ulen = nla_len(rta) >= (int)klen ? klen : sizeof(*up);
+	ulen = nla_len(rta) >= klen ? klen : sizeof(*up);
 
 	p = kzalloc(klen, GFP_KERNEL);
 	if (!p)
-- 
2.45.2