[PATCH on top of 82d4430] Smarter MTU calculation for GlobalProtect

[dlenski at gmail.com (Daniel Lenski)]

- If we have no ESP support, ESP disabled, or no ESP keys received, then
  we're definitely using the TLS tunne and we should calculate tunnel MTU
  based on the TCP MSS, subtracting the TLS+GPST overhead (and only use the
  base/wire MTU as a crude fallback if necessary).

- If we've got ESP enabled and ESP keys, then we should calculate tunnel
  MTU based on the base/wire MTU, subtracting the IP+UDP+ESP overhead.

Signed-off-by: Daniel Lenski <dlenski at gmail.com>
---
 gpst.c | 41 ++++++++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/gpst.c b/gpst.c
index 87cfbc9..cab05ce 100644
--- a/gpst.c
+++ b/gpst.c
@@ -270,6 +270,7 @@ out:
 #define ESP_HEADER_SIZE (4 /* SPI */ + 4 /* sequence number */)
 #define ESP_FOOTER_SIZE (1 /* pad length */ + 1 /* next header */)
 #define UDP_HEADER_SIZE 8
+#define TCP_HEADER_SIZE 20 /* with no options */
 #define IPV4_HEADER_SIZE 20
 #define IPV6_HEADER_SIZE 40
 
@@ -281,9 +282,10 @@ out:
 static int calculate_mtu(struct openconnect_info *vpninfo)
 {
 	int mtu = vpninfo->reqmtu, base_mtu = vpninfo->basemtu;
+	int mss = 0;
 
 #if defined(__linux__) && defined(TCP_INFO)
-	if (!mtu || !base_mtu) {
+	if (!mtu) {
 		struct tcp_info ti;
 		socklen_t ti_size = sizeof(ti);
 
@@ -297,23 +299,19 @@ static int calculate_mtu(struct openconnect_info *vpninfo)
 				base_mtu = ti.tcpi_pmtu;
 			}
 
-			if (!base_mtu) {
-				if (ti.tcpi_rcv_mss < ti.tcpi_snd_mss)
-					base_mtu = ti.tcpi_rcv_mss - 21;
-				else
-					base_mtu = ti.tcpi_snd_mss - 21;
-			}
+			/* XXX: GlobalProtect has no mechanism to inform the server about the
+			 * desired MTU, so could just ignore the "incoming" MSS (tcpi_rcv_mss).
+			 */
+			mss = MIN(ti.tcpi_rcv_mss, ti.tcpi_snd_mss);
 		}
 	}
 #endif
 #ifdef TCP_MAXSEG
-	if (!base_mtu) {
-		int mss;
+	if (!mtu && !mss) {
 		socklen_t mss_size = sizeof(mss);
 		if (!getsockopt(vpninfo->ssl_fd, IPPROTO_TCP, TCP_MAXSEG,
 				&mss, &mss_size)) {
 			vpn_progress(vpninfo, PRG_DEBUG, _("TCP_MAXSEG %d\n"), mss);
-			base_mtu = mss - 21;
 		}
 	}
 #endif
@@ -325,7 +323,11 @@ static int calculate_mtu(struct openconnect_info *vpninfo)
 	if (base_mtu < 1280)
 		base_mtu = 1280;
 
-	if (!mtu) {
+#ifdef HAVE_ESP
+	/* If we can use the ESP tunnel (got secrets in config),
+	 * then we should pick the optimal MTU for ESP.
+	 */
+	if (!mtu && vpninfo->dtls_state == DTLS_SECRET) {
 		/* remove ESP, UDP, IP headers from base (wire) MTU */
 		mtu = ( base_mtu - UDP_HEADER_SIZE - ESP_HEADER_SIZE
 		        - 12 /* both supported algos (SHA1 and MD5) have 96-bit MAC lengths (RFC2403 and RFC2404) */
@@ -338,6 +340,23 @@ static int calculate_mtu(struct openconnect_info *vpninfo)
 		mtu -= mtu % (vpninfo->enc_key_len ? : 32);
 		/* subtract ESP footer, which is included in the payload before padding to the blocksize */
 		mtu -= ESP_FOOTER_SIZE;
+
+	} else
+#endif
+
+    /* We are definitely using the TLS tunnel (no ESP secrets)
+	 * so we should base our MTU on the TCP MSS.
+	 */
+	if (!mtu) {
+		if (mss)
+			mtu = mss - 21;
+		else {
+			mtu = base_mtu - TCP_HEADER_SIZE - 21;
+			if (vpninfo->peer_addr->sa_family == AF_INET6)
+				mtu -= IPV6_HEADER_SIZE;
+			else
+				mtu -= IPV4_HEADER_SIZE;
+		}
 	}
 	return mtu;
 }
-- 
2.7.4