[RFC 2.6.10 0/22] Add hardware assist for IPSEC crypto

@@ -250,6 +250,9 @@
 };
 #define NETDEV_BOOT_SETUP_MAX 8
 
+struct xfrm_state;
+struct xfrm_offload;
+struct xfrm_bundle_list;
 
 /*
  *	The DEVICE structure.

@@ -415,6 +418,7 @@
 #define NETIF_F_VLAN_CHALLENGED	1024	/* Device cannot handle VLAN packets */
 #define NETIF_F_TSO		2048	/* Can offload TCP/IP segmentation */
 #define NETIF_F_LLTX		4096	/* LockLess TX */
+#define NETIF_F_IPSEC		8192	/* Can offload IPSEC crypto */
 
 	/* Called after device is detached from network. */
 	void			(*uninit)(struct net_device *dev);

@@ -464,6 +468,13 @@
 						   unsigned short vid);
 	void			(*vlan_rx_kill_vid)(struct net_device *dev,
 						    unsigned short vid);
+
+	void			(*xfrm_state_add)(struct net_device *dev,
+						  struct xfrm_state *x);
+	void			(*xfrm_bundle_add)(struct net_device *dev,
+						   struct xfrm_bundle_list *bl);
+	void			(*xfrm_state_del)(struct net_device *dev,
+						  struct xfrm_offload *xol);
 
 	int			(*hard_header_parse)(struct sk_buff *skb,
 						     unsigned char *haddr);

@@ -146,6 +146,9 @@
 	/* Private data of this transformer, format is opaque,
 	 * interpreted by xfrm_type methods. */
 	void			*data;
+
+	/* Intended direction of this state, used for offloading */
+	int			dir;
 };
 
 enum {

@@ -157,6 +160,12 @@
 	XFRM_STATE_DEAD
 };
 
+enum {
+	XFRM_STATE_DIR_UNKNOWN,
+	XFRM_STATE_DIR_IN,
+	XFRM_STATE_DIR_OUT,
+};
+
 struct xfrm_type;
 struct xfrm_dst;
 struct xfrm_policy_afinfo {

@@ -194,6 +203,7 @@
 	struct xfrm_state	*(*find_acq)(u8 mode, u32 reqid, u8 proto, 
 					     xfrm_address_t *daddr, xfrm_address_t *saddr, 
 					     int create);
+	void			(*map_direction)(struct xfrm_state *xfrm);
 };
 
 extern int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo);

@@ -106,12 +106,22 @@
 	return x0;
 }
 
+static void
+__xfrm4_map_direction(struct xfrm_state *x)
+{
+	if (inet_addr_type(x->id.daddr.a4) == RTN_LOCAL)
+		x->dir = XFRM_STATE_DIR_IN;
+	else
+		x->dir = XFRM_STATE_DIR_OUT;
+}
+
 static struct xfrm_state_afinfo xfrm4_state_afinfo = {
 	.family			= AF_INET,
 	.lock			= RW_LOCK_UNLOCKED,
 	.init_tempsel		= __xfrm4_init_tempsel,
 	.state_lookup		= __xfrm4_state_lookup,
 	.find_acq		= __xfrm4_find_acq,
+	.map_direction		= __xfrm4_map_direction,
 };
 
 void __init xfrm4_state_init(void)

@@ -116,12 +116,21 @@
 	return x0;
 }
 
+static void
+__xfrm6_map_direction(struct xfrm_state *x)
+{
+	/* XXX This needs to be implemented by someone who knows
+	 * IPv6 better then I.
+	 */
+}
+
 static struct xfrm_state_afinfo xfrm6_state_afinfo = {
 	.family			= AF_INET6,
 	.lock			= RW_LOCK_UNLOCKED,
 	.init_tempsel		= __xfrm6_init_tempsel,
 	.state_lookup		= __xfrm6_state_lookup,
 	.find_acq		= __xfrm6_find_acq,
+	.map_direction		= __xfrm6_map_direction,
 };
 
 void __init xfrm6_state_init(void)

@@ -186,6 +186,7 @@
 		x->lft.soft_packet_limit = XFRM_INF;
 		x->lft.hard_byte_limit = XFRM_INF;
 		x->lft.hard_packet_limit = XFRM_INF;
+		x->dir = XFRM_STATE_DIR_UNKNOWN;
 		spin_lock_init(&x->lock);
 	}
 	return x;

@@ -404,6 +405,8 @@
 	if (unlikely(afinfo == NULL))
 		return -EAFNOSUPPORT;
 
+	afinfo->map_direction(x);
+
 	spin_lock_bh(&xfrm_state_lock);
 
 	x1 = afinfo->state_lookup(&x->id.daddr, x->id.spi, x->id.proto);

@@ -451,6 +454,8 @@
 	afinfo = xfrm_state_get_afinfo(x->props.family);
 	if (unlikely(afinfo == NULL))
 		return -EAFNOSUPPORT;
+
+	afinfo->map_direction(x);
 
 	spin_lock_bh(&xfrm_state_lock);
 	x1 = afinfo->state_lookup(&x->id.daddr, x->id.spi, x->id.proto);

@@ -833,6 +833,8 @@
 extern struct xfrm_state *xfrm_find_acq_byseq(u32 seq);
 extern void xfrm_state_delete(struct xfrm_state *x);
 extern void xfrm_state_flush(u8 proto);
+extern void xfrm_state_accel_add(struct net_device *dev);
+extern void xfrm_state_accel_flush(struct net_device *dev);
 extern int xfrm_replay_check(struct xfrm_state *x, u32 seq);
 extern void xfrm_replay_advance(struct xfrm_state *x, u32 seq);
 extern int xfrm_state_check(struct xfrm_state *x, struct sk_buff *skb);

@@ -888,6 +890,8 @@
 extern int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol);
 extern struct xfrm_policy *xfrm_sk_policy_lookup(struct sock *sk, int dir, struct flowi *fl);
 extern int xfrm_flush_bundles(void);
+extern void xfrm_accel_add(struct net_device *dev);
+extern void xfrm_accel_flush(struct net_device *dev);
 
 extern wait_queue_head_t km_waitq;
 extern void km_state_expired(struct xfrm_state *x, int hard);

@@ -63,3 +63,5 @@
 EXPORT_SYMBOL_GPL(skb_icv_walk);
 
 EXPORT_SYMBOL_GPL(xfrm_state_offload_add);
+EXPORT_SYMBOL_GPL(xfrm_accel_add);
+EXPORT_SYMBOL_GPL(xfrm_accel_flush);

@@ -1121,6 +1121,23 @@
 	return 0;
 }
 
+static int bundle_uses_dev(struct dst_entry *dst, void *dev)
+{
+	return (dst->dev == dev);
+}
+
+void xfrm_accel_add(struct net_device *dev)
+{
+	xfrm_state_accel_add(dev);
+	xfrm_prune_bundles(bundle_uses_dev, dev);
+}
+
+void xfrm_accel_flush(struct net_device *dev)
+{
+	xfrm_state_accel_flush(dev);
+	xfrm_prune_bundles(bundle_uses_dev, dev);
+}
+
 /* Well... that's _TASK_. We need to scan through transformation
  * list and figure out what mss tcp should generate in order to
  * final datagram fit to mtu. Mama mia... :-)

@@ -998,6 +998,45 @@
 		xfrm_offload_release(old);
 }
 
+static int try_new_accel(struct xfrm_state *x, int unused, void *data)
+{
+	struct net_device *dev = data;
+
+	if (x->dir == XFRM_STATE_DIR_IN)
+		dev->xfrm_state_add(dev, x);
+	return 0;
+}
+
+static int remove_stale_accel(struct xfrm_state *x, int unused, void *dev)
+{
+	struct xfrm_offload *xol, *entry = NULL;
+
+	spin_lock(&x->lock);
+	list_for_each_entry(xol, &x->offloads, bydev) {
+		if (xol->dev == dev) {
+			list_del(&xol->bydev);
+			entry = xol;
+			break;
+		}
+	}
+	spin_unlock(&x->lock);
+
+	if (entry)
+		xfrm_offload_release(entry);
+
+	return 0;
+}
+
+void xfrm_state_accel_add(struct net_device *dev)
+{
+	xfrm_state_walk(IPSEC_PROTO_ANY, try_new_accel, dev);
+}
+
+void xfrm_state_accel_flush(struct net_device *dev)
+{
+	xfrm_state_walk(IPSEC_PROTO_ANY, remove_stale_accel, dev);
+}
+
 void __init xfrm_state_init(void)
 {
 	int i;

@@ -730,7 +730,7 @@
 		kfree(xbl);
 }
 
-static int stale_bundle(struct dst_entry *dst);
+static int stale_bundle(struct dst_entry *dst, void *unused);
 
 /* Main function: finds/creates a bundle for given flow.
  *

@@ -841,7 +841,7 @@
 		}
 
 		write_lock_bh(&policy->lock);
-		if (unlikely(policy->dead || stale_bundle(dst))) {
+		if (unlikely(policy->dead || stale_bundle(dst, NULL))) {
 			/* Wow! While we worked on resolving, this
 			 * policy has gone. Retry. It is not paranoia,
 			 * we just cannot enlist new bundle to dead object.

@@ -1022,14 +1022,14 @@
 
 static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
 {
-	if (!stale_bundle(dst))
+	if (!stale_bundle(dst, NULL))
 		return dst;
 
 	dst_release(dst);
 	return NULL;
 }
 
-static int stale_bundle(struct dst_entry *dst)
+static int stale_bundle(struct dst_entry *dst, void *unused)
 {
 	struct dst_entry *child = dst;
 

@@ -1072,7 +1072,8 @@
 	return dst;
 }
 
-static void xfrm_prune_bundles(int (*func)(struct dst_entry *))
+static void xfrm_prune_bundles(int (*func)(struct dst_entry *, void *),
+					void *data)
 {
 	int i;
 	struct xfrm_policy *pol;

@@ -1084,7 +1085,7 @@
 			write_lock(&pol->lock);
 			dstp = &pol->bundles;
 			while ((dst=*dstp) != NULL) {
-				if (func(dst)) {
+				if (func(dst, data)) {
 					*dstp = dst->next;
 					dst->next = gc_list;
 					gc_list = dst;

@@ -1104,19 +1105,19 @@
 	}
 }
 
-static int unused_bundle(struct dst_entry *dst)
+static int unused_bundle(struct dst_entry *dst, void *unused)
 {
 	return !atomic_read(&dst->__refcnt);
 }
 
 static void __xfrm_garbage_collect(void)
 {
-	xfrm_prune_bundles(unused_bundle);
+	xfrm_prune_bundles(unused_bundle, NULL);
 }
 
 int xfrm_flush_bundles(void)
 {
-	xfrm_prune_bundles(stale_bundle);
+	xfrm_prune_bundles(stale_bundle, NULL);
 	return 0;
 }
 

@@ -398,6 +398,26 @@
 	spin_unlock_bh(&xfrm_state_lock);
 }
 
+static void xfrm_state_inbound_accel(struct xfrm_state *x)
+{
+	/* Only called for an inbound xfrm_state. Since it could
+	 * possibly arrive on any interface, try to offload it
+	 * on all devices that are capable.
+	 */
+	struct net_device *dev;
+
+	rtnl_lock();
+	read_lock(&dev_base_lock);
+	dev = dev_base;
+	while (dev) {
+		if (netif_running(dev) && (dev->features & NETIF_F_IPSEC))
+			dev->xfrm_state_add(dev, x);
+		dev = dev->next;
+	}
+	read_unlock(&dev_base_lock);
+	rtnl_unlock();
+}
+
 static struct xfrm_state *__xfrm_find_acq_byseq(u32 seq);
 
 int xfrm_state_add(struct xfrm_state *x)

@@ -444,6 +464,9 @@
 	spin_unlock_bh(&xfrm_state_lock);
 	xfrm_state_put_afinfo(afinfo);
 
+	if (!err && x->dir == XFRM_STATE_DIR_IN)
+		xfrm_state_inbound_accel(x);
+
 	if (x1) {
 		xfrm_state_delete(x1);
 		xfrm_state_put(x1);

@@ -455,7 +478,7 @@
 int xfrm_state_update(struct xfrm_state *x)
 {
 	struct xfrm_state_afinfo *afinfo;
-	struct xfrm_state *x1;
+	struct xfrm_state *x1, *accel = NULL;
 	int err;
 
 	afinfo = xfrm_state_get_afinfo(x->props.family);

@@ -479,6 +502,7 @@
 
 	if (x1->km.state == XFRM_STATE_ACQ) {
 		__xfrm_state_insert(x);
+		accel = x;
 		x = NULL;
 	}
 	err = 0;

@@ -489,6 +513,9 @@
 
 	if (err)
 		return err;
+
+	if (accel && accel->dir == XFRM_STATE_DIR_IN)
+		xfrm_state_inbound_accel(accel);
 
 	if (!x) {
 		xfrm_state_delete(x1);

@@ -65,6 +65,7 @@
 	struct neighbour	*neighbour;
 	struct hh_cache		*hh;
 	struct xfrm_state	*xfrm;
+	struct xfrm_offload	*xfrm_offload;
 
 	int			(*input)(struct sk_buff*);
 	int			(*output)(struct sk_buff*);

@@ -178,6 +178,12 @@
 	atomic_t		refcnt;
 };
 
+struct xfrm_bundle_list
+{
+	struct list_head	node;
+	struct dst_entry *	dst;
+};
+
 struct xfrm_type;
 struct xfrm_dst;
 struct xfrm_policy_afinfo {

@@ -705,6 +705,31 @@
 	};
 }
 
+static void xfrm_accel_bundle(struct dst_entry *dst)
+{
+	struct xfrm_bundle_list bundle, *xbl, *tmp;
+	struct net_device *dev = dst->dev;
+	INIT_LIST_HEAD(&bundle.node);
+
+	if (dev && netif_running(dev) && (dev->features & NETIF_F_IPSEC)) {
+		while (dst) {
+			xbl = kmalloc(sizeof(*xbl), GFP_ATOMIC);
+			if (!xbl)
+				goto out;
+
+			xbl->dst = dst;
+			list_add_tail(&xbl->node, &bundle.node);
+			dst = dst->child;
+		}
+
+		dev->xfrm_bundle_add(dev, &bundle);
+	}
+
+out:
+	list_for_each_entry_safe(xbl, tmp, &bundle.node, node)
+		kfree(xbl);
+}
+
 static int stale_bundle(struct dst_entry *dst);
 
 /* Main function: finds/creates a bundle for given flow.

@@ -833,6 +858,7 @@
 		policy->bundles = dst;
 		dst_hold(dst);
 		write_unlock_bh(&policy->lock);
+		xfrm_accel_bundle(dst);
 	}
 	*dst_p = dst;
 	dst_release(dst_orig);

@@ -1023,8 +1049,10 @@
 {
 	if (!dst->xfrm)
 		return;
+	xfrm_offload_release(dst->xfrm_offload);
 	xfrm_state_put(dst->xfrm);
 	dst->xfrm = NULL;
+	dst->xfrm_offload = NULL;
 }
 
 static void xfrm_link_failure(struct sk_buff *skb)

@@ -83,31 +83,41 @@
 	ah->spi = x->id.spi;
 	ah->seq_no = htonl(x->replay.oseq + 1);
 
-	iph->tos = top_iph->tos;
-	iph->ttl = top_iph->ttl;
-	iph->frag_off = top_iph->frag_off;
-
-	if (top_iph->ihl != 5) {
-		iph->daddr = top_iph->daddr;
-		memcpy(iph+1, top_iph+1, top_iph->ihl*4 - sizeof(struct iphdr));
-		err = ip_clear_mutable_options(top_iph, &top_iph->daddr);
-		if (err)
+	if (dst->xfrm_offload) {
+		err = -ENOMEM;
+		xfrm_offload_hold(dst->xfrm_offload);
+		if (skb_push_xfrm_offload(skb, dst->xfrm_offload)) {
+			xfrm_offload_release(dst->xfrm_offload);
 			goto error;
-	}
+		}
+	} else {
+		/* Not offloaded, manually calculate the auth hash */
+		iph->tos = top_iph->tos;
+		iph->ttl = top_iph->ttl;
+		iph->frag_off = top_iph->frag_off;
+
+		if (top_iph->ihl != 5) {
+			iph->daddr = top_iph->daddr;
+			memcpy(iph+1, top_iph+1, top_iph->ihl*4 - sizeof(struct iphdr));
+			err = ip_clear_mutable_options(top_iph, &top_iph->daddr);
+			if (err)
+				goto error;
+		}
 
-	top_iph->tos = 0;
-	top_iph->frag_off = 0;
-	top_iph->ttl = 0;
-	top_iph->check = 0;
+		top_iph->tos = 0;
+		top_iph->frag_off = 0;
+		top_iph->ttl = 0;
+		top_iph->check = 0;
 
-	ahp->icv(ahp, skb, ah->auth_data);
+		ahp->icv(ahp, skb, ah->auth_data);
 
-	top_iph->tos = iph->tos;
-	top_iph->ttl = iph->ttl;
-	top_iph->frag_off = iph->frag_off;
-	if (top_iph->ihl != 5) {
-		top_iph->daddr = iph->daddr;
-		memcpy(top_iph+1, iph+1, top_iph->ihl*4 - sizeof(struct iphdr));
+		top_iph->tos = iph->tos;
+		top_iph->ttl = iph->ttl;
+		top_iph->frag_off = iph->frag_off;
+		if (top_iph->ihl != 5) {
+			top_iph->daddr = iph->daddr;
+			memcpy(top_iph+1, iph+1, top_iph->ihl*4 - sizeof(struct iphdr));
+		}
 	}
 
 	/* Delay incrementing the replay sequence until we know we're going

@@ -98,33 +98,47 @@
 	esph->spi = x->id.spi;
 	esph->seq_no = htonl(++x->replay.oseq);
 
-	if (esp->conf.ivlen)
-		crypto_cipher_set_iv(tfm, esp->conf.ivec, crypto_tfm_alg_ivsize(tfm));
+	if (dst->xfrm_offload) {
+		xfrm_offload_hold(dst->xfrm_offload);
+		if (skb_push_xfrm_offload(skb, dst->xfrm_offload)) {
+			xfrm_offload_release(dst->xfrm_offload);
+			goto error;
+		}
+
+		if (esp->conf.ivlen)
+			get_random_bytes(esph->enc_data, esp->conf.ivlen);
+	} else {
+		if (esp->conf.ivlen)
+			crypto_cipher_set_iv(tfm, esp->conf.ivec, crypto_tfm_alg_ivsize(tfm));
+
+		do {
+			struct scatterlist *sg = &esp->sgbuf[0];
 
-	do {
-		struct scatterlist *sg = &esp->sgbuf[0];
+			if (unlikely(nfrags > ESP_NUM_FAST_SG)) {
+				sg = kmalloc(sizeof(struct scatterlist)*nfrags, GFP_ATOMIC);
+				if (!sg)
+					goto error;
+			}
+			skb_to_sgvec(skb, sg, esph->enc_data+esp->conf.ivlen-skb->data, clen);
+			crypto_cipher_encrypt(tfm, sg, sg, clen);
+			if (unlikely(sg != &esp->sgbuf[0]))
+				kfree(sg);
+		} while (0);
 
-		if (unlikely(nfrags > ESP_NUM_FAST_SG)) {
-			sg = kmalloc(sizeof(struct scatterlist)*nfrags, GFP_ATOMIC);
-			if (!sg)
-				goto error;
+		if (esp->conf.ivlen) {
+			memcpy(esph->enc_data, esp->conf.ivec, crypto_tfm_alg_ivsize(tfm));
+			crypto_cipher_get_iv(tfm, esp->conf.ivec, crypto_tfm_alg_ivsize(tfm));
+		}
+
+		if (esp->auth.icv_full_len) {
+			esp->auth.icv(esp, skb, (u8*)esph-skb->data,
+		              	sizeof(struct ip_esp_hdr) + esp->conf.ivlen+clen, trailer->tail);
 		}
-		skb_to_sgvec(skb, sg, esph->enc_data+esp->conf.ivlen-skb->data, clen);
-		crypto_cipher_encrypt(tfm, sg, sg, clen);
-		if (unlikely(sg != &esp->sgbuf[0]))
-			kfree(sg);
-	} while (0);
-
-	if (esp->conf.ivlen) {
-		memcpy(esph->enc_data, esp->conf.ivec, crypto_tfm_alg_ivsize(tfm));
-		crypto_cipher_get_iv(tfm, esp->conf.ivec, crypto_tfm_alg_ivsize(tfm));
 	}
 
-	if (esp->auth.icv_full_len) {
-		esp->auth.icv(esp, skb, (u8*)esph-skb->data,
-		              sizeof(struct ip_esp_hdr) + esp->conf.ivlen+clen, trailer->tail);
+	/* Need to account for auth data, offloading or not... */
+	if (esp->auth.icv_full_len)
 		pskb_put(skb, trailer, alen);
-	}
 
 	ip_send_check(top_iph);
 

@@ -146,6 +146,14 @@
 	skb_frag_t	frags[MAX_SKB_FRAGS];
 };
 
+/* XXX UGH. We cannot include <net/xfrm.h> in this file without some
+ * header file surgery, so define our own max xfrm depth. This should
+ * be kept >= XFRM_MAX_DEPTH until we fix the includes, and it can
+ * go away.
+ */
+#define SKB_XFRM_MAX_DEPTH 4
+struct xfrm_offload;
+
 /** 
  *	struct sk_buff - socket buffer
  *	@next: Next buffer in list

@@ -187,6 +195,8 @@
  *	@nf_bridge: Saved data about a bridged frame - see br_netfilter.c
  *      @private: Data which is private to the HIPPI implementation
  *	@tc_index: Traffic control index
+ *	@xfrm_offload: Tx offload info, Rx offload results
+ *	@xfrm_offload_idx: The number of cookies/results stored currently
  */
 
 struct sk_buff {

@@ -272,6 +282,8 @@
 
 #endif
 
+	int			xfrm_offload_idx;
+	struct xfrm_offload *	xfrm_offload[SKB_XFRM_MAX_DEPTH];
 
 	/* These elements must be at the end, see alloc_skb() for details.  */
 	unsigned int		truesize;

@@ -1178,6 +1190,49 @@
 #else /* CONFIG_NETFILTER */
 static inline void nf_reset(struct sk_buff *skb) {}
 #endif /* CONFIG_NETFILTER */
+
+static inline int skb_push_xfrm_offload(struct sk_buff *skb,
+					struct xfrm_offload *xol)
+{
+	if (likely(skb->xfrm_offload_idx < SKB_XFRM_MAX_DEPTH)) {
+		skb->xfrm_offload[skb->xfrm_offload_idx++] = xol;
+		return 0;
+	}
+
+	return -ENOMEM;
+}
+
+static inline struct xfrm_offload *
+skb_get_xfrm_offload(const struct sk_buff *skb, int idx)
+{
+	if (likely(idx < skb->xfrm_offload_idx))
+		return skb->xfrm_offload[idx];
+	else
+		return NULL;
+}
+
+static inline int skb_has_xfrm_offload(const struct sk_buff *skb)
+{
+	return !!skb_get_xfrm_offload(skb, 0);
+}
+
+static inline int skb_put_xfrm_result(struct sk_buff *skb, int result, int idx)
+{
+	if (likely(idx < SKB_XFRM_MAX_DEPTH)) {
+		skb->xfrm_offload[idx] = (struct xfrm_offload *) result;
+		return 0;
+	}
+	return -ENOMEM;
+}
+
+static inline int skb_pop_xfrm_result(struct sk_buff *skb)
+{
+	/* XXX XFRM_OFFLOAD_NONE == 0, but cannot include <net/xfrm.h> */
+	int res = 0;
+	if (likely(skb->xfrm_offload_idx < SKB_XFRM_MAX_DEPTH))
+		res = (int) skb->xfrm_offload[skb->xfrm_offload_idx++];
+	return res;
+}
 
 #endif	/* __KERNEL__ */
 #endif	/* _LINUX_SKBUFF_H */

@@ -171,6 +171,15 @@
 	XFRM_STATE_DIR_OUT,
 };
 
+enum {
+	XFRM_OFFLOAD_NONE = 0,
+	XFRM_OFFLOAD_CONF = 1,
+	XFRM_OFFLOAD_AUTH = 2,
+	XFRM_OFFLOAD_AUTH_OK = 4,
+	XFRM_OFFLOAD_AUTH_FAIL = 8,
+	XFRM_OFFLOAD_FIELD = 0x0f
+};
+
 struct xfrm_offload
 {
 	struct list_head	bydev;

@@ -230,6 +230,14 @@
 
 	dst_release(skb->dst);
 #ifdef CONFIG_XFRM
+	{
+		int i = 0;
+		struct xfrm_offload *xol;
+		while ((xol = skb_get_xfrm_offload(skb, i++)) != NULL) {
+			if ((unsigned long) xol > XFRM_OFFLOAD_FIELD)
+				xfrm_offload_release(xol);
+		}
+	}
 	secpath_put(skb->sp);
 #endif
 	if(skb->destructor) {

@@ -334,6 +342,17 @@
 #endif
 
 #endif
+	C(xfrm_offload_idx);
+	memcpy(n->xfrm_offload, skb->xfrm_offload,
+			sizeof(struct xfrm_offload *) * SKB_XFRM_MAX_DEPTH);
+	{
+		int i = 0;
+		struct xfrm_offload *xol;
+		while ((xol = skb_get_xfrm_offload(skb, i++)) != NULL) {
+			if ((unsigned long) xol > XFRM_OFFLOAD_FIELD)
+				xfrm_offload_hold(xol);
+		}
+	}
 	C(truesize);
 	atomic_set(&n->users, 1);
 	C(head);

@@ -396,6 +415,18 @@
 	atomic_set(&new->users, 1);
 	skb_shinfo(new)->tso_size = skb_shinfo(old)->tso_size;
 	skb_shinfo(new)->tso_segs = skb_shinfo(old)->tso_segs;
+
+	new->xfrm_offload_idx = old->xfrm_offload_idx;
+	memcpy(new->xfrm_offload, old->xfrm_offload,
+			sizeof(struct xfrm_offload *) * SKB_XFRM_MAX_DEPTH);
+	{
+		int i = 0;
+		struct xfrm_offload *xol;
+		while ((xol = skb_get_xfrm_offload(old, i++)) != NULL) {
+			if ((unsigned long) xol > XFRM_OFFLOAD_FIELD)
+				xfrm_offload_hold(xol);
+		}
+	}
 }
 
 /**

@@ -81,6 +81,8 @@
       metrics. Plus, it will be made via sk->sk_dst_cache. Solved.
  */
 
+struct xfrm_offload;
+
 /* Full description of state of transformer. */
 struct xfrm_state
 {

@@ -149,6 +151,9 @@
 
 	/* Intended direction of this state, used for offloading */
 	int			dir;
+
+	/* List of offload cookies, per device */
+	struct list_head	offloads;
 };
 
 enum {

@@ -166,6 +171,13 @@
 	XFRM_STATE_DIR_OUT,
 };
 
+struct xfrm_offload
+{
+	struct list_head	bydev;
+	struct net_device *	dev;
+	atomic_t		refcnt;
+};
+
 struct xfrm_type;
 struct xfrm_dst;
 struct xfrm_policy_afinfo {

@@ -911,5 +923,72 @@
 				     (struct in6_addr *)b);
 	}
 }
+
+#define XFRM_OFFLOAD_ALIGN		8
+#define XFRM_OFFLOAD_ALIGN_CONST	(XFRM_OFFLOAD_ALIGN - 1)
+
+static inline struct xfrm_offload *
+xfrm_offload_alloc(int sizeof_priv, struct net_device *dev)
+{
+	struct xfrm_offload *xol;
+	int alloc_size;
+	
+	alloc_size = (sizeof(*xol) + XFRM_OFFLOAD_ALIGN_CONST)
+				& ~XFRM_OFFLOAD_ALIGN_CONST;
+	alloc_size += sizeof_priv;
+	xol = kmalloc(alloc_size, GFP_ATOMIC);
+	if (xol) {
+		memset(xol, 0, alloc_size);
+		INIT_LIST_HEAD(&xol->bydev);
+		atomic_set(&xol->refcnt, 1);
+		xol->dev = dev;
+	}
+
+	return xol;
+}
+
+static inline void *xfrm_offload_priv(struct xfrm_offload *xol)
+{
+	return (char *)xol + ((sizeof(*xol) + XFRM_OFFLOAD_ALIGN_CONST)
+					& ~XFRM_OFFLOAD_ALIGN_CONST);
+}
+
+static inline void xfrm_offload_hold(struct xfrm_offload *xol)
+{
+	atomic_inc(&xol->refcnt);
+}
+
+static inline void xfrm_offload_release(struct xfrm_offload *xol)
+{
+	if (xol) {
+		WARN_ON(atomic_read(&xol->refcnt) < 1);
+		if (atomic_dec_and_test(&xol->refcnt)) {
+			xol->dev->xfrm_state_del(xol->dev, xol);
+			dev_put(xol->dev);
+			kfree(xol);
+		}
+	}
+}
+
+static inline struct xfrm_offload *xfrm_offload_get(struct xfrm_state *x,
+                                               struct net_device *dev)
+{
+	struct xfrm_offload *xol, *ret = NULL;
+
+	spin_lock(&x->lock);
+	list_for_each_entry(xol, &x->offloads, bydev) {
+		if (xol->dev == dev) {
+			xfrm_offload_hold(xol);
+			ret = xol;
+			break;
+		}
+	}
+	spin_unlock(&x->lock);
+
+	return ret;
+}
+
+extern void xfrm_state_offload_add(struct xfrm_state *x,
+					struct xfrm_offload *xol);
 
 #endif	/* _NET_XFRM_H */

@@ -61,3 +61,5 @@
 EXPORT_SYMBOL_GPL(xfrm_ealg_get_byname);
 EXPORT_SYMBOL_GPL(xfrm_calg_get_byname);
 EXPORT_SYMBOL_GPL(skb_icv_walk);
+
+EXPORT_SYMBOL_GPL(xfrm_state_offload_add);

@@ -53,6 +53,12 @@
 
 static void xfrm_state_gc_destroy(struct xfrm_state *x)
 {
+	if (!list_empty(&x->offloads)) {
+		struct xfrm_offload *xol, *next;
+
+		list_for_each_entry_safe(xol, next, &x->offloads, bydev)
+			xfrm_offload_release(xol);
+	}
 	if (del_timer(&x->timer))
 		BUG();
 	if (x->aalg)

@@ -178,6 +184,7 @@
 		atomic_set(&x->tunnel_users, 0);
 		INIT_LIST_HEAD(&x->bydst);
 		INIT_LIST_HEAD(&x->byspi);
+		INIT_LIST_HEAD(&x->offloads);
 		init_timer(&x->timer);
 		x->timer.function = xfrm_timer_handler;
 		x->timer.data	  = (unsigned long)x;

@@ -941,6 +948,27 @@
 		xfrm_state_put(t);
 		x->tunnel = NULL;
 	}
+}
+
+void xfrm_state_offload_add(struct xfrm_state *x, struct xfrm_offload *xol)
+{
+	struct xfrm_offload *entry, *old = NULL;
+
+	spin_lock(&x->lock);
+	list_for_each_entry(entry, &x->offloads, bydev) {
+		if (entry->dev == xol->dev) {
+			list_del(&entry->bydev);
+			old = entry;
+			break;
+		}
+	}
+
+	dev_hold(xol->dev);
+	list_add_tail(&xol->bydev, &x->offloads);
+	spin_unlock(&x->lock);
+
+	if(old)
+		xfrm_offload_release(old);
 }
 
 void __init xfrm_state_init(void)

@@ -131,6 +131,7 @@
 #include <asm/checksum.h>
 #include <linux/version.h>
 #include <linux/dma-mapping.h>
+#include <net/xfrm.h>
 
 #include "typhoon.h"
 #include "typhoon-firmware.h"

@@ -1681,6 +1682,43 @@
 	return 0;
 }
 
+static inline void
+typhoon_ipsec_rx(struct sk_buff *skb, u16 results)
+{
+#define CHECK_OFFLOAD(good, bad) \
+	do { if(results & (good|bad)) { \
+		unsigned int tmp = XFRM_OFFLOAD_CONF | XFRM_OFFLOAD_AUTH; \
+		tmp |= (results & good) ?  XFRM_OFFLOAD_AUTH_OK : \
+					   XFRM_OFFLOAD_AUTH_FAIL; \
+		if(skb_put_xfrm_result(skb, tmp, i)) \
+				return; \
+		i++; \
+	} } while(0)
+
+	/* We have no way to determine what the order of the SAs were on
+	 * the wire, just the 1st AH seen, the 1st ESP seen, etc.
+	 *
+	 * We just walk the stack, and pretend that AH SAs get decypted
+	 * so that if we get the order wrong, the worst case scenerio is
+	 * that we indicate the failure on the wrong SA, since we'll need
+	 * to match all SAs against the policy.
+	 *
+	 * We get a "ESP good" indication for null auth hash on ESP.
+	 */
+	/* XXX think more about security indications -- can I craft a
+	 * packet to do bad things -- maybe a NULL auth ESP packet,
+	 * and a failed AH packet?
+	 */
+	int i = 0;
+
+	CHECK_OFFLOAD(TYPHOON_RX_AH1_GOOD, TYPHOON_RX_AH1_FAIL);
+	CHECK_OFFLOAD(TYPHOON_RX_ESP1_GOOD, TYPHOON_RX_ESP1_FAIL);
+	CHECK_OFFLOAD(TYPHOON_RX_AH2_GOOD, TYPHOON_RX_AH2_FAIL);
+	CHECK_OFFLOAD(TYPHOON_RX_ESP2_GOOD, TYPHOON_RX_ESP2_FAIL);
+
+#undef CHECK_OFFLOAD
+}
+
 static int
 typhoon_rx(struct typhoon *tp, struct basic_ring *rxRing, volatile u32 * ready,
 	   volatile u32 * cleared, int budget)

@@ -1745,6 +1783,10 @@
 			new_skb->ip_summed = CHECKSUM_UNNECESSARY;
 		} else
 			new_skb->ip_summed = CHECKSUM_NONE;
+
+		if((rx->rxStatus & TYPHOON_RX_IPSEC) &&
+				!(rx->rxStatus & TYPHOON_RX_IP_FRAG))
+			typhoon_ipsec_rx(new_skb, rx->ipsecResults);
 
 		spin_lock(&tp->state_lock);
 		if(tp->vlgrp != NULL && rx->rxStatus & TYPHOON_RX_VLAN)

@@ -210,7 +210,10 @@
  * flags:	descriptor type
  * numDesc:	must be 1
  * ipsecFlags:	bit 0: 0 -- generate IV, 1 -- use supplied IV
- * sa1, sa2:	Security Association IDs for this packet
+ * sa[0]:	the inner AH header offload cookie (or ESP if no AH)
+ * sa[1]:	the inner ESP header offload cookie (or 0 if no AH)
+ * sa[2]:	the outer AH header offload cookie (or ESP if no AH)
+ * sa[3]:	the outer ESP header offload cookie (or 0 if no AH)
  * reserved:	set to 0
  */
 struct ipsec_desc {

@@ -219,8 +222,7 @@
 	u16 ipsecFlags;
 #define TYPHOON_IPSEC_GEN_IV	__constant_cpu_to_le16(0x0000)
 #define TYPHOON_IPSEC_USE_IV	__constant_cpu_to_le16(0x0001)
-	u32 sa1;
-	u32 sa2;
+	u16 sa[4];
 	u32 reserved;
 } __attribute__ ((packed));
 

@@ -268,14 +270,14 @@
 #define TYPHOON_RX_FILTER_MASK		__constant_cpu_to_le16(0x7fff)
 #define TYPHOON_RX_FILTERED		__constant_cpu_to_le16(0x8000)
 	u16 ipsecResults;
-#define TYPHOON_RX_OUTER_AH_GOOD	__constant_cpu_to_le16(0x0001)
-#define TYPHOON_RX_OUTER_ESP_GOOD	__constant_cpu_to_le16(0x0002)
-#define TYPHOON_RX_INNER_AH_GOOD	__constant_cpu_to_le16(0x0004)
-#define TYPHOON_RX_INNER_ESP_GOOD	__constant_cpu_to_le16(0x0008)
-#define TYPHOON_RX_OUTER_AH_FAIL	__constant_cpu_to_le16(0x0010)
-#define TYPHOON_RX_OUTER_ESP_FAIL	__constant_cpu_to_le16(0x0020)
-#define TYPHOON_RX_INNER_AH_FAIL	__constant_cpu_to_le16(0x0040)
-#define TYPHOON_RX_INNER_ESP_FAIL	__constant_cpu_to_le16(0x0080)
+#define TYPHOON_RX_AH1_GOOD		__constant_cpu_to_le16(0x0001)
+#define TYPHOON_RX_ESP1_GOOD		__constant_cpu_to_le16(0x0002)
+#define TYPHOON_RX_AH2_GOOD		__constant_cpu_to_le16(0x0004)
+#define TYPHOON_RX_ESP2_GOOD		__constant_cpu_to_le16(0x0008)
+#define TYPHOON_RX_AH1_FAIL		__constant_cpu_to_le16(0x0010)
+#define TYPHOON_RX_ESP1_FAIL		__constant_cpu_to_le16(0x0020)
+#define TYPHOON_RX_AH2_FAIL		__constant_cpu_to_le16(0x0040)
+#define TYPHOON_RX_ESP2_FAIL		__constant_cpu_to_le16(0x0080)
 #define TYPHOON_RX_UNKNOWN_SA		__constant_cpu_to_le16(0x0100)
 #define TYPHOON_RX_ESP_FORMAT_ERR	__constant_cpu_to_le16(0x0200)
 	u32 vlanTag;

@@ -260,6 +260,8 @@
 int ethtool_op_set_sg(struct net_device *dev, u32 data);
 u32 ethtool_op_get_tso(struct net_device *dev);
 int ethtool_op_set_tso(struct net_device *dev, u32 data);
+u32 ethtool_op_get_ipsec(struct net_device *dev);
+int ethtool_op_set_ipsec(struct net_device *dev, u32 data);
 
 /**
  * &ethtool_ops - Alter and report network device settings

@@ -293,6 +295,8 @@
  * get_strings: Return a set of strings that describe the requested objects 
  * phys_id: Identify the device
  * get_stats: Return statistics about the device
+ * get_ipsec: Report whether IPSEC crypto offload is enabled
+ * set_ipsec: Turn IPSEC crypto offload on or off
  *
  * Description:
  *

@@ -345,6 +349,8 @@
 	int	(*set_sg)(struct net_device *, u32);
 	u32	(*get_tso)(struct net_device *);
 	int	(*set_tso)(struct net_device *, u32);
+	u32	(*get_ipsec)(struct net_device *);
+	int	(*set_ipsec)(struct net_device *, u32);
 	int	(*self_test_count)(struct net_device *);
 	void	(*self_test)(struct net_device *, struct ethtool_test *, u64 *);
 	void	(*get_strings)(struct net_device *, u32 stringset, u8 *);

@@ -388,6 +394,8 @@
 #define ETHTOOL_GSTATS		0x0000001d /* get NIC-specific statistics */
 #define ETHTOOL_GTSO		0x0000001e /* Get TSO enable (ethtool_value) */
 #define ETHTOOL_STSO		0x0000001f /* Set TSO enable (ethtool_value) */
+#define ETHTOOL_GIPSEC		0x00000020 /* Get IPSEC enable (ethtool_value) */
+#define ETHTOOL_SIPSEC		0x00000021 /* Set IPSEC enable (ethtool_value) */
 
 /* compatibility with older code */
 #define SPARC_ETH_GSET		ETHTOOL_GSET

@@ -14,6 +14,7 @@
 #include <linux/errno.h>
 #include <linux/ethtool.h>
 #include <linux/netdevice.h>
+#include <net/xfrm.h>
 #include <asm/uaccess.h>
 
 /* 

@@ -72,6 +73,24 @@
 	return 0;
 }
 
+u32 ethtool_op_get_ipsec(struct net_device *dev)
+{
+	return (dev->features & NETIF_F_IPSEC) != 0;
+}
+
+int ethtool_op_set_ipsec(struct net_device *dev, u32 data)
+{
+	if (data) {
+		dev->features |= NETIF_F_IPSEC;
+		xfrm_accel_add(dev);
+	} else {
+		dev->features &= ~NETIF_F_IPSEC;
+		xfrm_accel_flush(dev);
+	}
+
+	return 0;
+}
+
 /* Handlers for each ethtool command */
 
 static int ethtool_get_settings(struct net_device *dev, void __user *useraddr)

@@ -548,6 +567,33 @@
 	return dev->ethtool_ops->set_tso(dev, edata.data);
 }
 
+static int ethtool_get_ipsec(struct net_device *dev, char __user *useraddr)
+{
+	struct ethtool_value edata = { ETHTOOL_GIPSEC };
+
+	if (!dev->ethtool_ops->get_ipsec)
+		return -EOPNOTSUPP;
+
+	edata.data = dev->ethtool_ops->get_ipsec(dev);
+
+	if (copy_to_user(useraddr, &edata, sizeof(edata)))
+		return -EFAULT;
+	return 0;
+}
+
+static int ethtool_set_ipsec(struct net_device *dev, char __user *useraddr)
+{
+	struct ethtool_value edata;
+
+	if (!dev->ethtool_ops->set_ipsec)
+		return -EOPNOTSUPP;
+
+	if (copy_from_user(&edata, useraddr, sizeof(edata)))
+		return -EFAULT;
+
+	return dev->ethtool_ops->set_ipsec(dev, edata.data);
+}
+
 static int ethtool_self_test(struct net_device *dev, char __user *useraddr)
 {
 	struct ethtool_test test;

@@ -783,6 +829,12 @@
 	case ETHTOOL_STSO:
 		rc = ethtool_set_tso(dev, useraddr);
 		break;
+	case ETHTOOL_GIPSEC:
+		rc = ethtool_get_ipsec(dev, useraddr);
+		break;
+	case ETHTOOL_SIPSEC:
+		rc = ethtool_set_ipsec(dev, useraddr);
+		break;
 	case ETHTOOL_TEST:
 		rc = ethtool_self_test(dev, useraddr);
 		break;

@@ -813,7 +865,9 @@
 EXPORT_SYMBOL(ethtool_op_get_link);
 EXPORT_SYMBOL(ethtool_op_get_sg);
 EXPORT_SYMBOL(ethtool_op_get_tso);
+EXPORT_SYMBOL(ethtool_op_get_ipsec);
 EXPORT_SYMBOL(ethtool_op_get_tx_csum);
 EXPORT_SYMBOL(ethtool_op_set_sg);
 EXPORT_SYMBOL(ethtool_op_set_tso);
+EXPORT_SYMBOL(ethtool_op_set_ipsec);
 EXPORT_SYMBOL(ethtool_op_set_tx_csum);

@@ -138,6 +138,7 @@
 	struct iphdr *iph;
 	struct ip_auth_hdr *ah;
 	struct ah_data *ahp;
+	int offload;
 	char work_buf[60];
 
 	if (!pskb_may_pull(skb, sizeof(struct ip_auth_hdr)))

@@ -164,6 +165,7 @@
 
 	ah = (struct ip_auth_hdr*)skb->data;
 	iph = skb->nh.iph;
+	offload = skb_pop_xfrm_result(skb);
 
 	memcpy(work_buf, iph, iph->ihl*4);
 

@@ -181,10 +183,17 @@
 		
 		memcpy(auth_data, ah->auth_data, ahp->icv_trunc_len);
 		skb_push(skb, skb->data - skb->nh.raw);
-		ahp->icv(ahp, skb, ah->auth_data);
-		if (memcmp(ah->auth_data, auth_data, ahp->icv_trunc_len)) {
-			x->stats.integrity_failed++;
-			goto out;
+		if (offload & XFRM_OFFLOAD_AUTH) {
+			if (unlikely(offload & XFRM_OFFLOAD_AUTH_FAIL)) {
+				x->stats.integrity_failed++;
+				goto out;
+			}
+		} else {
+			ahp->icv(ahp, skb, ah->auth_data);
+			if (memcmp(ah->auth_data, auth_data, ahp->icv_trunc_len)) {
+				x->stats.integrity_failed++;
+				goto out;
+			}
 		}
 	}
 	((struct iphdr*)work_buf)->protocol = ah->nexthdr;

@@ -164,6 +164,7 @@
 	int elen = skb->len - sizeof(struct ip_esp_hdr) - esp->conf.ivlen - alen;
 	int nfrags;
 	int encap_len = 0;
+	int offload;
 
 	if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr)))
 		goto out;

@@ -171,22 +172,32 @@
 	if (elen <= 0 || (elen & (blksize-1)))
 		goto out;
 
+	offload = skb_pop_xfrm_result(skb);
+
 	/* If integrity check is required, do this. */
 	if (esp->auth.icv_full_len) {
-		u8 sum[esp->auth.icv_full_len];
-		u8 sum1[alen];
+		if (unlikely(offload & XFRM_OFFLOAD_AUTH_FAIL)) {
+			x->stats.integrity_failed++;
+			goto out;
+		}
+
+		if (!(offload & XFRM_OFFLOAD_AUTH)) {
+			u8 sum[esp->auth.icv_full_len];
+			u8 sum1[alen];
 		
-		esp->auth.icv(esp, skb, 0, skb->len-alen, sum);
+			esp->auth.icv(esp, skb, 0, skb->len-alen, sum);
 
-		if (skb_copy_bits(skb, skb->len-alen, sum1, alen))
-			BUG();
+			if (skb_copy_bits(skb, skb->len-alen, sum1, alen))
+				BUG();
 
-		if (unlikely(memcmp(sum, sum1, alen))) {
-			x->stats.integrity_failed++;
-			goto out;
+			if (unlikely(memcmp(sum, sum1, alen))) {
+				x->stats.integrity_failed++;
+				goto out;
+			}
 		}
 	}
 
+	/* XXX I think this can be moved to the !offload case */
 	if ((nfrags = skb_cow_data(skb, 0, &trailer)) < 0)
 		goto out;
 

@@ -195,15 +206,12 @@
 	esph = (struct ip_esp_hdr*)skb->data;
 	iph = skb->nh.iph;
 
-	/* Get ivec. This can be wrong, check against another impls. */
-	if (esp->conf.ivlen)
-		crypto_cipher_set_iv(esp->conf.tfm, esph->enc_data, crypto_tfm_alg_ivsize(esp->conf.tfm));
-
-        {
-		u8 nexthdr[2];
+	if (!(offload & XFRM_OFFLOAD_CONF)) {
 		struct scatterlist *sg = &esp->sgbuf[0];
-		u8 workbuf[60];
-		int padlen;
+
+		/* Get ivec. This can be wrong, check against another impls. */
+		if (esp->conf.ivlen)
+			crypto_cipher_set_iv(esp->conf.tfm, esph->enc_data, crypto_tfm_alg_ivsize(esp->conf.tfm));
 
 		if (unlikely(nfrags > ESP_NUM_FAST_SG)) {
 			sg = kmalloc(sizeof(struct scatterlist)*nfrags, GFP_ATOMIC);

@@ -214,6 +222,12 @@
 		crypto_cipher_decrypt(esp->conf.tfm, sg, sg, elen);
 		if (unlikely(sg != &esp->sgbuf[0]))
 			kfree(sg);
+	}
+
+        {
+		u8 nexthdr[2];
+		u8 workbuf[60];
+		int padlen;
 
 		if (skb_copy_bits(skb, skb->len-alen-2, nexthdr, 2))
 			BUG();

@@ -69,6 +69,20 @@
 	top_iph = skb->nh.iph;
 	iph = &tmp_iph.iph;
 
+	ah = (struct ip_auth_hdr *)((char *)top_iph+top_iph->ihl*4);
+	ah->nexthdr = top_iph->protocol;
+
+	top_iph->tot_len = htons(skb->len);
+	top_iph->protocol = IPPROTO_AH;
+
+	ahp = x->data;
+	ah->hdrlen  = (XFRM_ALIGN8(sizeof(struct ip_auth_hdr) + 
+				   ahp->icv_trunc_len) >> 2) - 2;
+
+	ah->reserved = 0;
+	ah->spi = x->id.spi;
+	ah->seq_no = htonl(x->replay.oseq + 1);
+
 	iph->tos = top_iph->tos;
 	iph->ttl = top_iph->ttl;
 	iph->frag_off = top_iph->frag_off;

@@ -81,23 +95,11 @@
 			goto error;
 	}
 
-	ah = (struct ip_auth_hdr *)((char *)top_iph+top_iph->ihl*4);
-	ah->nexthdr = top_iph->protocol;
-
 	top_iph->tos = 0;
-	top_iph->tot_len = htons(skb->len);
 	top_iph->frag_off = 0;
 	top_iph->ttl = 0;
-	top_iph->protocol = IPPROTO_AH;
 	top_iph->check = 0;
 
-	ahp = x->data;
-	ah->hdrlen  = (XFRM_ALIGN8(sizeof(struct ip_auth_hdr) + 
-				   ahp->icv_trunc_len) >> 2) - 2;
-
-	ah->reserved = 0;
-	ah->spi = x->id.spi;
-	ah->seq_no = htonl(++x->replay.oseq);
 	ahp->icv(ahp, skb, ah->auth_data);
 
 	top_iph->tos = iph->tos;

@@ -108,6 +110,10 @@
 		memcpy(top_iph+1, iph+1, top_iph->ihl*4 - sizeof(struct iphdr));
 	}
 
+	/* Delay incrementing the replay sequence until we know we're going
+	 * to send this packet to prevent gaps.
+	 */
+	x->replay.oseq++;
 	ip_send_check(top_iph);
 
 	err = 0;

@@ -2420,6 +2420,173 @@
 #undef REQUIRED
 #undef UNSUPPORTED
 
+static struct xfrm_offload *
+typhoon_offload_ipsec(struct typhoon *tp, struct xfrm_state *x)
+{
+	struct cmd_desc xp_cmd[5];
+	struct resp_desc xp_resp;
+	struct sa_descriptor *sa = (struct sa_descriptor *)xp_cmd;
+	struct xfrm_offload *xol;
+	struct typhoon_xfrm_offload *txo;
+	u16 *dir_sa_avail = &tp->rx_sa_avail;
+	u16 cookie;
+	int keylen, err;
+
+	if(!typhoon_validate_xfrm(tp, x))
+		goto error;
+
+	memset(xp_cmd, 0, 5 * sizeof(xp_cmd[0]));
+	INIT_COMMAND_WITH_RESPONSE(xp_cmd, TYPHOON_CMD_CREATE_SA);
+	sa->numDesc = 4;
+
+	sa->mode = TYPHOON_SA_MODE_AH;
+	if(x->type->proto == IPPROTO_ESP)
+		sa->mode = TYPHOON_SA_MODE_ESP;
+
+	if(x->dir == XFRM_STATE_DIR_OUT) {
+		sa->direction = TYPHOON_SA_DIR_TX;
+		dir_sa_avail = &tp->tx_sa_avail;
+	}
+
+	spin_lock_bh(&tp->offload_lock);
+	if(!*dir_sa_avail) {
+		spin_unlock_bh(&tp->offload_lock);
+		goto error;
+	}
+	*dir_sa_avail--;
+	if(!tp->sa_count++) {
+		tp->offload |= TYPHOON_OFFLOAD_IPSEC;
+		err = typhoon_set_offload(tp);
+		if(err < 0) {
+			spin_unlock_bh(&tp->offload_lock);
+			printk(KERN_ERR "%s: unable to enable IPSEC "
+					"offload (%d)\n", tp->name, -err);
+			goto error_counted;
+		}
+	}
+	spin_unlock_bh(&tp->offload_lock);
+
+	if(x->props.aalgo != SADB_X_AALG_NULL && x->aalg) {
+		keylen = (x->aalg->alg_key_len + 7) / 8;
+
+		sa->hashFlags = TYPHOON_SA_HASH_SHA1;
+		if(x->props.aalgo == SADB_AALG_MD5HMAC)
+			sa->hashFlags = TYPHOON_SA_HASH_MD5;
+		sa->hashFlags |= TYPHOON_SA_HASH_ENABLE;
+
+		memcpy(sa->integKey, x->aalg->alg_key, keylen);
+	}
+
+	if(x->props.ealgo != SADB_EALG_NULL && x->ealg) {
+		keylen = (x->ealg->alg_key_len + 7) / 8;
+
+		sa->encryptionFlags = TYPHOON_SA_ENCRYPT_ENABLE |
+						TYPHOON_SA_ENCRYPT_CBC;
+		if(x->props.ealgo == SADB_EALG_DESCBC)
+			sa->encryptionFlags |= TYPHOON_SA_ENCRYPT_DES;
+		else if(x->ealg->alg_key_len == 192)
+			sa->encryptionFlags |= TYPHOON_SA_ENCRYPT_3DES_3KEY;
+		else {
+			sa->encryptionFlags |= TYPHOON_SA_ENCRYPT_3DES_2KEY;
+			memcpy(&sa->confKey[16], x->ealg->alg_key, 8);
+		}
+
+		memcpy(sa->confKey, x->ealg->alg_key, keylen);
+	}
+
+	/* The 3XP expects the SPI to be in host order, litte endian.
+	 * It expects the address to be in network order.
+	 */
+	sa->SPI = cpu_to_le32(ntohl(x->id.spi));
+	sa->destAddr = x->id.daddr.a4;
+	sa->destMask = (u32) ~0UL;
+
+	err = typhoon_issue_command(tp, 5, xp_cmd, 1, &xp_resp);
+	cookie = le16_to_cpu(xp_resp.parm1);
+	if(err < 0 || !cookie || cookie == 0xffff)
+		goto error_counted;
+
+	xol = xfrm_offload_alloc(sizeof(*txo), tp->dev);
+	if(!xol)
+		goto error_cookie;
+
+	txo = xfrm_offload_priv(xol);
+	txo->sa_cookie = cookie;
+	txo->tunnel = !!x->props.mode;
+	txo->ah = (x->id.proto == IPPROTO_AH);
+	txo->inbound = (x->dir == XFRM_STATE_DIR_IN);
+
+	xfrm_state_offload_add(x, xol);
+
+	return xol;
+
+error_cookie:
+	INIT_COMMAND_NO_RESPONSE(xp_cmd, TYPHOON_CMD_DELETE_SA);
+	xp_cmd[0].parm1 = xp_resp.parm1;
+	typhoon_issue_command(tp, 1, xp_cmd, 0, NULL);
+
+error_counted:
+	spin_lock_bh(&tp->offload_lock);
+	*dir_sa_avail++;
+	tp->sa_count--;
+	if(!tp->sa_count) {
+		tp->offload &= ~TYPHOON_OFFLOAD_IPSEC;
+		err = typhoon_set_offload(tp);
+		if(err < 0)
+			printk(KERN_ERR "%s: unable to disable IPSEC "
+					"offload (%d)\n", tp->name, -err);
+	}
+	spin_unlock_bh(&tp->offload_lock);
+
+error:
+	return NULL;
+}
+
+static void
+typhoon_xfrm_state_add(struct net_device *dev, struct xfrm_state *x)
+{
+	struct typhoon *tp = netdev_priv(dev);
+
+	smp_rmb();
+	if(tp->card_state == Running)
+		typhoon_offload_ipsec(tp, x);
+}
+
+static void
+typhoon_xfrm_state_del(struct net_device *dev, struct xfrm_offload *xol)
+{
+	struct typhoon *tp = netdev_priv(dev);
+	struct typhoon_xfrm_offload *txo = xfrm_offload_priv(xol);
+	struct cmd_desc xp_cmd;
+	int err;
+
+	smp_rmb();
+	if(tp->card_state != Running)
+		return;
+
+	INIT_COMMAND_NO_RESPONSE(&xp_cmd, TYPHOON_CMD_DELETE_SA);
+	xp_cmd.parm1 = cpu_to_le16(txo->sa_cookie);
+	if(typhoon_issue_command(tp, 1, &xp_cmd, 0, NULL) < 0) {
+		printk(KERN_ERR "%s: unable to remove offloaded SA 0x%04x\n",
+				tp->name,  txo->sa_cookie);
+	}
+
+	spin_lock_bh(&tp->offload_lock);
+	if(txo->inbound)
+		tp->rx_sa_avail++;
+	else
+		tp->tx_sa_avail++;
+	tp->sa_count--;
+	if(!tp->sa_count) {
+		tp->offload &= ~TYPHOON_OFFLOAD_IPSEC;
+		err = typhoon_set_offload(tp);
+		if(err < 0)
+			printk(KERN_ERR "%s: unable to disable IPSEC "
+					"offload (%d)\n", tp->name, -err);
+	}
+	spin_unlock_bh(&tp->offload_lock);
+}
+
 static void
 typhoon_tx_timeout(struct net_device *dev)
 {

@@ -304,6 +304,7 @@
 	u16			xcvr_select;
 	u16			wol_events;
 	u32			offload;
+	spinlock_t		offload_lock;
 
 	u16			tx_sa_max;
 	u16			rx_sa_max;

@@ -725,11 +726,28 @@
 	return err;
 }
 
+static int
+typhoon_set_offload(struct typhoon *tp)
+{
+	/* Caller should hold tp->offload_lock, or otherwise guarantee
+         * exclusitivity to this routine.
+         */
+	struct cmd_desc xp_cmd;
+
+	smp_rmb();
+	if(tp->card_state != Running)
+		return 0;
+
+	INIT_COMMAND_WITH_RESPONSE(&xp_cmd, TYPHOON_CMD_SET_OFFLOAD_TASKS);
+	xp_cmd.parm2 = tp->offload;
+	xp_cmd.parm3 = tp->offload;
+	return typhoon_issue_command(tp, 1, &xp_cmd, 0, NULL);
+}
+
 static void
 typhoon_vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
 {
 	struct typhoon *tp = netdev_priv(dev);
-	struct cmd_desc xp_cmd;
 	int err;
 
 	spin_lock_bh(&tp->state_lock);

@@ -737,25 +755,16 @@
 		/* We've either been turned on for the first time, or we've
 		 * been turned off. Update the 3XP.
 		 */
+		spin_lock_bh(&tp->offload_lock);
 		if(grp)
 			tp->offload |= TYPHOON_OFFLOAD_VLAN;
 		else
 			tp->offload &= ~TYPHOON_OFFLOAD_VLAN;
+		err = typhoon_set_offload(tp);
+		spin_unlock_bh(&tp->offload_lock);
 
-		/* If the interface is up, the runtime is running -- and we
-		 * must be up for the vlan core to call us.
-		 *
-		 * Do the command outside of the spin lock, as it is slow.
-		 */
-		INIT_COMMAND_WITH_RESPONSE(&xp_cmd,
-					TYPHOON_CMD_SET_OFFLOAD_TASKS);
-		xp_cmd.parm2 = tp->offload;
-		xp_cmd.parm3 = tp->offload;
-		spin_unlock_bh(&tp->state_lock);
-		err = typhoon_issue_command(tp, 1, &xp_cmd, 0, NULL);
 		if(err < 0)
 			printk("%s: vlan offload error %d\n", tp->name, -err);
-		spin_lock_bh(&tp->state_lock);
 	}
 
 	/* now make the change visible */

@@ -1486,6 +1495,7 @@
 
 	spin_lock_init(&tp->command_lock);
 	spin_lock_init(&tp->state_lock);
+	spin_lock_init(&tp->offload_lock);
 }
 
 static void

@@ -2218,12 +2228,13 @@
 	if(err < 0)
 		goto error_out;
 
+	/* tp->card_state != Running, so nothing will change this out
+	 * from under us.
+	 */
 	INIT_COMMAND_NO_RESPONSE(&xp_cmd, TYPHOON_CMD_SET_OFFLOAD_TASKS);
-	spin_lock_bh(&tp->state_lock);
 	xp_cmd.parm2 = tp->offload;
 	xp_cmd.parm3 = tp->offload;
 	err = typhoon_issue_command(tp, 1, &xp_cmd, 0, NULL);
-	spin_unlock_bh(&tp->state_lock);
 	if(err < 0)
 		goto error_out;
 

@@ -73,3 +73,19 @@
 		dev_close code and comments in net/core/dev.c for more info.
 	Context: softirq
 
+dev->xfrm_state_add:
+	Synchronization: None, but can be called inside dev_base_lock rwlock
+	Context: nominally process, but don't sleep inside an rwlock
+	Notes: Only called for inbound xfrm_state(s). Can be invoked during
+       		xfrm_accel_add() call.
+
+dev->xfrm_state_del:
+	Synchronization: None, but can be called inside dev->xmit_lock spinlock.
+	Context: BHs disabled/softirq
+	Notes: Called for all offloaded xfrm_state(s). Can be invoked during
+		xfrm_accel_flush() call.
+
+dev->xfrm_bundle_add:
+	Synchronization: None
+	Context: softirq/process
+	Notes: Called for newly created outbound xfrm bundles.

@@ -33,9 +33,12 @@
 	*) Waiting for a command response takes 8ms due to non-preemptable
 		polling. Only significant for getting stats and creating
 		SAs, but an ugly wart never the less.
+	*) Inbound IPSEC packets of the form outer ESP transport, inner
+		ESP tunnel seems to fail the hash on the inner ESP
+	*) Inbound IPSEC packets of the form outer AH transport, inner
+		AH tunnel seems to fail the hash on the outer AH
 
 	TODO:
-	*) Doesn't do IPSEC offloading. Yet. Keep yer pants on, it's coming.
 	*) Add more support for ethtool (especially for NIC stats)
 	*) Allow disabling of RX checksum offloading
 	*) Fix MAC changing to work while the interface is up

@@ -100,8 +103,8 @@
 #define PKT_BUF_SZ		1536
 
 #define DRV_MODULE_NAME		"typhoon"
-#define DRV_MODULE_VERSION 	"1.5.6"
-#define DRV_MODULE_RELDATE	"04/12/17"
+#define DRV_MODULE_VERSION 	"1.5.6-ipsec"
+#define DRV_MODULE_RELDATE	"04/12/19"
 #define PFX			DRV_MODULE_NAME ": "
 #define ERR_PFX			KERN_ERR PFX
 

@@ -1406,6 +1409,8 @@
 	.get_tso		= ethtool_op_get_tso,
 	.set_tso		= ethtool_op_set_tso,
 	.get_ringparam		= typhoon_get_ringparam,
+	.get_ipsec		= ethtool_op_get_ipsec,
+	.set_ipsec		= ethtool_op_set_ipsec,
 };
 
 static int

@@ -2253,6 +2258,9 @@
 	tp->card_state = Running;
 	smp_wmb();
 
+	if(dev->features & NETIF_F_IPSEC)
+		xfrm_accel_add(dev);
+
 	iowrite32(TYPHOON_INTR_ENABLE_ALL, ioaddr + TYPHOON_REG_INTR_ENABLE);
 	iowrite32(TYPHOON_INTR_NONE, ioaddr + TYPHOON_REG_INTR_MASK);
 	typhoon_post_pci_writes(ioaddr);

@@ -2327,6 +2335,14 @@
 		typhoon_clean_tx(tp, &tp->txLoRing, &indexes->txLoCleared);
 	}
 
+	if(tp->dev->features & NETIF_F_IPSEC)
+		xfrm_accel_flush(tp->dev);
+
+	/* tp->card_state != Running, so nothing will change this out
+         * from under us.
+         */
+	tp->offload &= ~TYPHOON_OFFLOAD_IPSEC;
+
 	return 0;
 }
 

@@ -3183,6 +3199,9 @@
 	dev->set_mac_address	= typhoon_set_mac_address;
 	dev->vlan_rx_register	= typhoon_vlan_rx_register;
 	dev->vlan_rx_kill_vid	= typhoon_vlan_rx_kill_vid;
+	dev->xfrm_state_add	= typhoon_xfrm_state_add;
+	dev->xfrm_state_del	= typhoon_xfrm_state_del;
+	dev->xfrm_bundle_add	= typhoon_xfrm_bundle_add;
 	SET_ETHTOOL_OPS(dev, &typhoon_ethtool_ops);
 
 	/* We can handle scatter gather, up to 16 entries, and

@@ -3190,7 +3209,7 @@
 	 */
 	dev->features |= NETIF_F_SG | NETIF_F_IP_CSUM;
 	dev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
-	dev->features |= NETIF_F_TSO;
+	dev->features |= NETIF_F_TSO | NETIF_F_IPSEC;
 
 	if(register_netdev(dev) < 0)
 		goto error_out_reset;

@@ -2330,6 +2330,96 @@
 	return 0;
 }
 
+#define UNSUPPORTED	goto unsupported
+#define REQUIRED(x)	if(!(x)) goto unsupported
+
+static inline int
+typhoon_validate_ealgo(struct typhoon *tp, struct xfrm_state *x)
+{
+	switch(x->props.ealgo) {
+	case SADB_EALG_NULL:
+		break;
+	case SADB_EALG_DESCBC:
+		REQUIRED(x->ealg);
+		REQUIRED(tp->capabilities & TYPHOON_CRYPTO_DES);
+		REQUIRED(x->ealg->alg_key_len == 64);
+		break;
+	case SADB_EALG_3DESCBC:
+		REQUIRED(x->ealg);
+		REQUIRED(tp->capabilities & TYPHOON_CRYPTO_3DES);
+		REQUIRED(x->ealg->alg_key_len == 128 ||
+					x->ealg->alg_key_len == 192);
+		break;
+	default:
+		UNSUPPORTED;
+	}
+
+	return 1;
+
+unsupported:
+	return 0;
+}
+
+static inline int
+typhoon_validate_aalgo(struct typhoon *tp, struct xfrm_state *x)
+{
+	switch(x->props.aalgo) {
+	case SADB_X_AALG_NULL:
+		break;
+	case SADB_AALG_MD5HMAC:
+		REQUIRED(x->aalg);
+		REQUIRED(x->aalg->alg_key_len == 128);
+		break;
+	case SADB_AALG_SHA1HMAC:
+		REQUIRED(x->aalg);
+		REQUIRED(x->aalg->alg_key_len == 160);
+		break;
+	default:
+		UNSUPPORTED;
+	}
+
+	return 1;
+
+unsupported:
+	return 0;
+}
+
+static inline int
+typhoon_validate_xfrm(struct typhoon *tp, struct xfrm_state *x)
+{
+	u8 ealgo, aalgo, need_auth = 1;
+
+	REQUIRED(x->props.family == AF_INET);
+	REQUIRED(x->dir == XFRM_STATE_DIR_OUT || x->dir == XFRM_STATE_DIR_IN);
+	REQUIRED(!x->encap);
+
+	aalgo = x->props.aalgo;
+	ealgo = x->props.ealgo;
+
+	switch(x->type->proto) {
+	case IPPROTO_ESP:
+		need_auth = 0;
+		REQUIRED(aalgo != SADB_X_AALG_NULL || ealgo != SADB_EALG_NULL);
+		REQUIRED(typhoon_validate_ealgo(tp, x));
+		/* fall through to validate auth algorithm */
+	case IPPROTO_AH:
+		REQUIRED(typhoon_validate_aalgo(tp, x));
+		if(need_auth)
+			REQUIRED(aalgo != SADB_X_AALG_NULL);
+		break;
+	default:
+		UNSUPPORTED;
+	}
+
+	return 1;
+
+unsupported:
+	return 0;
+}
+
+#undef REQUIRED
+#undef UNSUPPORTED
+
 static void
 typhoon_tx_timeout(struct net_device *dev)
 {

@@ -2587,6 +2587,140 @@
 	spin_unlock_bh(&tp->offload_lock);
 }
 
+static inline int
+typhoon_max_offload(struct xfrm_bundle_list *xbl)
+{
+	/* Pre-scan the bundle to avoid offloading problematic sequences.
+	 * Only reduces the offload level to keep as much advantage as
+	 * possible.
+	 *
+	 * For 03.001.002 -- still problematic for 03.001.008, but need
+	 *	re-verify symptoms.
+	 *
+	 * inner AH tunnel, outer AH transport
+	 *	--> 3XP seems to put the inner hash at the wrong location
+	 * inner AH tunnel, outer ESP tunnel
+	 *	--> 3XP corrupts outer hash, maybe wrong place?
+	 * inner AH transport, outer ESP tunnel
+	 *	--> 3XP seems to encrypt the wrong portion of the packet
+	 * inner ESP transport, outer AH tunnel
+	 *	--> 3XP lockup, requires reset
+	 */
+	struct xfrm_bundle_list *bundle;
+	struct dst_entry *dst;
+	struct xfrm_state *x;
+	int last_was_ah = 0, last_was_tunnel = 0;
+	int max_level = 2;
+	int proto;
+
+	list_for_each_entry_reverse(bundle, &xbl->node, node) {
+		dst = bundle->dst;
+		x = dst->xfrm;
+
+		proto = x ? x->type->proto : IPPROTO_IP;
+
+		if(proto == IPPROTO_AH && x->props.mode &&
+					(last_was_ah ^ last_was_tunnel))
+			goto problem_offload;
+
+		if(proto == IPPROTO_AH && !x->props.mode &&
+					(!last_was_ah && last_was_tunnel))
+			goto problem_offload;
+
+		if(proto == IPPROTO_ESP && last_was_ah && last_was_tunnel)
+			goto problem_offload;
+
+		last_was_ah = (proto == IPPROTO_AH) ? 1 : 0;
+		last_was_tunnel = x ? x->props.mode : 0;
+		continue;
+
+problem_offload:
+		max_level--;
+		break;
+	}
+
+	return max_level;
+}
+
+static void
+typhoon_xfrm_bundle_add(struct net_device *dev, struct xfrm_bundle_list *xbl)
+{
+	/* Walk from the outermost dst back up the chain, offloading
+	 * until we hit something we cannot deal with.
+	 */
+	struct typhoon *tp = netdev_priv(dev);
+	struct xfrm_bundle_list *bundle;
+	struct dst_entry *dst;
+	struct xfrm_state *x;
+	struct xfrm_offload *xol;
+	struct typhoon_xfrm_offload *txo;
+	int proto;
+	int level = 0, max_level;
+	int last = -1;
+
+	smp_rmb();
+	if(tp->card_state != Running)
+		return;
+
+	max_level = typhoon_max_offload(xbl);
+
+	list_for_each_entry_reverse(bundle, &xbl->node, node) {
+		dst = bundle->dst;
+		x = dst->xfrm;
+
+		/* Only support IPv4 */
+		if(dst->ops->family != AF_INET)
+			goto cannot_offload;
+
+		proto = x ? x->type->proto : IPPROTO_IP;
+
+		switch(proto) {
+		case IPPROTO_IP:
+		case IPPROTO_IPIP:
+			if(last == IPPROTO_IP || last == IPPROTO_IPIP)
+				goto cannot_offload;
+			if(level)
+				level++;
+			last = proto;
+			continue;
+		case IPPROTO_ESP:
+			if(last != IPPROTO_AH)
+				level++;
+			break;
+		case IPPROTO_AH:
+			level++;
+			break;
+		default:
+			goto cannot_offload;
+		}
+
+		last = proto;
+		if(level > max_level)
+			goto cannot_offload;
+
+		if(dst->xfrm_offload)
+			continue;
+
+		xol = xfrm_offload_get(x, dev);
+		if(!xol) {
+			xol = typhoon_offload_ipsec(tp, x);
+			if(xol)
+				xfrm_offload_hold(xol);
+		}
+
+		if(!xol)
+			goto cannot_offload;
+
+		dst->xfrm_offload = xol;
+		txo = xfrm_offload_priv(xol);
+		if(txo->tunnel)
+			last = IPPROTO_IPIP;
+	}
+
+cannot_offload:
+	return;
+}
+
 static void
 typhoon_tx_timeout(struct net_device *dev)
 {

@@ -305,6 +305,12 @@
 	u16			wol_events;
 	u32			offload;
 
+	u16			tx_sa_max;
+	u16			rx_sa_max;
+	u16			tx_sa_avail;
+	u16			rx_sa_avail;
+	int			sa_count;
+
 	/* unused stuff (future use) */
 	int			capabilities;
 	struct transmit_ring 	txHiRing;

@@ -2105,6 +2111,53 @@
 	return 0;
 }
 
+static inline int
+typhoon_ipsec_init(struct typhoon *tp)
+{
+	struct cmd_desc xp_cmd;
+	struct resp_desc xp_resp;
+	struct ipsec_info_desc *info = (struct ipsec_info_desc *) &xp_resp;
+	u16 last_tx, last_rx, last_cap;
+	int err;
+
+	last_tx = tp->tx_sa_max;
+	last_rx = tp->rx_sa_max;
+	last_cap = tp->capabilities;
+
+	INIT_COMMAND_WITH_RESPONSE(&xp_cmd, TYPHOON_CMD_READ_IPSEC_INFO);
+	err = typhoon_issue_command(tp, 1, &xp_cmd, 1, &xp_resp);
+	if(err < 0)
+		goto out;
+
+	/* We're not up yet, so no need to lock this -- we cannot modify
+	 * these fields yet.
+	 */
+	tp->tx_sa_avail = tp->tx_sa_max = le16_to_cpu(info->tx_sa_max);
+	tp->rx_sa_avail = tp->rx_sa_max = le16_to_cpu(info->rx_sa_max);
+	tp->sa_count = 0;
+
+	/* Typhoon2 was originally going to have variable crypto capabilities,
+	 * subject to registration with 3Com. It appears they have decided
+	 * to just enable 3DES as well.
+	 */
+	if(tp->capabilities & TYPHOON_CRYPTO_VARIABLE) {
+		tp->capabilities &= ~TYPHOON_CRYPTO_VARIABLE;
+		tp->capabilities |= TYPHOON_CRYPTO_DES | TYPHOON_CRYPTO_3DES;
+	}
+
+	if(last_tx != tp->tx_sa_max || last_rx != tp->rx_sa_max ||
+					last_cap != tp->capabilities) {
+		printk(KERN_INFO "%s: IPSEC offload %s%s %d Tx %d Rx\n",
+			tp->name,
+			tp->capabilities & TYPHOON_CRYPTO_DES ? "DES " : "",
+			tp->capabilities & TYPHOON_CRYPTO_3DES ? "3DES" : "",
+			tp->tx_sa_max, tp->rx_sa_max);
+	}
+
+out:
+	return err;
+}
+
 static int
 typhoon_start_runtime(struct typhoon *tp)
 {

@@ -2127,6 +2180,9 @@
 		err = -EIO;
 		goto error_out;
 	}
+
+	if(typhoon_ipsec_init(tp))
+		goto error_out;
 
 	INIT_COMMAND_NO_RESPONSE(&xp_cmd, TYPHOON_CMD_SET_MAX_PKT_SIZE);
 	xp_cmd.parm1 = cpu_to_le16(PKT_BUF_SZ);

@@ -487,6 +487,20 @@
 	u32 unused2;
 } __attribute__ ((packed));
 
+/* TYPHOON_CMD_READ_IPSEC_INFO response descriptor
+ */
+struct ipsec_info_desc {
+	u8  flags;
+	u8  numDesc;
+	u16 cmd;
+	u16 seqNo;
+	u16 des_enabled;
+	u16 tx_sa_max;
+	u16 rx_sa_max;
+	u16 tx_sa_count;
+	u16 rx_sa_count;
+} __attribute__ ((packed));
+
 /* TYPHOON_CMD_SET_OFFLOAD_TASKS bits (cmd.parm2 (Tx) & cmd.parm3 (Rx))
  * This is all for IPv4.
  */

@@ -352,6 +352,15 @@
 #define TSO_OFFLOAD_ON		0
 #endif
 
+#define IPSEC_NUM_DESCRIPTORS	1
+
+struct typhoon_xfrm_offload {
+	u16	sa_cookie;
+	u16	tunnel:1,
+		ah:1,
+		inbound:1;
+};
+
 static inline void
 typhoon_inc_index(u32 *index, const int count, const int num_entries)
 {

@@ -779,12 +788,115 @@
 	tcpd->status = 0;
 }
 
+static inline int
+typhoon_ipsec_fill(struct typhoon *tp, struct sk_buff *skb,
+				struct transmit_ring *txRing)
+{
+	struct xfrm_offload *xol;
+	struct typhoon_xfrm_offload *txo;
+	struct ipsec_desc *ipsec;
+	int last_was_esp = 0;
+	int i, entry;
+	u32 sa[3];
+
+	ipsec = (struct ipsec_desc *) (txRing->ringBase + txRing->lastWrite);
+	typhoon_inc_tx_index(&txRing->lastWrite, 1);
+
+	ipsec->flags = TYPHOON_OPT_DESC | TYPHOON_OPT_IPSEC;
+	ipsec->numDesc = 1;
+	ipsec->ipsecFlags = TYPHOON_IPSEC_USE_IV;
+	ipsec->reserved = 0;
+	sa[0] = sa[1] = sa[2] = 0;
+
+	/* Fill the offload descriptor with the cookies to indicate
+	 * which key set to use when. While we're looping through the
+	 * offloaded xfrms, if the last xfrm was ESP, and we're doing
+	 * AH now, * then we can move the ESP part to the top of the
+	 * descriptor. Otherwise, we'll need to move to the next one.
+	 * We overrun into sa[2] to prevent needing to check the entry
+	 * limit in the middile of things.
+	 */
+	entry = i = 0;
+	xol = skb_get_xfrm_offload(skb, i++);
+	while(xol && entry < 2) {
+		xfrm_offload_hold(xol);
+		txo = xfrm_offload_priv(xol);
+		if(sa[entry] && txo->tunnel)
+			entry++;
+		if(sa[entry] & 0xffff) {
+			if(last_was_esp && txo->ah)
+				sa[entry] <<= 16;
+			else
+				entry++;
+		}
+
+		sa[entry] |= txo->sa_cookie;
+		last_was_esp = !txo->ah;
+
+		xol = skb_get_xfrm_offload(skb, i++);
+	}
+
+	/* Make sure we used all of the xfrms that were offloaded.
+	 */
+	if(unlikely(entry == 2 && xol)) {
+		if(net_ratelimit())
+			printk(KERN_ERR "%s: failing to offload IPSEC packet "
+					"with too many xfrms!\n", tp->name);
+		goto bad_packet;
+	}
+
+	ipsec->sa[0] = cpu_to_le16(sa[0] & 0xffff);
+	ipsec->sa[1] = cpu_to_le16(sa[0] >> 16);
+	ipsec->sa[2] = cpu_to_le16(sa[1] & 0xffff);
+	ipsec->sa[3] = cpu_to_le16(sa[1] >> 16);
+
+	/* The current 3XP firmware seems to hang if we try to feed it
+	 * the same (non-zero) SA twice on the same packet. So, detect
+	 * and drop those packets as it is likely a stack bug, or
+	 * misconfiguration of policy.
+	 *
+	 * I.e., we should never hit this.
+	 */
+	if(unlikely(ipsec->sa[2])) {
+		if(unlikely(ipsec->sa[2] == ipsec->sa[3]))
+			goto avoiding_sa_hang;
+		if(unlikely(ipsec->sa[2] == ipsec->sa[0] ||
+					ipsec->sa[2] == ipsec->sa[1]))
+			goto avoiding_sa_hang;
+		if(unlikely(ipsec->sa[3] && (ipsec->sa[3] == ipsec->sa[0] ||
+					ipsec->sa[3] == ipsec->sa[1])))
+			goto avoiding_sa_hang;
+	}
+
+	if(unlikely(ipsec->sa[1] && ipsec->sa[0] == ipsec->sa[1]))
+		goto avoiding_sa_hang;
+
+	return 0;
+
+avoiding_sa_hang:
+	if(net_ratelimit())
+		printk(KERN_ERR "%s: failing attempted IPSEC offload with "
+				"duplicate SAs %08x %08x\n", tp->name,
+				sa[0], sa[1]);
+
+bad_packet:
+	/* Any xfrm_offloads we've attached to this skb will be
+	 * released for us when typhoon_start_tx() calls dev_kfree_skb_any()
+	 * on it.
+	 *
+	 * Return an error to indicate this packet cannot be offloaded as
+	 * specified and should never make it to the wire.
+	 */
+	return -EINVAL;
+}
+
 static int
 typhoon_start_tx(struct sk_buff *skb, struct net_device *dev)
 {
 	struct typhoon *tp = netdev_priv(dev);
 	struct transmit_ring *txRing;
 	struct tx_desc *txd, *first_txd;
+	u32 origLastWrite;
 	dma_addr_t skb_dma;
 	int numDesc;
 

@@ -811,6 +923,9 @@
 	if(skb_tso_size(skb))
 		numDesc++;
 
+	if(skb_has_xfrm_offload(skb))
+		numDesc++;
+
 	/* When checking for free space in the ring, we need to also
 	 * account for the initial Tx descriptor, and we always must leave
 	 * at least one descriptor unused in the ring so that it doesn't

@@ -823,6 +938,7 @@
 	while(unlikely(typhoon_num_free_tx(txRing) < (numDesc + 2)))
 		smp_rmb();
 
+	origLastWrite = txRing->lastWrite;
 	first_txd = (struct tx_desc *) (txRing->ringBase + txRing->lastWrite);
 	typhoon_inc_tx_index(&txRing->lastWrite, 1);
 

@@ -855,6 +971,14 @@
 		typhoon_tso_fill(skb, txRing, tp->txlo_dma_addr);
 	}
 
+	if(skb_has_xfrm_offload(skb)) {
+		first_txd->processFlags |= TYPHOON_TX_PF_IPSEC;
+		first_txd->numDesc++;
+
+		if(typhoon_ipsec_fill(tp, skb, txRing))
+			goto error;
+	}
+
 	txd = (struct tx_desc *) (txRing->ringBase + txRing->lastWrite);
 	typhoon_inc_tx_index(&txRing->lastWrite, 1);
 

@@ -915,6 +1039,7 @@
 	 * Tx header.
 	 */
 	numDesc = MAX_SKB_FRAGS + TSO_NUM_DESCRIPTORS + 1;
+	numDesc += IPSEC_NUM_DESCRIPTORS;
 
 	if(typhoon_num_free_tx(txRing) < (numDesc + 2)) {
 		netif_stop_queue(dev);

@@ -927,6 +1052,11 @@
 			netif_wake_queue(dev);
 	}
 
+	return 0;
+
+error:
+	txRing->lastWrite = origLastWrite;
+	dev_kfree_skb_any(skb);
 	return 0;
 }