diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index 16d6fe9546959fd679b7db7dde1e994ec8977503..b51e021354b536430ec0260b2a4f5e09d675dbd2 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -1407,7 +1407,7 @@ static int bond_compute_features(struct bonding *bond)
int i;
features &= ~(NETIF_F_ALL_CSUM | BOND_VLAN_FEATURES);
- features |= NETIF_F_GSO_MASK | NETIF_F_NO_CSUM;
+ features |= NETIF_F_GSO_MASK | NETIF_F_NO_CSUM | NETIF_F_NOCACHE_COPY;
if (!bond->first_slave)
goto done;
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index a4664cc68e2bdd4f5e966780a7b55c91adf49430..09d262415769aef45f76baf86b27344ed78e5fb8 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1066,6 +1066,7 @@ struct net_device {
#define NETIF_F_NTUPLE (1 << 27) /* N-tuple filters supported */
#define NETIF_F_RXHASH (1 << 28) /* Receive hashing offload */
#define NETIF_F_RXCSUM (1 << 29) /* Receive checksumming offload */
+#define NETIF_F_NOCACHE_COPY (1 << 30) /* Use no-cache copyfromuser */
/* Segmentation offload features */
#define NETIF_F_GSO_SHIFT 16
@@ -1081,7 +1082,7 @@ struct net_device {
/* = all defined minus driver/device-class-related */
#define NETIF_F_NEVER_CHANGE (NETIF_F_HIGHDMA | NETIF_F_VLAN_CHALLENGED | \
NETIF_F_LLTX | NETIF_F_NETNS_LOCAL)
-#define NETIF_F_ETHTOOL_BITS (0x3f3fffff & ~NETIF_F_NEVER_CHANGE)
+#define NETIF_F_ETHTOOL_BITS (0x7f3fffff & ~NETIF_F_NEVER_CHANGE)
/* List of features with software fallbacks. */
#define NETIF_F_GSO_SOFTWARE (NETIF_F_TSO | NETIF_F_TSO_ECN | \
diff --git a/include/net/sock.h b/include/net/sock.h
index da0534d3401cdee0ed17ce84150365b73c3c9327..43bd515e92fdea4988876abe7afb65f1c7826bca 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -52,6 +52,7 @@
#include <linux/mm.h>
#include <linux/security.h>
#include <linux/slab.h>
+#include <linux/uaccess.h>
#include <linux/filter.h>
#include <linux/rculist_nulls.h>
@@ -1389,6 +1390,58 @@ static inline void sk_nocaps_add(struct sock *sk, int flags)
sk->sk_route_caps &= ~flags;
}
+static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
+ char __user *from, char *to,
+ int copy)
+{
+ if (skb->ip_summed == CHECKSUM_NONE) {
+ int err = 0;
+ __wsum csum = csum_and_copy_from_user(from, to, copy, 0, &err);
+ if (err)
+ return err;
+ skb->csum = csum_block_add(skb->csum, csum, skb->len);
+ } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
+ if (!access_ok(VERIFY_READ, from, copy) ||
+ __copy_from_user_nocache(to, from, copy))
+ return -EFAULT;
+ } else if (copy_from_user(to, from, copy))
+ return -EFAULT;
+
+ return 0;
+}
+
+static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
+ char __user *from, int copy)
+{
+ int err;
+
+ err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy), copy);
+ if (err)
+ __skb_trim(skb, skb->len);
+
+ return err;
+}
+
+static inline int skb_copy_to_page_nocache(struct sock *sk, char __user *from,
+ struct sk_buff *skb,
+ struct page *page,
+ int off, int copy)
+{
+ int err;
+
+ err = skb_do_copy_data_nocache(sk, skb, from,
+ page_address(page) + off, copy);
+ if (err)
+ return err;
+
+ skb->len += copy;
+ skb->data_len += copy;
+ skb->truesize += copy;
+ sk->sk_wmem_queued += copy;
+ sk_mem_charge(sk, copy);
+ return 0;
+}
+
static inline int skb_copy_to_page(struct sock *sk, char __user *from,
struct sk_buff *skb, struct page *page,
int off, int copy)
diff --git a/net/core/dev.c b/net/core/dev.c
index 02f56376fe993910b3317e45e73626b399f0f0ba..5d0b4f6f1a72db3d66ce0a2542e80a8d719c8580 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -5425,6 +5425,14 @@ int register_netdevice(struct net_device *dev)
dev->features &= ~NETIF_F_GSO;
}
+ /* Turn on no cache copy if HW is doing checksum */
+ dev->hw_features |= NETIF_F_NOCACHE_COPY;
+ if ((dev->features & NETIF_F_ALL_CSUM) &&
+ !(dev->features & NETIF_F_NO_CSUM)) {
+ dev->wanted_features |= NETIF_F_NOCACHE_COPY;
+ dev->features |= NETIF_F_NOCACHE_COPY;
+ }
+
/* Enable GRO and NETIF_F_HIGHDMA for vlans by default,
* vlan_dev_init() will do the dev->features check, so these features
* are enabled only if supported by underlying device.
@@ -6182,6 +6190,10 @@ u32 netdev_increment_features(u32 all, u32 one, u32 mask)
}
}
+ /* If device can't no cache copy, don't do for all */
+ if (!(one & NETIF_F_NOCACHE_COPY))
+ all &= ~NETIF_F_NOCACHE_COPY;
+
one |= NETIF_F_ALL_CSUM;
one |= all & NETIF_F_ONE_FOR_ALL;
diff --git a/net/core/ethtool.c b/net/core/ethtool.c
index 439e4b0e1312658b402ab33186c2e17a04961c9f..719670ae199c35ec28e5aeb7f1a5aedeba01b02c 100644
--- a/net/core/ethtool.c
+++ b/net/core/ethtool.c
@@ -359,7 +359,7 @@ static const char netdev_features_strings[ETHTOOL_DEV_FEATURE_WORDS * 32][ETH_GS
/* NETIF_F_NTUPLE */ "rx-ntuple-filter",
/* NETIF_F_RXHASH */ "rx-hashing",
/* NETIF_F_RXCSUM */ "rx-checksum",
- "",
+ /* NETIF_F_NOCACHE_COPY */ "tx-nocache-copy"
"",
};
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index b22d450105450ae089d42a06486938fcaa2c80dc..054a59d21eb0d4211bf4857f6a15b33d5e80abdc 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -999,7 +999,8 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
/* We have some space in skb head. Superb! */
if (copy > skb_tailroom(skb))
copy = skb_tailroom(skb);
- if ((err = skb_add_data(skb, from, copy)) != 0)
+ err = skb_add_data_nocache(sk, skb, from, copy);
+ if (err)
goto do_fault;
} else {
int merge = 0;
@@ -1042,8 +1043,8 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
/* Time to copy data. We are close to
* the end! */
- err = skb_copy_to_page(sk, from, skb, page,
- off, copy);
+ err = skb_copy_to_page_nocache(sk, from, skb,
+ page, off, copy);
if (err) {
/* If this page was new, give it to the
* socket so it does not get leaked.