diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h
index 89d1d0556b6e83a44fc6986d080072d6e93c2045..630a3fcf65bc8113fd6b67ce587f26fbf46c9774 100644
--- a/drivers/net/xen-netback/common.h
+++ b/drivers/net/xen-netback/common.h
@@ -124,6 +124,7 @@ struct xenvif {
struct pending_tx_info pending_tx_info[MAX_PENDING_REQS];
grant_handle_t grant_tx_handle[MAX_PENDING_REQS];
+ struct gnttab_copy tx_copy_ops[MAX_PENDING_REQS];
struct gnttab_map_grant_ref tx_map_ops[MAX_PENDING_REQS];
struct gnttab_unmap_grant_ref tx_unmap_ops[MAX_PENDING_REQS];
/* passed to gnttab_[un]map_refs with pages under (un)mapping */
diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c
index 4bb78862afc8027867c1c78287a01a2b054d45b9..99c8f09b465437318d69010bb199741adf8bbbdc 100644
--- a/drivers/net/xen-netback/netback.c
+++ b/drivers/net/xen-netback/netback.c
@@ -940,35 +940,37 @@ static inline void xenvif_grant_handle_reset(struct xenvif *vif,
static int xenvif_tx_check_gop(struct xenvif *vif,
struct sk_buff *skb,
- struct gnttab_map_grant_ref **gopp_map)
+ struct gnttab_map_grant_ref **gopp_map,
+ struct gnttab_copy **gopp_copy)
{
struct gnttab_map_grant_ref *gop_map = *gopp_map;
u16 pending_idx = XENVIF_TX_CB(skb)->pending_idx;
struct skb_shared_info *shinfo = skb_shinfo(skb);
- struct pending_tx_info *tx_info;
int nr_frags = shinfo->nr_frags;
- int i, err, start;
+ int i, err;
struct sk_buff *first_skb = NULL;
/* Check status of header. */
- err = gop_map->status;
- if (unlikely(err))
+ err = (*gopp_copy)->status;
+ (*gopp_copy)++;
+ if (unlikely(err)) {
+ if (net_ratelimit())
+ netdev_dbg(vif->dev,
+ "Grant copy of header failed! status: %d pending_idx% %u ref: %u\n",
+ (*gopp_copy)->status,
+ pending_idx,
+ (*gopp_copy)->source.u.ref);
xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR);
- else
- xenvif_grant_handle_set(vif, pending_idx , gop_map->handle);
-
- /* Skip first skb fragment if it is on same page as header fragment. */
- start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
+ }
check_frags:
- for (i = start; i < nr_frags; i++) {
+ for (i = 0; i < nr_frags; i++, gop_map++) {
int j, newerr;
pending_idx = frag_get_pending_idx(&shinfo->frags[i]);
- tx_info = &vif->pending_tx_info[pending_idx];
/* Check error status: if okay then remember grant handle. */
- newerr = (++gop_map)->status;
+ newerr = gop_map->status;
if (likely(!newerr)) {
xenvif_grant_handle_set(vif,
@@ -981,18 +983,20 @@ static int xenvif_tx_check_gop(struct xenvif *vif,
}
/* Error on this fragment: respond to client with an error. */
+ if (net_ratelimit())
+ netdev_dbg(vif->dev,
+ "Grant map of %d. frag failed! status: %d pending_idx% %u ref: %u\n",
+ i,
+ gop_map->status,
+ pending_idx,
+ gop_map->ref);
xenvif_idx_release(vif, pending_idx, XEN_NETIF_RSP_ERROR);
/* Not the first error? Preceding frags already invalidated. */
if (err)
continue;
- /* First error: invalidate header and preceding fragments. */
- if (!first_skb)
- pending_idx = XENVIF_TX_CB(skb)->pending_idx;
- else
- pending_idx = XENVIF_TX_CB(skb)->pending_idx;
- xenvif_idx_unmap(vif, pending_idx);
- for (j = start; j < i; j++) {
+ /* First error: invalidate preceding fragments. */
+ for (j = 0; j < i; j++) {
pending_idx = frag_get_pending_idx(&shinfo->frags[j]);
xenvif_idx_unmap(vif, pending_idx);
}
@@ -1006,7 +1010,6 @@ static int xenvif_tx_check_gop(struct xenvif *vif,
skb = shinfo->frag_list;
shinfo = skb_shinfo(skb);
nr_frags = shinfo->nr_frags;
- start = 0;
goto check_frags;
}
@@ -1017,15 +1020,13 @@ static int xenvif_tx_check_gop(struct xenvif *vif,
if (first_skb && err) {
int j;
shinfo = skb_shinfo(first_skb);
- pending_idx = XENVIF_TX_CB(skb)->pending_idx;
- start = (frag_get_pending_idx(&shinfo->frags[0]) == pending_idx);
- for (j = start; j < shinfo->nr_frags; j++) {
+ for (j = 0; j < shinfo->nr_frags; j++) {
pending_idx = frag_get_pending_idx(&shinfo->frags[j]);
xenvif_idx_unmap(vif, pending_idx);
}
}
- *gopp_map = gop_map + 1;
+ *gopp_map = gop_map;
return err;
}
@@ -1036,9 +1037,6 @@ static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb)
int i;
u16 prev_pending_idx = INVALID_PENDING_IDX;
- if (skb_shinfo(skb)->destructor_arg)
- prev_pending_idx = XENVIF_TX_CB(skb)->pending_idx;
-
for (i = 0; i < nr_frags; i++) {
skb_frag_t *frag = shinfo->frags + i;
struct xen_netif_tx_request *txp;
@@ -1048,10 +1046,10 @@ static void xenvif_fill_frags(struct xenvif *vif, struct sk_buff *skb)
pending_idx = frag_get_pending_idx(frag);
/* If this is not the first frag, chain it to the previous*/
- if (unlikely(prev_pending_idx == INVALID_PENDING_IDX))
+ if (prev_pending_idx == INVALID_PENDING_IDX)
skb_shinfo(skb)->destructor_arg =
&callback_param(vif, pending_idx);
- else if (likely(pending_idx != prev_pending_idx))
+ else
callback_param(vif, prev_pending_idx).ctx =
&callback_param(vif, pending_idx);
@@ -1191,7 +1189,10 @@ static bool tx_credit_exceeded(struct xenvif *vif, unsigned size)
return false;
}
-static unsigned xenvif_tx_build_gops(struct xenvif *vif, int budget)
+static void xenvif_tx_build_gops(struct xenvif *vif,
+ int budget,
+ unsigned *copy_ops,
+ unsigned *map_ops)
{
struct gnttab_map_grant_ref *gop = vif->tx_map_ops, *request_gop;
struct sk_buff *skb;
@@ -1294,22 +1295,36 @@ static unsigned xenvif_tx_build_gops(struct xenvif *vif, int budget)
}
}
- xenvif_tx_create_map_op(vif, pending_idx, &txreq, gop);
-
- gop++;
-
XENVIF_TX_CB(skb)->pending_idx = pending_idx;
__skb_put(skb, data_len);
+ vif->tx_copy_ops[*copy_ops].source.u.ref = txreq.gref;
+ vif->tx_copy_ops[*copy_ops].source.domid = vif->domid;
+ vif->tx_copy_ops[*copy_ops].source.offset = txreq.offset;
+
+ vif->tx_copy_ops[*copy_ops].dest.u.gmfn =
+ virt_to_mfn(skb->data);
+ vif->tx_copy_ops[*copy_ops].dest.domid = DOMID_SELF;
+ vif->tx_copy_ops[*copy_ops].dest.offset =
+ offset_in_page(skb->data);
+
+ vif->tx_copy_ops[*copy_ops].len = data_len;
+ vif->tx_copy_ops[*copy_ops].flags = GNTCOPY_source_gref;
+
+ (*copy_ops)++;
skb_shinfo(skb)->nr_frags = ret;
if (data_len < txreq.size) {
skb_shinfo(skb)->nr_frags++;
frag_set_pending_idx(&skb_shinfo(skb)->frags[0],
pending_idx);
+ xenvif_tx_create_map_op(vif, pending_idx, &txreq, gop);
+ gop++;
} else {
frag_set_pending_idx(&skb_shinfo(skb)->frags[0],
INVALID_PENDING_IDX);
+ memcpy(&vif->pending_tx_info[pending_idx].req, &txreq,
+ sizeof(txreq));
}
vif->pending_cons++;
@@ -1326,11 +1341,13 @@ static unsigned xenvif_tx_build_gops(struct xenvif *vif, int budget)
vif->tx.req_cons = idx;
- if ((gop-vif->tx_map_ops) >= ARRAY_SIZE(vif->tx_map_ops))
+ if (((gop-vif->tx_map_ops) >= ARRAY_SIZE(vif->tx_map_ops)) ||
+ (*copy_ops >= ARRAY_SIZE(vif->tx_copy_ops)))
break;
}
- return gop - vif->tx_map_ops;
+ (*map_ops) = gop - vif->tx_map_ops;
+ return;
}
/* Consolidate skb with a frag_list into a brand new one with local pages on
@@ -1402,6 +1419,7 @@ static int xenvif_handle_frag_list(struct xenvif *vif, struct sk_buff *skb)
static int xenvif_tx_submit(struct xenvif *vif)
{
struct gnttab_map_grant_ref *gop_map = vif->tx_map_ops;
+ struct gnttab_copy *gop_copy = vif->tx_copy_ops;
struct sk_buff *skb;
int work_done = 0;
@@ -1414,27 +1432,22 @@ static int xenvif_tx_submit(struct xenvif *vif)
txp = &vif->pending_tx_info[pending_idx].req;
/* Check the remap error code. */
- if (unlikely(xenvif_tx_check_gop(vif, skb, &gop_map))) {
- netdev_dbg(vif->dev, "netback grant failed.\n");
+ if (unlikely(xenvif_tx_check_gop(vif, skb, &gop_map, &gop_copy))) {
skb_shinfo(skb)->nr_frags = 0;
kfree_skb(skb);
continue;
}
data_len = skb->len;
- memcpy(skb->data,
- (void *)(idx_to_kaddr(vif, pending_idx)|txp->offset),
- data_len);
callback_param(vif, pending_idx).ctx = NULL;
if (data_len < txp->size) {
/* Append the packet payload as a fragment. */
txp->offset += data_len;
txp->size -= data_len;
- skb_shinfo(skb)->destructor_arg =
- &callback_param(vif, pending_idx);
} else {
/* Schedule a response immediately. */
- xenvif_idx_unmap(vif, pending_idx);
+ xenvif_idx_release(vif, pending_idx,
+ XEN_NETIF_RSP_OKAY);
}
if (txp->flags & XEN_NETTXF_csum_blank)
@@ -1613,22 +1626,25 @@ static inline void xenvif_tx_dealloc_action(struct xenvif *vif)
/* Called after netfront has transmitted */
int xenvif_tx_action(struct xenvif *vif, int budget)
{
- unsigned nr_mops;
+ unsigned nr_mops, nr_cops = 0;
int work_done, ret;
if (unlikely(!tx_work_todo(vif)))
return 0;
- nr_mops = xenvif_tx_build_gops(vif, budget);
+ xenvif_tx_build_gops(vif, budget, &nr_cops, &nr_mops);
- if (nr_mops == 0)
+ if (nr_cops == 0)
return 0;
- ret = gnttab_map_refs(vif->tx_map_ops,
- NULL,
- vif->pages_to_map,
- nr_mops);
- BUG_ON(ret);
+ gnttab_batch_copy(vif->tx_copy_ops, nr_cops);
+ if (nr_mops != 0) {
+ ret = gnttab_map_refs(vif->tx_map_ops,
+ NULL,
+ vif->pages_to_map,
+ nr_mops);
+ BUG_ON(ret);
+ }
work_done = xenvif_tx_submit(vif);