iscsi_tcp.c

/*
 * iSCSI Initiator over TCP/IP Data-Path
 *
 * Copyright (C) 2004 Dmitry Yusupov
 * Copyright (C) 2004 Alex Aizman
 * Copyright (C) 2005 Mike Christie
 * maintained by open-iscsi@googlegroups.com
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published
 * by the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 *
 * See the file COPYING included with this distribution for more details.
 *
 * Credits:
 *	Christoph Hellwig
 *	FUJITA Tomonori
 *	Arne Redlich
 *	Zhenyu Wang
 */

#include <linux/types.h>
#include <linux/list.h>
#include <linux/inet.h>
#include <linux/blkdev.h>
#include <linux/crypto.h>
#include <linux/delay.h>
#include <linux/kfifo.h>
#include <linux/scatterlist.h>
#include <linux/mutex.h>
#include <net/tcp.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_eh.h>
#include <scsi/scsi_request.h>
#include <scsi/scsi_tcq.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi.h>
#include <scsi/scsi_transport_iscsi.h>

#include "iscsi_tcp.h"

MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
	      "Alex Aizman <itn780@yahoo.com>");
MODULE_DESCRIPTION("iSCSI/TCP data-path");
MODULE_LICENSE("GPL");
MODULE_VERSION("0:4.445");
/* #define DEBUG_TCP */
/* #define DEBUG_SCSI */
#define DEBUG_ASSERT

#ifdef DEBUG_TCP
#define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt)
#else
#define debug_tcp(fmt...)
#endif

#ifdef DEBUG_SCSI
#define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt)
#else
#define debug_scsi(fmt...)
#endif

#ifndef DEBUG_ASSERT
#ifdef BUG_ON
#undef BUG_ON
#endif
#define BUG_ON(expr)
#endif

#define INVALID_SN_DELTA	0xffff

static unsigned int iscsi_max_lun = 512;
module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);

/* global data */
static kmem_cache_t *taskcache;

static inline void
iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size)
{
	sg_init_one(&ibuf->sg, (u8 *)vbuf, size);
	ibuf->sent = 0;
	ibuf->use_sendmsg = 0;
}

static inline void
iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
{
	ibuf->sg.page = virt_to_page(vbuf);
	ibuf->sg.offset = offset_in_page(vbuf);
	ibuf->sg.length = size;
	ibuf->sent = 0;
	ibuf->use_sendmsg = 1;
}

static inline void
iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
{
	ibuf->sg.page = sg->page;
	ibuf->sg.offset = sg->offset;
	ibuf->sg.length = sg->length;
	/*
	 * Fastpath: sg element fits into single page
	 */
	if (sg->length + sg->offset <= PAGE_SIZE && !PageSlab(sg->page))
		ibuf->use_sendmsg = 0;
	else
		ibuf->use_sendmsg = 1;
	ibuf->sent = 0;
}

static inline int
iscsi_buf_left(struct iscsi_buf *ibuf)
{
	int rc;

	rc = ibuf->sg.length - ibuf->sent;
	BUG_ON(rc < 0);
	return rc;
}

static inline void
iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
		 u8* crc)
{
	crypto_digest_digest(conn->tx_tfm, &buf->sg, 1, crc);
	buf->sg.length += sizeof(uint32_t);
}

static void
iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err)
{
	struct iscsi_session *session = conn->session;
	unsigned long flags;

	spin_lock_irqsave(&session->lock, flags);
	if (session->conn_cnt == 1 || session->leadconn == conn)
		session->state = ISCSI_STATE_FAILED;
	spin_unlock_irqrestore(&session->lock, flags);
	set_bit(SUSPEND_BIT, &conn->suspend_tx);
	set_bit(SUSPEND_BIT, &conn->suspend_rx);
	iscsi_conn_error(conn->cls_conn, err);
}

static inline int
iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr)
{
	uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn);
	uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn);

	if (max_cmdsn < exp_cmdsn -1 &&
	    max_cmdsn > exp_cmdsn - INVALID_SN_DELTA)
		return ISCSI_ERR_MAX_CMDSN;
	if (max_cmdsn > session->max_cmdsn ||
	    max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA)
		session->max_cmdsn = max_cmdsn;
	if (exp_cmdsn > session->exp_cmdsn ||
	    exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA)
		session->exp_cmdsn = exp_cmdsn;

	return 0;
}

static inline int
iscsi_hdr_extract(struct iscsi_conn *conn)
{
	struct sk_buff *skb = conn->in.skb;

	if (conn->in.copy >= conn->hdr_size &&
	    conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
		/*
		 * Zero-copy PDU Header: using connection context
		 * to store header pointer.
		 */
		if (skb_shinfo(skb)->frag_list == NULL &&
		    !skb_shinfo(skb)->nr_frags)
			conn->in.hdr = (struct iscsi_hdr *)
				((char*)skb->data + conn->in.offset);
		else {
			/* ignoring return code since we checked
			 * in.copy before */
			skb_copy_bits(skb, conn->in.offset,
				&conn->hdr, conn->hdr_size);
			conn->in.hdr = &conn->hdr;
		}
		conn->in.offset += conn->hdr_size;
		conn->in.copy -= conn->hdr_size;
	} else {
		int hdr_remains;
		int copylen;

		/*
		 * PDU header scattered across SKB's,
		 * copying it... This'll happen quite rarely.
		 */

		if (conn->in_progress == IN_PROGRESS_WAIT_HEADER)
			conn->in.hdr_offset = 0;

		hdr_remains = conn->hdr_size - conn->in.hdr_offset;
		BUG_ON(hdr_remains <= 0);

		copylen = min(conn->in.copy, hdr_remains);
		skb_copy_bits(skb, conn->in.offset,
			(char*)&conn->hdr + conn->in.hdr_offset, copylen);

		debug_tcp("PDU gather offset %d bytes %d in.offset %d "
		       "in.copy %d\n", conn->in.hdr_offset, copylen,
		       conn->in.offset, conn->in.copy);

		conn->in.offset += copylen;
		conn->in.copy -= copylen;
		if (copylen < hdr_remains)  {
			conn->in_progress = IN_PROGRESS_HEADER_GATHER;
			conn->in.hdr_offset += copylen;
		        return -EAGAIN;
		}
		conn->in.hdr = &conn->hdr;
		conn->discontiguous_hdr_cnt++;
	        conn->in_progress = IN_PROGRESS_WAIT_HEADER;
	}

	return 0;
}

static inline void
iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
{
	struct scsi_cmnd *sc = ctask->sc;
	struct iscsi_session *session = conn->session;

	spin_lock(&session->lock);
	if (unlikely(!sc)) {
		spin_unlock(&session->lock);
		return;
	}
	if (sc->sc_data_direction == DMA_TO_DEVICE) {
		struct iscsi_data_task *dtask, *n;
		/* WRITE: cleanup Data-Out's if any */
		list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
			list_del(&dtask->item);
			mempool_free(dtask, ctask->datapool);
		}
	}
	ctask->xmstate = XMSTATE_IDLE;
	ctask->r2t = NULL;
	ctask->sc = NULL;
	__kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
	spin_unlock(&session->lock);
}

/**
 * iscsi_cmd_rsp - SCSI Command Response processing
 * @conn: iscsi connection
 * @ctask: scsi command task
 **/
static int
iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
{
	int rc;
	struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr;
	struct iscsi_session *session = conn->session;
	struct scsi_cmnd *sc = ctask->sc;

	rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
	if (rc) {
		sc->result = (DID_ERROR << 16);
		goto out;
	}

	conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;

	sc->result = (DID_OK << 16) | rhdr->cmd_status;

	if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) {
		sc->result = (DID_ERROR << 16);
		goto out;
	}

	if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) {
		int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE);

		memcpy(sc->sense_buffer, conn->data + 2, sensecopy);
		debug_scsi("copied %d bytes of sense\n", sensecopy);
	}

	if (sc->sc_data_direction == DMA_TO_DEVICE)
		goto out;

	if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
		int res_count = be32_to_cpu(rhdr->residual_count);

		if (res_count > 0 && res_count <= sc->request_bufflen)
			sc->resid = res_count;
		else
			sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
	} else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
		sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
	else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW)
		sc->resid = be32_to_cpu(rhdr->residual_count);

out:
	debug_scsi("done [sc %lx res %d itt 0x%x]\n",
		   (long)sc, sc->result, ctask->itt);
	conn->scsirsp_pdus_cnt++;
	iscsi_ctask_cleanup(conn, ctask);
	sc->scsi_done(sc);
	return rc;
}

/**
 * iscsi_data_rsp - SCSI Data-In Response processing
 * @conn: iscsi connection
 * @ctask: scsi command task
 **/
static int
iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
{
	int rc;
	struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr;
	struct iscsi_session *session = conn->session;
	int datasn = be32_to_cpu(rhdr->datasn);

	rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
	if (rc)
		return rc;
	/*
	 * setup Data-In byte counter (gets decremented..)
	 */
	ctask->data_count = conn->in.datalen;

	if (conn->in.datalen == 0)
		return 0;

	if (ctask->datasn != datasn)
		return ISCSI_ERR_DATASN;

	ctask->datasn++;

	ctask->data_offset = be32_to_cpu(rhdr->offset);
	if (ctask->data_offset + conn->in.datalen > ctask->total_length)
		return ISCSI_ERR_DATA_OFFSET;

	if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
		struct scsi_cmnd *sc = ctask->sc;

		conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
		if (rhdr->flags & ISCSI_FLAG_DATA_UNDERFLOW) {
			int res_count = be32_to_cpu(rhdr->residual_count);

			if (res_count > 0 &&
			    res_count <= sc->request_bufflen) {
				sc->resid = res_count;
				sc->result = (DID_OK << 16) | rhdr->cmd_status;
			} else
				sc->result = (DID_BAD_TARGET << 16) |
					rhdr->cmd_status;
		} else if (rhdr->flags & ISCSI_FLAG_DATA_OVERFLOW) {
			sc->resid = be32_to_cpu(rhdr->residual_count);
			sc->result = (DID_OK << 16) | rhdr->cmd_status;
		} else
			sc->result = (DID_OK << 16) | rhdr->cmd_status;
	}

	conn->datain_pdus_cnt++;
	return 0;
}

/**
 * iscsi_solicit_data_init - initialize first Data-Out
 * @conn: iscsi connection
 * @ctask: scsi command task
 * @r2t: R2T info
 *
 * Notes:
 *	Initialize first Data-Out within this R2T sequence and finds
 *	proper data_offset within this SCSI command.
 *
 *	This function is called with connection lock taken.
 **/
static void
iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
			struct iscsi_r2t_info *r2t)
{
	struct iscsi_data *hdr;
	struct iscsi_data_task *dtask;
	struct scsi_cmnd *sc = ctask->sc;

	dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
	BUG_ON(!dtask);
	hdr = &dtask->hdr;
	memset(hdr, 0, sizeof(struct iscsi_data));
	hdr->ttt = r2t->ttt;
	hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
	r2t->solicit_datasn++;
	hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
	memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
	hdr->itt = ctask->hdr.itt;
	hdr->exp_statsn = r2t->exp_statsn;
	hdr->offset = cpu_to_be32(r2t->data_offset);
	if (r2t->data_length > conn->max_xmit_dlength) {
		hton24(hdr->dlength, conn->max_xmit_dlength);
		r2t->data_count = conn->max_xmit_dlength;
		hdr->flags = 0;
	} else {
		hton24(hdr->dlength, r2t->data_length);
		r2t->data_count = r2t->data_length;
		hdr->flags = ISCSI_FLAG_CMD_FINAL;
	}
	conn->dataout_pdus_cnt++;

	r2t->sent = 0;

	iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr,
			   sizeof(struct iscsi_hdr));

	r2t->dtask = dtask;

	if (sc->use_sg) {
		int i, sg_count = 0;
		struct scatterlist *sg = sc->request_buffer;

		r2t->sg = NULL;
		for (i = 0; i < sc->use_sg; i++, sg += 1) {
			/* FIXME: prefetch ? */
			if (sg_count + sg->length > r2t->data_offset) {
				int page_offset;

				/* sg page found! */

				/* offset within this page */
				page_offset = r2t->data_offset - sg_count;

				/* fill in this buffer */
				iscsi_buf_init_sg(&r2t->sendbuf, sg);
				r2t->sendbuf.sg.offset += page_offset;
				r2t->sendbuf.sg.length -= page_offset;

				/* xmit logic will continue with next one */
				r2t->sg = sg + 1;
				break;
			}
			sg_count += sg->length;
		}
		BUG_ON(r2t->sg == NULL);
	} else
		iscsi_buf_init_iov(&ctask->sendbuf,
			    (char*)sc->request_buffer + r2t->data_offset,
			    r2t->data_count);

	list_add(&dtask->item, &ctask->dataqueue);
}

/**
 * iscsi_r2t_rsp - iSCSI R2T Response processing
 * @conn: iscsi connection
 * @ctask: scsi command task
 **/
static int
iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
{
	struct iscsi_r2t_info *r2t;
	struct iscsi_session *session = conn->session;
	struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr;
	int r2tsn = be32_to_cpu(rhdr->r2tsn);
	int rc;

	if (conn->in.ahslen)
		return ISCSI_ERR_AHSLEN;

	if (conn->in.datalen)
		return ISCSI_ERR_DATALEN;

	if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn)
		return ISCSI_ERR_R2TSN;

	rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
	if (rc)
		return rc;

	/* FIXME: use R2TSN to detect missing R2T */

	/* fill-in new R2T associated with the task */
	spin_lock(&session->lock);
	if (!ctask->sc || ctask->mtask ||
	     session->state != ISCSI_STATE_LOGGED_IN) {
		printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
		       "recovery...\n", ctask->itt);
		spin_unlock(&session->lock);
		return 0;
	}
	rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
	BUG_ON(!rc);

	r2t->exp_statsn = rhdr->statsn;
	r2t->data_length = be32_to_cpu(rhdr->data_length);
	if (r2t->data_length == 0 ||
	    r2t->data_length > session->max_burst) {
		spin_unlock(&session->lock);
		return ISCSI_ERR_DATALEN;
	}

	r2t->data_offset = be32_to_cpu(rhdr->data_offset);
	if (r2t->data_offset + r2t->data_length > ctask->total_length) {
		spin_unlock(&session->lock);
		return ISCSI_ERR_DATALEN;
	}

	r2t->ttt = rhdr->ttt; /* no flip */
	r2t->solicit_datasn = 0;

	iscsi_solicit_data_init(conn, ctask, r2t);

	ctask->exp_r2tsn = r2tsn + 1;
	ctask->xmstate |= XMSTATE_SOL_HDR;
	__kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*));
	__kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*));

	scsi_queue_work(session->host, &conn->xmitwork);
	conn->r2t_pdus_cnt++;
	spin_unlock(&session->lock);

	return 0;
}

static int
iscsi_hdr_recv(struct iscsi_conn *conn)
{
	int rc = 0;
	struct iscsi_hdr *hdr;
	struct iscsi_cmd_task *ctask;
	struct iscsi_session *session = conn->session;
	uint32_t cdgst, rdgst = 0;

	hdr = conn->in.hdr;

	/* verify PDU length */
	conn->in.datalen = ntoh24(hdr->dlength);
	if (conn->in.datalen > conn->max_recv_dlength) {
		printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
		       conn->in.datalen, conn->max_recv_dlength);
		return ISCSI_ERR_DATALEN;
	}
	conn->data_copied = 0;

	/* read AHS */
	conn->in.ahslen = hdr->hlength * 4;
	conn->in.offset += conn->in.ahslen;
	conn->in.copy -= conn->in.ahslen;
	if (conn->in.copy < 0) {
		printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
		       "%d bytes\n", conn->in.ahslen);
		return ISCSI_ERR_AHSLEN;
	}

	/* calculate read padding */
	conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1);
	if (conn->in.padding) {
		conn->in.padding = ISCSI_PAD_LEN - conn->in.padding;
		debug_scsi("read padding %d bytes\n", conn->in.padding);
	}

	if (conn->hdrdgst_en) {
		struct scatterlist sg;

		sg_init_one(&sg, (u8 *)hdr,
			    sizeof(struct iscsi_hdr) + conn->in.ahslen);
		crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
		rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
				     conn->in.ahslen);
		if (cdgst != rdgst) {
			printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error "
			       "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst,
			       cdgst);
			return ISCSI_ERR_HDR_DGST;
		}
	}

	/* save opcode for later */
	conn->in.opcode = hdr->opcode & ISCSI_OPCODE_MASK;

	/* verify itt (itt encoding: age+cid+itt) */
	if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
		if ((hdr->itt & AGE_MASK) !=
				(session->age << AGE_SHIFT)) {
			printk(KERN_ERR "iscsi_tcp: received itt %x expected "
				"session age (%x)\n", hdr->itt,
				session->age & AGE_MASK);
			return ISCSI_ERR_BAD_ITT;
		}

		if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) {
			printk(KERN_ERR "iscsi_tcp: received itt %x, expected "
				"CID (%x)\n", hdr->itt, conn->id);
			return ISCSI_ERR_BAD_ITT;
		}
		conn->in.itt = hdr->itt & ITT_MASK;
	} else
		conn->in.itt = hdr->itt;

	debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
		  hdr->opcode, conn->in.offset, conn->in.copy,
		  conn->in.ahslen, conn->in.datalen);

	if (conn->in.itt < session->cmds_max) {
		ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt];

		if (!ctask->sc) {
			printk(KERN_INFO "iscsi_tcp: dropping ctask with "
			       "itt 0x%x\n", ctask->itt);
			conn->in.datalen = 0; /* force drop */
			return 0;
		}

		if (ctask->sc->SCp.phase != session->age) {
			printk(KERN_ERR "iscsi_tcp: ctask's session age %d, "
				"expected %d\n", ctask->sc->SCp.phase,
				session->age);
			return ISCSI_ERR_SESSION_FAILED;
		}

		conn->in.ctask = ctask;

		debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n",
			   hdr->opcode, conn->id, (long)ctask->sc,
			   ctask->itt, conn->in.datalen);

		switch(conn->in.opcode) {
		case ISCSI_OP_SCSI_CMD_RSP:
			BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
			if (!conn->in.datalen)
				rc = iscsi_cmd_rsp(conn, ctask);
			else
				/*
				 * got sense or response data; copying PDU
				 * Header to the connection's header
				 * placeholder
				 */
				memcpy(&conn->hdr, hdr,
				       sizeof(struct iscsi_hdr));
			break;
		case ISCSI_OP_SCSI_DATA_IN:
			BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
			/* save flags for non-exceptional status */
			conn->in.flags = hdr->flags;
			/* save cmd_status for sense data */
			conn->in.cmd_status =
				((struct iscsi_data_rsp*)hdr)->cmd_status;
			rc = iscsi_data_rsp(conn, ctask);
			break;
		case ISCSI_OP_R2T:
			BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
			if (ctask->sc->sc_data_direction == DMA_TO_DEVICE)
				rc = iscsi_r2t_rsp(conn, ctask);
			else
				rc = ISCSI_ERR_PROTO;
			break;
		default:
			rc = ISCSI_ERR_BAD_OPCODE;
			break;
		}
	} else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET &&
		   conn->in.itt < ISCSI_MGMT_ITT_OFFSET +
					session->mgmtpool_max) {
		struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *)
					session->mgmt_cmds[conn->in.itt -
						ISCSI_MGMT_ITT_OFFSET];

		debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n",
			   conn->in.opcode, conn->id, mtask->itt,
			   conn->in.datalen);

		switch(conn->in.opcode) {
		case ISCSI_OP_LOGIN_RSP:
		case ISCSI_OP_TEXT_RSP:
		case ISCSI_OP_LOGOUT_RSP:
			rc = iscsi_check_assign_cmdsn(session,
						 (struct iscsi_nopin*)hdr);
			if (rc)
				break;

			if (!conn->in.datalen) {
				rc = iscsi_recv_pdu(conn->cls_conn, hdr,
						    NULL, 0);
				if (conn->login_mtask != mtask) {
					spin_lock(&session->lock);
					__kfifo_put(session->mgmtpool.queue,
					    (void*)&mtask, sizeof(void*));
					spin_unlock(&session->lock);
				}
			}
			break;
		case ISCSI_OP_SCSI_TMFUNC_RSP:
			rc = iscsi_check_assign_cmdsn(session,
						 (struct iscsi_nopin*)hdr);
			if (rc)
				break;

			if (conn->in.datalen || conn->in.ahslen) {
				rc = ISCSI_ERR_PROTO;
				break;
			}
			conn->tmfrsp_pdus_cnt++;
			spin_lock(&session->lock);
			if (conn->tmabort_state == TMABORT_INITIAL) {
				__kfifo_put(session->mgmtpool.queue,
						(void*)&mtask, sizeof(void*));
				conn->tmabort_state =
					((struct iscsi_tm_rsp *)hdr)->
					response == ISCSI_TMF_RSP_COMPLETE ?
						TMABORT_SUCCESS:TMABORT_FAILED;
				/* unblock eh_abort() */
				wake_up(&conn->ehwait);
			}
			spin_unlock(&session->lock);
			break;
		case ISCSI_OP_NOOP_IN:
			if (hdr->ttt != ISCSI_RESERVED_TAG) {
				rc = ISCSI_ERR_PROTO;
				break;
			}
			rc = iscsi_check_assign_cmdsn(session,
						(struct iscsi_nopin*)hdr);
			if (rc)
				break;
			conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;

			if (!conn->in.datalen) {
				struct iscsi_mgmt_task *mtask;

				rc = iscsi_recv_pdu(conn->cls_conn, hdr,
						    NULL, 0);
				mtask = (struct iscsi_mgmt_task *)
					session->mgmt_cmds[conn->in.itt -
							ISCSI_MGMT_ITT_OFFSET];
				if (conn->login_mtask != mtask) {
					spin_lock(&session->lock);
					__kfifo_put(session->mgmtpool.queue,
						  (void*)&mtask, sizeof(void*));
					spin_unlock(&session->lock);
				}
			}
			break;
		default:
			rc = ISCSI_ERR_BAD_OPCODE;
			break;
		}
	} else if (conn->in.itt == ISCSI_RESERVED_TAG) {
		switch(conn->in.opcode) {
		case ISCSI_OP_NOOP_IN:
			if (!conn->in.datalen) {
				rc = iscsi_check_assign_cmdsn(session,
						 (struct iscsi_nopin*)hdr);
				if (!rc && hdr->ttt != ISCSI_RESERVED_TAG)
					rc = iscsi_recv_pdu(conn->cls_conn,
							    hdr, NULL, 0);
			} else
				rc = ISCSI_ERR_PROTO;
			break;
		case ISCSI_OP_REJECT:
			/* we need sth like iscsi_reject_rsp()*/
		case ISCSI_OP_ASYNC_EVENT:
			/* we need sth like iscsi_async_event_rsp() */
			rc = ISCSI_ERR_BAD_OPCODE;
			break;
		default:
			rc = ISCSI_ERR_BAD_OPCODE;
			break;
		}
	} else
		rc = ISCSI_ERR_BAD_ITT;

	return rc;
}

/**
 * iscsi_ctask_copy - copy skb bits to the destanation cmd task
 * @conn: iscsi connection
 * @ctask: scsi command task
 * @buf: buffer to copy to
 * @buf_size: size of buffer
 * @offset: offset within the buffer
 *
 * Notes:
 *	The function calls skb_copy_bits() and updates per-connection and
 *	per-cmd byte counters.
 *
 *	Read counters (in bytes):
 *
 *	conn->in.offset		offset within in progress SKB
 *	conn->in.copy		left to copy from in progress SKB
 *				including padding
 *	conn->in.copied		copied already from in progress SKB
 *	conn->data_copied	copied already from in progress buffer
 *	ctask->sent		total bytes sent up to the MidLayer
 *	ctask->data_count	left to copy from in progress Data-In
 *	buf_left		left to copy from in progress buffer
 **/
static inline int
iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
		void *buf, int buf_size, int offset)
{
	int buf_left = buf_size - (conn->data_copied + offset);
	int size = min(conn->in.copy, buf_left);
	int rc;

	size = min(size, ctask->data_count);

	debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
	       size, conn->in.offset, conn->in.copied);

	BUG_ON(size <= 0);
	BUG_ON(ctask->sent + size > ctask->total_length);

	rc = skb_copy_bits(conn->in.skb, conn->in.offset,
			   (char*)buf + (offset + conn->data_copied), size);
	/* must fit into skb->len */
	BUG_ON(rc);

	conn->in.offset += size;
	conn->in.copy -= size;
	conn->in.copied += size;
	conn->data_copied += size;
	ctask->sent += size;
	ctask->data_count -= size;

	BUG_ON(conn->in.copy < 0);
	BUG_ON(ctask->data_count < 0);

	if (buf_size != (conn->data_copied + offset)) {
		if (!ctask->data_count) {
			BUG_ON(buf_size - conn->data_copied < 0);
			/* done with this PDU */
			return buf_size - conn->data_copied;
		}
		return -EAGAIN;
	}

	/* done with this buffer or with both - PDU and buffer */
	conn->data_copied = 0;
	return 0;
}

/**
 * iscsi_tcp_copy - copy skb bits to the destanation buffer
 * @conn: iscsi connection
 * @buf: buffer to copy to
 * @buf_size: number of bytes to copy
 *
 * Notes:
 *	The function calls skb_copy_bits() and updates per-connection
 *	byte counters.
 **/
static inline int
iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size)
{
	int buf_left = buf_size - conn->data_copied;
	int size = min(conn->in.copy, buf_left);
	int rc;

	debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
	       size, conn->in.offset, conn->data_copied);
	BUG_ON(size <= 0);

	rc = skb_copy_bits(conn->in.skb, conn->in.offset,
			   (char*)buf + conn->data_copied, size);
	BUG_ON(rc);

	conn->in.offset += size;
	conn->in.copy -= size;
	conn->in.copied += size;
	conn->data_copied += size;

	if (buf_size != conn->data_copied)
		return -EAGAIN;

	return 0;
}

static inline void
partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg,
			 int offset, int length)
{
	struct scatterlist temp;

	memcpy(&temp, sg, sizeof(struct scatterlist));
	temp.offset = offset;
	temp.length = length;
	crypto_digest_update(conn->data_rx_tfm, &temp, 1);
}

static void
iscsi_recv_digest_update(struct iscsi_conn *conn, char* buf, int len)
{
	struct scatterlist tmp;

	sg_init_one(&tmp, buf, len);
	crypto_digest_update(conn->data_rx_tfm, &tmp, 1);
}

static int iscsi_scsi_data_in(struct iscsi_conn *conn)
{
	struct iscsi_cmd_task *ctask = conn->in.ctask;
	struct scsi_cmnd *sc = ctask->sc;
	struct scatterlist *sg;
	int i, offset, rc = 0;

	BUG_ON((void*)ctask != sc->SCp.ptr);

	/*
	 * copying Data-In into the Scsi_Cmnd
	 */
	if (!sc->use_sg) {
		i = ctask->data_count;
		rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer,
				      sc->request_bufflen, ctask->data_offset);
		if (rc == -EAGAIN)
			return rc;
		if (conn->datadgst_en)
			iscsi_recv_digest_update(conn, sc->request_buffer, i);
		rc = 0;
		goto done;
	}

	offset = ctask->data_offset;
	sg = sc->request_buffer;

	if (ctask->data_offset)
		for (i = 0; i < ctask->sg_count; i++)
			offset -= sg[i].length;
	/* we've passed through partial sg*/
	if (offset < 0)
		offset = 0;

	for (i = ctask->sg_count; i < sc->use_sg; i++) {
		char *dest;

		dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
		rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset,
				      sg[i].length, offset);
		kunmap_atomic(dest, KM_SOFTIRQ0);
		if (rc == -EAGAIN)
			/* continue with the next SKB/PDU */
			return rc;
		if (!rc) {
			if (conn->datadgst_en) {
				if (!offset)
					crypto_digest_update(conn->data_rx_tfm,
							     &sg[i], 1);
				else
					partial_sg_digest_update(conn, &sg[i],
							sg[i].offset + offset,
							sg[i].length - offset);
			}
			offset = 0;
			ctask->sg_count++;
		}

		if (!ctask->data_count) {
			if (rc && conn->datadgst_en)
				/*
				 * data-in is complete, but buffer not...
				 */
				partial_sg_digest_update(conn, &sg[i],
						sg[i].offset, sg[i].length-rc);
			rc = 0;
			break;
		}

		if (!conn->in.copy)
			return -EAGAIN;
	}
	BUG_ON(ctask->data_count);

done:
	/* check for non-exceptional status */
	if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) {
		debug_scsi("done [sc %lx res %d itt 0x%x]\n",
			   (long)sc, sc->result, ctask->itt);
		conn->scsirsp_pdus_cnt++;
		iscsi_ctask_cleanup(conn, ctask);
		sc->scsi_done(sc);
	}

	return rc;
}

static int
iscsi_data_recv(struct iscsi_conn *conn)
{
	struct iscsi_session *session = conn->session;
	int rc = 0;