On 09/30/2014 12:11 PM, Mike Christie wrote:
> On 09/24/2014 03:01 AM, Jamie Clark wrote:
>> On machines running Ubuntu 12.04 open-iscsi initiator we have noticed
>> intermittent "pdu ... rejected due to DataDigest error. (data digests
>> configured and negotiated ok on initiator and target).
>>
>> After extensive wiresharking (the problem only affects something like 1
>> in 100000 write PDUs) I captured a failed write from the initiator and
>> found that the CRC32C value following the payload was incorrect, while
>> surrounding writes were OK.
>>
>> The target then rejects this bad pdu and follows that with an RST on the
>> TCP connection. This is expected behaviour. This seems to repeat about
>> every 10 minutes or so.
>>
>> The writes are all 4k blocks originating from iscsi-backed libvirt
>> managed KVM virtual machines. The particular failure I captured appeared
>> to be syslog data. The first 100 bytes or so of the 4k data was a syslog
>> entry, the remainder was zeroed out.
>>
>> Just wondering if digest calculation still has the potential failure
>> modes described back
>> here https://groups.google.com/d/msg/open-iscsi/OCYS5wUIYhc/Gno7edWCyH0J
>>
>
> What FS are you using and what kernel?
>
> The problem was fixed in the upper layers by default a while back for
> man file systems. Later, it was changed to a opt-in type of feature
> since it affects performance. iSCSI was not setting the correct bits to
> turn it on. I attached a patch for the current upstream kernel which
> should tell a FS to do stable pages when using iscsi data digests.
>
Actually try the attached patch instead of the last one. It might
improve performance too. It allows us to also do zero copy sends when
data digests are enabled.
--
You received this message because you are subscribed to the Google Groups
"open-iscsi" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
Visit this group at http://groups.google.com/group/open-iscsi.
For more options, visit https://groups.google.com/d/optout.
diff --git a/block/blk-settings.c b/block/blk-settings.c
index f1a1795..addc32b 100644
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -20,6 +20,27 @@ EXPORT_SYMBOL(blk_max_low_pfn);
unsigned long blk_max_pfn;
/**
+ * blk_queue_stable_pages_required - indicate stable writes are required
+ * @q: queue
+ */
+void blk_queue_stable_pages_required(struct request_queue *q)
+{
+ q->backing_dev_info.capabilities |= BDI_CAP_STABLE_WRITES;
+}
+EXPORT_SYMBOL(blk_queue_stable_pages_required);
+
+
+/**
+ * blk_queue_stable_pages_optional - indicate stable writes are not required
+ * @q: queue
+ */
+void blk_queue_stable_pages_optional(struct request_queue *q)
+{
+ q->backing_dev_info.capabilities &= ~BDI_CAP_STABLE_WRITES;
+}
+EXPORT_SYMBOL(blk_queue_stable_pages_optional);
+
+/**
* blk_queue_prep_rq - set a prepare_request function for queue
* @q: queue
* @pfn: prepare_request function
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index a669f2d..be11a3c 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -285,8 +285,9 @@ static int iscsi_sw_tcp_xmit_segment(struct iscsi_tcp_conn *tcp_conn,
if (!segment->data) {
sg = segment->sg;
offset += segment->sg_offset + sg->offset;
- r = tcp_sw_conn->sendpage(sk, sg_page(sg), offset,
- copy, flags);
+ r = tcp_sw_conn->sock->ops->sendpage(sk, sg_page(sg),
+ offset, copy,
+ flags);
} else {
struct msghdr msg = { .msg_flags = flags };
struct kvec iov = {
@@ -676,7 +677,6 @@ iscsi_sw_tcp_conn_bind(struct iscsi_cls_session *cls_session,
sk_set_memalloc(sk);
iscsi_sw_tcp_conn_set_callbacks(conn);
- tcp_sw_conn->sendpage = tcp_sw_conn->sock->ops->sendpage;
/*
* set receive state machine into initial state
*/
@@ -688,6 +688,23 @@ free_socket:
return err;
}
+
+static void iscsi_sw_tcp_stable_pages_optional(struct Scsi_Host *shost)
+{
+ struct scsi_device *sdev;
+
+ shost_for_each_device(sdev, shost)
+ blk_queue_stable_pages_optional(sdev->request_queue);
+}
+
+static void iscsi_sw_tcp_stable_pages_required(struct Scsi_Host *shost)
+{
+ struct scsi_device *sdev;
+
+ shost_for_each_device(sdev, shost)
+ blk_queue_stable_pages_required(sdev->request_queue);
+}
+
static int iscsi_sw_tcp_conn_set_param(struct iscsi_cls_conn *cls_conn,
enum iscsi_param param, char *buf,
int buflen)
@@ -702,8 +719,10 @@ static int iscsi_sw_tcp_conn_set_param(struct iscsi_cls_conn *cls_conn,
break;
case ISCSI_PARAM_DATADGST_EN:
iscsi_set_param(cls_conn, param, buf, buflen);
- tcp_sw_conn->sendpage = conn->datadgst_en ?
- sock_no_sendpage : tcp_sw_conn->sock->ops->sendpage;
+ if (conn->datadgst_en)
+ iscsi_sw_tcp_stable_pages_required(conn->session->host);
+ else
+ iscsi_sw_tcp_stable_pages_optional(conn->session->host);
break;
case ISCSI_PARAM_MAX_R2T:
return iscsi_tcp_set_max_r2t(conn, buf);
@@ -937,8 +956,17 @@ static int iscsi_sw_tcp_slave_alloc(struct scsi_device *sdev)
static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev)
{
+ struct iscsi_cls_session *cls_session =
+ starget_to_session(scsi_target(sdev));
+ struct iscsi_session *session = cls_session->dd_data;
+ struct iscsi_conn *conn = session->leadconn;
+
blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
blk_queue_dma_alignment(sdev->request_queue, 0);
+
+ if (conn->datadgst_en)
+ blk_queue_stable_pages_required(sdev->request_queue);
+
return 0;
}
diff --git a/drivers/scsi/iscsi_tcp.h b/drivers/scsi/iscsi_tcp.h
index f42ecb2..d2f2692 100644
--- a/drivers/scsi/iscsi_tcp.h
+++ b/drivers/scsi/iscsi_tcp.h
@@ -51,8 +51,6 @@ struct iscsi_sw_tcp_conn {
/* MIB custom statistics */
uint32_t sendpage_failures_cnt;
uint32_t discontiguous_hdr_cnt;
-
- ssize_t (*sendpage)(struct socket *, struct page *, int, size_t, int);
};
struct iscsi_sw_tcp_host {
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 518b465..c340bc9 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1048,6 +1048,8 @@ extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
extern void blk_queue_flush(struct request_queue *q, unsigned int flush);
extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev);
+extern void blk_queue_stable_pages_required(struct request_queue *q);
+extern void blk_queue_stable_pages_optional(struct request_queue *q);
extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
extern int blk_bio_map_sg(struct request_queue *q, struct bio *bio,
