Author: kadesai
Date: Wed May 6 10:41:27 2015
New Revision: 282527
URL: https://svnweb.freebsd.org/changeset/base/282527
Log:
Current driver does fast path read load balancing between arm and mirror disk
for two Drive Raid-1 configuration only.
Now, Driver support fast path read load balancing for all (any number of
disk) Raid-1 configuration.
Reviewed by: ambrisko
MFC after: 2 weeks
Sponsored by: AVAGO Technologies
Modified:
head/sys/dev/mrsas/mrsas.c
head/sys/dev/mrsas/mrsas.h
head/sys/dev/mrsas/mrsas_cam.c
head/sys/dev/mrsas/mrsas_fp.c
Modified: head/sys/dev/mrsas/mrsas.c
==============================================================================
--- head/sys/dev/mrsas/mrsas.c Wed May 6 10:36:53 2015 (r282526)
+++ head/sys/dev/mrsas/mrsas.c Wed May 6 10:41:27 2015 (r282527)
@@ -462,6 +462,11 @@ mrsas_get_tunables(struct mrsas_softc *s
*/
TUNABLE_INT_FETCH("hw.mrsas.debug_level", &sc->mrsas_debug);
+ /*
+ * Grab the global variables.
+ */
+ TUNABLE_INT_FETCH("hw.mrsas.lb_pending_cmds", &sc->lb_pending_cmds);
+
/* Grab the unit-instance variables */
snprintf(tmpstr, sizeof(tmpstr), "dev.mrsas.%d.debug_level",
device_get_unit(sc->mrsas_dev));
@@ -1468,7 +1473,7 @@ mrsas_complete_cmd(struct mrsas_softc *s
MRSAS_RAID_SCSI_IO_REQUEST *scsi_io_req;
struct mrsas_mpt_cmd *cmd_mpt;
struct mrsas_mfi_cmd *cmd_mfi;
- u_int8_t arm, reply_descript_type;
+ u_int8_t reply_descript_type;
u_int16_t smid, num_completed;
u_int8_t status, extStatus;
union desc_value desc_val;
@@ -1506,8 +1511,7 @@ mrsas_complete_cmd(struct mrsas_softc *s
device_id = cmd_mpt->ccb_ptr->ccb_h.target_id;
lbinfo = &sc->load_balance_info[device_id];
if (cmd_mpt->load_balance == MRSAS_LOAD_BALANCE_FLAG) {
- arm = lbinfo->raid1DevHandle[0] ==
scsi_io_req->DevHandle ? 0 : 1;
-
mrsas_atomic_dec(&lbinfo->scsi_pending_cmds[arm]);
+
mrsas_atomic_dec(&lbinfo->scsi_pending_cmds[cmd_mpt->pd_r1_lb]);
cmd_mpt->load_balance &=
~MRSAS_LOAD_BALANCE_FLAG;
}
/* Fall thru and complete IO */
@@ -2331,6 +2335,7 @@ mrsas_ioc_init(struct mrsas_softc *sc)
init_frame->driver_ver_lo = (bus_addr_t)sc->verbuf_phys_addr;
init_frame->driver_ver_hi = 0;
}
+ init_frame->driver_operations.mfi_capabilities.support_ndrive_r1_lb = 1;
init_frame->driver_operations.mfi_capabilities.support_max_255lds = 1;
init_frame->driver_operations.mfi_capabilities.security_protocol_cmds_fw = 1;
phys_addr = (bus_addr_t)sc->ioc_init_phys_mem + 1024;
Modified: head/sys/dev/mrsas/mrsas.h
==============================================================================
--- head/sys/dev/mrsas/mrsas.h Wed May 6 10:36:53 2015 (r282526)
+++ head/sys/dev/mrsas/mrsas.h Wed May 6 10:41:27 2015 (r282527)
@@ -813,9 +813,8 @@ typedef struct _MR_DRV_RAID_MAP_ALL {
typedef struct _LD_LOAD_BALANCE_INFO {
u_int8_t loadBalanceFlag;
u_int8_t reserved1;
- u_int16_t raid1DevHandle[2];
- mrsas_atomic_t scsi_pending_cmds[2];
- u_int64_t last_accessed_block[2];
+ mrsas_atomic_t scsi_pending_cmds[MAX_PHYSICAL_DEVICES];
+ u_int64_t last_accessed_block[MAX_PHYSICAL_DEVICES];
} LD_LOAD_BALANCE_INFO, *PLD_LOAD_BALANCE_INFO;
/* SPAN_SET is info caclulated from span info from Raid map per ld */
@@ -858,6 +857,9 @@ struct IO_REQUEST_INFO {
u_int8_t start_span;
u_int8_t reserved;
u_int64_t start_row;
+ /* span[7:5], arm[4:0] */
+ u_int8_t span_arm;
+ u_int8_t pd_after_lb;
};
typedef struct _MR_LD_TARGET_SYNC {
@@ -1357,6 +1359,7 @@ struct mrsas_mpt_cmd {
u_int32_t sync_cmd_idx;
u_int32_t index;
u_int8_t flags;
+ u_int8_t pd_r1_lb;
u_int8_t load_balance;
bus_size_t length;
u_int32_t error_code;
@@ -2734,6 +2737,7 @@ struct mrsas_softc {
struct task ev_task;
u_int32_t CurLdCount;
u_int64_t reset_flags;
+ int lb_pending_cmds;
LD_LOAD_BALANCE_INFO load_balance_info[MAX_LOGICAL_DRIVES_EXT];
LD_SPAN_INFO log_to_span[MAX_LOGICAL_DRIVES_EXT];
Modified: head/sys/dev/mrsas/mrsas_cam.c
==============================================================================
--- head/sys/dev/mrsas/mrsas_cam.c Wed May 6 10:36:53 2015
(r282526)
+++ head/sys/dev/mrsas/mrsas_cam.c Wed May 6 10:41:27 2015
(r282527)
@@ -112,9 +112,8 @@ MR_BuildRaidContext(struct mrsas_softc *
extern u_int16_t
MR_LdSpanArrayGet(u_int32_t ld, u_int32_t span,
MR_DRV_RAID_MAP_ALL * map);
-extern u_int16_t
-mrsas_get_updated_dev_handle(PLD_LOAD_BALANCE_INFO lbInfo,
- struct IO_REQUEST_INFO *io_info);
+extern u_int16_t mrsas_get_updated_dev_handle(struct mrsas_softc *sc,
+ PLD_LOAD_BALANCE_INFO lbInfo, struct IO_REQUEST_INFO *io_info);
extern u_int8_t
megasas_get_best_arm(PLD_LOAD_BALANCE_INFO lbInfo, u_int8_t arm,
u_int64_t block, u_int32_t count);
@@ -824,9 +823,10 @@ mrsas_setup_io(struct mrsas_softc *sc, s
if ((sc->load_balance_info[device_id].loadBalanceFlag) &&
(io_info.isRead)) {
io_info.devHandle =
-
mrsas_get_updated_dev_handle(&sc->load_balance_info[device_id],
- &io_info);
+ mrsas_get_updated_dev_handle(sc,
+ &sc->load_balance_info[device_id],
&io_info);
cmd->load_balance = MRSAS_LOAD_BALANCE_FLAG;
+ cmd->pd_r1_lb = io_info.pd_after_lb;
} else
cmd->load_balance = 0;
cmd->request_desc->SCSIIO.DevHandle = io_info.devHandle;
Modified: head/sys/dev/mrsas/mrsas_fp.c
==============================================================================
--- head/sys/dev/mrsas/mrsas_fp.c Wed May 6 10:36:53 2015
(r282526)
+++ head/sys/dev/mrsas/mrsas_fp.c Wed May 6 10:41:27 2015
(r282527)
@@ -54,9 +54,8 @@ __FBSDID("$FreeBSD$");
* Function prototypes
*/
u_int8_t MR_ValidateMapInfo(struct mrsas_softc *sc);
-u_int8_t
-mrsas_get_best_arm(PLD_LOAD_BALANCE_INFO lbInfo, u_int8_t arm,
- u_int64_t block, u_int32_t count);
+u_int8_t mrsas_get_best_arm_pd(struct mrsas_softc *sc,
+ PLD_LOAD_BALANCE_INFO lbInfo, struct IO_REQUEST_INFO *io_info);
u_int8_t
MR_BuildRaidContext(struct mrsas_softc *sc,
struct IO_REQUEST_INFO *io_info,
@@ -69,17 +68,15 @@ MR_GetPhyParams(struct mrsas_softc *sc,
u_int16_t MR_TargetIdToLdGet(u_int32_t ldTgtId, MR_DRV_RAID_MAP_ALL * map);
u_int32_t MR_LdBlockSizeGet(u_int32_t ldTgtId, MR_DRV_RAID_MAP_ALL * map);
u_int16_t MR_GetLDTgtId(u_int32_t ld, MR_DRV_RAID_MAP_ALL * map);
-u_int16_t
-mrsas_get_updated_dev_handle(PLD_LOAD_BALANCE_INFO lbInfo,
- struct IO_REQUEST_INFO *io_info);
+u_int16_t mrsas_get_updated_dev_handle(struct mrsas_softc *sc,
+ PLD_LOAD_BALANCE_INFO lbInfo, struct IO_REQUEST_INFO *io_info);
u_int32_t mega_mod64(u_int64_t dividend, u_int32_t divisor);
u_int32_t
MR_GetSpanBlock(u_int32_t ld, u_int64_t row, u_int64_t *span_blk,
MR_DRV_RAID_MAP_ALL * map, int *div_error);
u_int64_t mega_div64_32(u_int64_t dividend, u_int32_t divisor);
-void
-mrsas_update_load_balance_params(MR_DRV_RAID_MAP_ALL * map,
- PLD_LOAD_BALANCE_INFO lbInfo);
+void mrsas_update_load_balance_params(struct mrsas_softc *sc,
+ MR_DRV_RAID_MAP_ALL *map, PLD_LOAD_BALANCE_INFO lbInfo);
void
mrsas_set_pd_lba(MRSAS_RAID_SCSI_IO_REQUEST * io_request,
u_int8_t cdb_len, struct IO_REQUEST_INFO *io_info, union ccb *ccb,
@@ -146,6 +143,8 @@ typedef u_int32_t REGION_LEN;
#define FALSE 0
#define TRUE 1
+#define LB_PENDING_CMDS_DEFAULT 4
+
/*
* Related Macros
@@ -379,7 +378,7 @@ MR_ValidateMapInfo(struct mrsas_softc *s
printf("Updating span set\n\n");
mr_update_span_set(drv_map, ldSpanInfo);
}
- mrsas_update_load_balance_params(drv_map, sc->load_balance_info);
+ mrsas_update_load_balance_params(sc, drv_map, sc->load_balance_info);
return 0;
}
@@ -723,9 +722,11 @@ get_arm(struct mrsas_softc *sc, u_int32_
* This routine calculates the arm, span and block for the specified stripe and
* reference in stripe using spanset
*
- * Inputs : Logical drive number
- * stripRow: Stripe number
- * stripRef: Reference in stripe
+ * Inputs :
+ * sc - HBA instance
+ * ld - Logical drive number
+ * stripRow: Stripe number
+ * stripRef: Reference in stripe
*
* Outputs : span - Span number block - Absolute Block
* number in the physical disk
@@ -785,6 +786,7 @@ mr_spanset_get_phy_params(struct mrsas_s
*pdBlock += stripRef + MR_LdSpanPtrGet(ld, span, map)->startBlk;
pRAID_Context->spanArm = (span << RAID_CTX_SPANARM_SPAN_SHIFT) |
physArm;
+ io_info->span_arm = pRAID_Context->spanArm;
return retval;
}
@@ -1097,46 +1099,39 @@ mr_update_span_set(MR_DRV_RAID_MAP_ALL *
/*
* mrsas_update_load_balance_params: Update load balance parmas
- * Inputs: map
pointer
- *
Load balance info
+ * Inputs:
+ * sc - driver softc instance
+ * drv_map - driver RAID map
+ * lbInfo - Load balance info
*
* This function updates the load balance parameters for the LD config of a two
* drive optimal RAID-1.
*/
void
-mrsas_update_load_balance_params(MR_DRV_RAID_MAP_ALL * map,
- PLD_LOAD_BALANCE_INFO lbInfo)
+mrsas_update_load_balance_params(struct mrsas_softc *sc,
+ MR_DRV_RAID_MAP_ALL * drv_map, PLD_LOAD_BALANCE_INFO lbInfo)
{
int ldCount;
u_int16_t ld;
- u_int32_t pd, arRef;
MR_LD_RAID *raid;
- for (ldCount = 0; ldCount < MAX_LOGICAL_DRIVES; ldCount++) {
- ld = MR_TargetIdToLdGet(ldCount, map);
- if (ld >= MAX_LOGICAL_DRIVES) {
+ if(sc->lb_pending_cmds > 128 || sc->lb_pending_cmds < 1)
+ sc-> lb_pending_cmds = LB_PENDING_CMDS_DEFAULT;
+
+ for (ldCount = 0; ldCount < MAX_LOGICAL_DRIVES_EXT; ldCount++) {
+ ld = MR_TargetIdToLdGet(ldCount, drv_map);
+ if (ld >= MAX_LOGICAL_DRIVES_EXT) {
lbInfo[ldCount].loadBalanceFlag = 0;
continue;
}
- raid = MR_LdRaidGet(ld, map);
- /* Two drive Optimal RAID 1 */
- if ((raid->level == 1) && (raid->rowSize == 2) &&
- (raid->spanDepth == 1)
- && raid->ldState == MR_LD_STATE_OPTIMAL) {
- lbInfo[ldCount].loadBalanceFlag = 1;
-
- /* Get the array on which this span is present */
- arRef = MR_LdSpanArrayGet(ld, 0, map);
-
- /* Get the PD */
- pd = MR_ArPdGet(arRef, 0, map);
- /* Get dev handle from PD */
- lbInfo[ldCount].raid1DevHandle[0] =
MR_PdDevHandleGet(pd, map);
- pd = MR_ArPdGet(arRef, 1, map);
- lbInfo[ldCount].raid1DevHandle[1] =
MR_PdDevHandleGet(pd, map);
- } else
+ raid = MR_LdRaidGet(ld, drv_map);
+ if ((raid->level != 1) ||
+ (raid->ldState != MR_LD_STATE_OPTIMAL)) {
lbInfo[ldCount].loadBalanceFlag = 0;
+ continue;
+ }
+ lbInfo[ldCount].loadBalanceFlag = 1;
}
}
@@ -1332,57 +1327,92 @@ mrsas_set_pd_lba(MRSAS_RAID_SCSI_IO_REQU
}
/*
- * mrsas_get_best_arm: Determine the best spindle arm
- * Inputs: Load balance info
+ * mrsas_get_best_arm_pd: Determine the best spindle arm
+ * Inputs:
+ * sc - HBA instance
+ * lbInfo - Load balance info
+ * io_info - IO request info
*
* This function determines and returns the best arm by looking at the
* parameters of the last PD access.
*/
-u_int8_t
-mrsas_get_best_arm(PLD_LOAD_BALANCE_INFO lbInfo, u_int8_t arm,
- u_int64_t block, u_int32_t count)
+u_int8_t mrsas_get_best_arm_pd(struct mrsas_softc *sc,
+ PLD_LOAD_BALANCE_INFO lbInfo, struct IO_REQUEST_INFO *io_info)
{
- u_int16_t pend0, pend1;
+ MR_LD_RAID *raid;
+ MR_DRV_RAID_MAP_ALL *drv_map;
+ u_int16_t pend0, pend1, ld;
u_int64_t diff0, diff1;
- u_int8_t bestArm;
+ u_int8_t bestArm, pd0, pd1, span, arm;
+ u_int32_t arRef, span_row_size;
+
+ u_int64_t block = io_info->ldStartBlock;
+ u_int32_t count = io_info->numBlocks;
+
+ span = ((io_info->span_arm & RAID_CTX_SPANARM_SPAN_MASK)
+ >> RAID_CTX_SPANARM_SPAN_SHIFT);
+ arm = (io_info->span_arm & RAID_CTX_SPANARM_ARM_MASK);
+
+ drv_map = sc->ld_drv_map[(sc->map_id & 1)];
+ ld = MR_TargetIdToLdGet(io_info->ldTgtId, drv_map);
+ raid = MR_LdRaidGet(ld, drv_map);
+ span_row_size = sc->UnevenSpanSupport ?
+ SPAN_ROW_SIZE(drv_map, ld, span) :
raid->rowSize;
+
+ arRef = MR_LdSpanArrayGet(ld, span, drv_map);
+ pd0 = MR_ArPdGet(arRef, arm, drv_map);
+ pd1 = MR_ArPdGet(arRef, (arm + 1) >= span_row_size ?
+ (arm + 1 - span_row_size): arm + 1, drv_map);
/* get the pending cmds for the data and mirror arms */
- pend0 = mrsas_atomic_read(&lbInfo->scsi_pending_cmds[0]);
- pend1 = mrsas_atomic_read(&lbInfo->scsi_pending_cmds[1]);
+ pend0 = mrsas_atomic_read(&lbInfo->scsi_pending_cmds[pd0]);
+ pend1 = mrsas_atomic_read(&lbInfo->scsi_pending_cmds[pd1]);
/* Determine the disk whose head is nearer to the req. block */
- diff0 = ABS_DIFF(block, lbInfo->last_accessed_block[0]);
- diff1 = ABS_DIFF(block, lbInfo->last_accessed_block[1]);
- bestArm = (diff0 <= diff1 ? 0 : 1);
+ diff0 = ABS_DIFF(block, lbInfo->last_accessed_block[pd0]);
+ diff1 = ABS_DIFF(block, lbInfo->last_accessed_block[pd1]);
+ bestArm = (diff0 <= diff1 ? arm : arm ^ 1);
- if ((bestArm == arm && pend0 > pend1 + 16) || (bestArm != arm && pend1
> pend0 + 16))
+ if ((bestArm == arm && pend0 > pend1 + sc->lb_pending_cmds) ||
+ (bestArm != arm && pend1 > pend0 + sc->lb_pending_cmds))
bestArm ^= 1;
/* Update the last accessed block on the correct pd */
- lbInfo->last_accessed_block[bestArm] = block + count - 1;
+ lbInfo->last_accessed_block[bestArm==arm ? pd0 : pd1] = block + count -
1;
+ io_info->span_arm = (span << RAID_CTX_SPANARM_SPAN_SHIFT) | bestArm;
+ io_info->pd_after_lb = (bestArm == arm) ? pd0:pd1;
+#if SPAN_DEBUG
+ if(arm != bestArm)
+ printf("LSI Debug R1 Load balance occur - span 0x%x arm 0x%x bestArm
0x%x "
+ "io_info->span_arm 0x%x\n",
+ span, arm, bestArm, io_info->span_arm);
+#endif
- return bestArm;
+ return io_info->pd_after_lb;
}
/*
* mrsas_get_updated_dev_handle: Get the update dev handle
- * Inputs: Load balance
info io_info pointer
+ * Inputs:
+ * sc - Adapter instance soft state
+ * lbInfo - Load balance info
+ * io_info - io_info pointer
*
* This function determines and returns the updated dev handle.
*/
-u_int16_t
-mrsas_get_updated_dev_handle(PLD_LOAD_BALANCE_INFO lbInfo,
- struct IO_REQUEST_INFO *io_info)
+u_int16_t mrsas_get_updated_dev_handle(struct mrsas_softc *sc,
+ PLD_LOAD_BALANCE_INFO lbInfo, struct IO_REQUEST_INFO
*io_info)
{
- u_int8_t arm, old_arm;
+ u_int8_t arm_pd;
u_int16_t devHandle;
+ MR_DRV_RAID_MAP_ALL *drv_map;
- old_arm = lbInfo->raid1DevHandle[0] == io_info->devHandle ? 0 : 1;
+ drv_map = sc->ld_drv_map[(sc->map_id & 1)];
/* get best new arm */
- arm = mrsas_get_best_arm(lbInfo, old_arm, io_info->ldStartBlock,
io_info->numBlocks);
- devHandle = lbInfo->raid1DevHandle[arm];
- mrsas_atomic_inc(&lbInfo->scsi_pending_cmds[arm]);
+ arm_pd = mrsas_get_best_arm_pd(sc, lbInfo, io_info);
+ devHandle = MR_PdDevHandleGet(arm_pd, drv_map);
+ mrsas_atomic_inc(&lbInfo->scsi_pending_cmds[arm_pd]);
return devHandle;
}
@@ -1471,6 +1501,7 @@ MR_GetPhyParams(struct mrsas_softc *sc,
*pdBlock += stripRef + MR_LdSpanPtrGet(ld, span, map)->startBlk;
pRAID_Context->spanArm = (span << RAID_CTX_SPANARM_SPAN_SHIFT) |
physArm;
+ io_info->span_arm = pRAID_Context->spanArm;
return retval;
}
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
