Author: mav
Date: Mon Oct 21 12:00:26 2013
New Revision: 256843
URL: http://svnweb.freebsd.org/changeset/base/256843
Log:
Merge CAM locking changes from the projects/camlock branch to radically
reduce lock congestion and improve SMP scalability of the SCSI/ATA stack,
preparing the ground for the coming next GEOM direct dispatch support.
Replace big per-SIM locks with bunch of smaller ones:
- per-LUN locks to protect device and peripheral drivers state;
- per-target locks to protect list of LUNs on target;
- per-bus locks to protect reference counting;
- per-send queue locks to protect queue of CCBs to be sent;
- per-done queue locks to protect queue of completed CCBs;
- remaining per-SIM locks now protect only HBA driver internals.
While holding LUN lock it is allowed (while not recommended for performance
reasons) to take SIM lock. The opposite acquisition order is forbidden.
All the other locks are leaf locks, that can be taken anywhere, but should
not be cascaded. Many functions, such as: xpt_action(), xpt_done(),
xpt_async(), xpt_create_path(), etc. are no longer require (but allow) SIM
lock to be held.
To keep compatibility and solve cases where SIM lock can't be dropped, all
xpt_async() calls in addition to xpt_done() calls are queued to completion
threads for async processing in clean environment without SIM lock held.
Instead of single CAM SWI thread, used for commands completion processing
before, use multiple (depending on number of CPUs) threads. Load balanced
between them using "hash" of the device B:T:L address.
HBA drivers that can drop SIM lock during completion processing and have
sufficient number of completion threads to efficiently scale to multiple
CPUs can use new function xpt_done_direct() to avoid extra context switch.
Make ahci(4) driver to use this mechanism depending on hardware setup.
Sponsored by: iXsystems, Inc.
MFC after: 2 months
Modified:
head/sys/cam/ata/ata_da.c
head/sys/cam/ata/ata_pmp.c
head/sys/cam/ata/ata_xpt.c
head/sys/cam/cam_ccb.h
head/sys/cam/cam_periph.c
head/sys/cam/cam_periph.h
head/sys/cam/cam_queue.c
head/sys/cam/cam_queue.h
head/sys/cam/cam_sim.c
head/sys/cam/cam_sim.h
head/sys/cam/cam_xpt.c
head/sys/cam/cam_xpt.h
head/sys/cam/cam_xpt_internal.h
head/sys/cam/cam_xpt_sim.h
head/sys/cam/ctl/ctl_frontend_cam_sim.c
head/sys/cam/ctl/scsi_ctl.c
head/sys/cam/scsi/scsi_cd.c
head/sys/cam/scsi/scsi_ch.c
head/sys/cam/scsi/scsi_da.c
head/sys/cam/scsi/scsi_enc.c
head/sys/cam/scsi/scsi_enc_internal.h
head/sys/cam/scsi/scsi_enc_safte.c
head/sys/cam/scsi/scsi_enc_ses.c
head/sys/cam/scsi/scsi_pass.c
head/sys/cam/scsi/scsi_pt.c
head/sys/cam/scsi/scsi_sa.c
head/sys/cam/scsi/scsi_sg.c
head/sys/cam/scsi/scsi_targ_bh.c
head/sys/cam/scsi/scsi_target.c
head/sys/cam/scsi/scsi_xpt.c
head/sys/dev/ahci/ahci.c
head/sys/dev/ahci/ahci.h
head/sys/dev/ata/ata-all.c
head/sys/dev/isp/isp_freebsd.c
head/sys/dev/mvs/mvs.c
head/sys/dev/siis/siis.c
Modified: head/sys/cam/ata/ata_da.c
==============================================================================
--- head/sys/cam/ata/ata_da.c Mon Oct 21 10:09:48 2013 (r256842)
+++ head/sys/cam/ata/ata_da.c Mon Oct 21 12:00:26 2013 (r256843)
@@ -103,7 +103,6 @@ typedef enum {
ADA_CCB_RAHEAD = 0x01,
ADA_CCB_WCACHE = 0x02,
ADA_CCB_BUFFER_IO = 0x03,
- ADA_CCB_WAITING = 0x04,
ADA_CCB_DUMP = 0x05,
ADA_CCB_TRIM = 0x06,
ADA_CCB_TYPE_MASK = 0x0F,
@@ -154,6 +153,7 @@ struct ada_softc {
struct sysctl_oid *sysctl_tree;
struct callout sendordered_c;
struct trim_request trim_req;
+ int refcount;
};
struct ada_quirk_entry {
@@ -637,14 +637,8 @@ adaclose(struct disk *dp)
int error;
periph = (struct cam_periph *)dp->d_drv1;
- cam_periph_lock(periph);
- if (cam_periph_hold(periph, PRIBIO) != 0) {
- cam_periph_unlock(periph);
- cam_periph_release(periph);
- return (0);
- }
-
softc = (struct ada_softc *)periph->softc;
+ cam_periph_lock(periph);
CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
("adaclose\n"));
@@ -652,7 +646,8 @@ adaclose(struct disk *dp)
/* We only sync the cache if the drive is capable of it. */
if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
(softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
- (periph->flags & CAM_PERIPH_INVALID) == 0) {
+ (periph->flags & CAM_PERIPH_INVALID) == 0 &&
+ cam_periph_hold(periph, PRIBIO) == 0) {
ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
cam_fill_ataio(&ccb->ataio,
@@ -676,10 +671,13 @@ adaclose(struct disk *dp)
else
softc->flags &= ~ADA_FLAG_DIRTY;
xpt_release_ccb(ccb);
+ cam_periph_unhold(periph);
}
softc->flags &= ~ADA_FLAG_OPEN;
- cam_periph_unhold(periph);
+
+ while (softc->refcount != 0)
+ cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose",
1);
cam_periph_unlock(periph);
cam_periph_release(periph);
return (0);
@@ -689,23 +687,15 @@ static void
adaschedule(struct cam_periph *periph)
{
struct ada_softc *softc = (struct ada_softc *)periph->softc;
- uint32_t prio;
if (softc->state != ADA_STATE_NORMAL)
return;
- /* Check if cam_periph_getccb() was called. */
- prio = periph->immediate_priority;
-
/* Check if we have more work to do. */
if (bioq_first(&softc->bio_queue) ||
(!softc->trim_running && bioq_first(&softc->trim_queue))) {
- prio = CAM_PRIORITY_NORMAL;
+ xpt_schedule(periph, CAM_PRIORITY_NORMAL);
}
-
- /* Schedule CCB if any of above is true. */
- if (prio != CAM_PRIORITY_NONE)
- xpt_schedule(periph, prio);
}
/*
@@ -969,7 +959,7 @@ adaasync(void *callback_arg, u_int32_t c
status = cam_periph_alloc(adaregister, adaoninvalidate,
adacleanup, adastart,
"ada", CAM_PERIPH_BIO,
- cgd->ccb_h.path, adaasync,
+ path, adaasync,
AC_FOUND_DEVICE, cgd);
if (status != CAM_REQ_CMP
@@ -1045,8 +1035,10 @@ adaasync(void *callback_arg, u_int32_t c
softc->state = ADA_STATE_WCACHE;
else
break;
- cam_periph_acquire(periph);
- xpt_schedule(periph, CAM_PRIORITY_DEV);
+ if (cam_periph_acquire(periph) != CAM_REQ_CMP)
+ softc->state = ADA_STATE_NORMAL;
+ else
+ xpt_schedule(periph, CAM_PRIORITY_DEV);
}
default:
cam_periph_async(periph, code, path, arg);
@@ -1353,8 +1345,8 @@ adaregister(struct cam_periph *periph, v
* Create our sysctl variables, now that we know
* we have successfully attached.
*/
- cam_periph_acquire(periph);
- taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
+ if (cam_periph_acquire(periph) == CAM_REQ_CMP)
+ taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
/*
* Add async callbacks for bus reset and
@@ -1372,7 +1364,7 @@ adaregister(struct cam_periph *periph, v
* Schedule a periodic event to occasionally send an
* ordered tag to a device.
*/
- callout_init_mtx(&softc->sendordered_c, periph->sim->mtx, 0);
+ callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
callout_reset(&softc->sendordered_c,
(ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
adasendorderedtag, softc);
@@ -1380,16 +1372,17 @@ adaregister(struct cam_periph *periph, v
if (ADA_RA >= 0 &&
cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD) {
softc->state = ADA_STATE_RAHEAD;
- cam_periph_acquire(periph);
- xpt_schedule(periph, CAM_PRIORITY_DEV);
} else if (ADA_WC >= 0 &&
cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE) {
softc->state = ADA_STATE_WCACHE;
- cam_periph_acquire(periph);
- xpt_schedule(periph, CAM_PRIORITY_DEV);
- } else
+ } else {
softc->state = ADA_STATE_NORMAL;
-
+ return(CAM_REQ_CMP);
+ }
+ if (cam_periph_acquire(periph) != CAM_REQ_CMP)
+ softc->state = ADA_STATE_NORMAL;
+ else
+ xpt_schedule(periph, CAM_PRIORITY_DEV);
return(CAM_REQ_CMP);
}
@@ -1407,19 +1400,6 @@ adastart(struct cam_periph *periph, unio
struct bio *bp;
u_int8_t tag_code;
- /* Execute immediate CCB if waiting. */
- if (periph->immediate_priority <= periph->pinfo.priority) {
- CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE,
- ("queuing for immediate ccb\n"));
- start_ccb->ccb_h.ccb_state = ADA_CCB_WAITING;
- SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h,
- periph_links.sle);
- periph->immediate_priority = CAM_PRIORITY_NONE;
- wakeup(&periph->ccb_list);
- /* Have more work to do, so ensure we stay scheduled */
- adaschedule(periph);
- break;
- }
/* Run TRIM if not running yet. */
if (!softc->trim_running &&
(bp = bioq_first(&softc->trim_queue)) != 0) {
@@ -1494,6 +1474,7 @@ adastart(struct cam_periph *periph, unio
ATA_DSM_TRIM, 0, (ranges + ATA_DSM_BLK_RANGES -
1) / ATA_DSM_BLK_RANGES);
start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
+ start_ccb->ccb_h.flags |= CAM_UNLOCKED;
goto out;
}
/* Run regular command. */
@@ -1662,10 +1643,15 @@ adastart(struct cam_periph *periph, unio
break;
}
start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
+ start_ccb->ccb_h.flags |= CAM_UNLOCKED;
out:
start_ccb->ccb_h.ccb_bp = bp;
softc->outstanding_cmds++;
+ softc->refcount++;
+ cam_periph_unlock(periph);
xpt_action(start_ccb);
+ cam_periph_lock(periph);
+ softc->refcount--;
/* May have more work to do, so ensure we stay scheduled */
adaschedule(periph);
@@ -1674,13 +1660,6 @@ out:
case ADA_STATE_RAHEAD:
case ADA_STATE_WCACHE:
{
- if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
- softc->state = ADA_STATE_NORMAL;
- xpt_release_ccb(start_ccb);
- cam_periph_release_locked(periph);
- return;
- }
-
cam_fill_ataio(ataio,
1,
adadone,
@@ -1729,10 +1708,12 @@ adadone(struct cam_periph *periph, union
struct bio *bp;
int error;
+ cam_periph_lock(periph);
if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
error = adaerror(done_ccb, 0, 0);
if (error == ERESTART) {
/* A retry was scheduled, so just return. */
+ cam_periph_unlock(periph);
return;
}
if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
@@ -1762,6 +1743,7 @@ adadone(struct cam_periph *periph, union
softc->outstanding_cmds--;
if (softc->outstanding_cmds == 0)
softc->flags |= ADA_FLAG_WENT_IDLE;
+ xpt_release_ccb(done_ccb);
if (state == ADA_CCB_TRIM) {
TAILQ_HEAD(, bio) queue;
struct bio *bp1;
@@ -1769,6 +1751,8 @@ adadone(struct cam_periph *periph, union
TAILQ_INIT(&queue);
TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
softc->trim_running = 0;
+ adaschedule(periph);
+ cam_periph_unlock(periph);
while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
TAILQ_REMOVE(&queue, bp1, bio_queue);
bp1->bio_error = error;
@@ -1779,10 +1763,11 @@ adadone(struct cam_periph *periph, union
bp1->bio_resid = 0;
biodone(bp1);
}
- adaschedule(periph);
- } else
+ } else {
+ cam_periph_unlock(periph);
biodone(bp);
- break;
+ }
+ return;
}
case ADA_CCB_RAHEAD:
{
@@ -1858,12 +1843,6 @@ out:
cam_periph_release_locked(periph);
return;
}
- case ADA_CCB_WAITING:
- {
- /* Caller will release the CCB */
- wakeup(&done_ccb->ccb_h.cbfcnp);
- return;
- }
case ADA_CCB_DUMP:
/* No-op. We're polling */
return;
Modified: head/sys/cam/ata/ata_pmp.c
==============================================================================
--- head/sys/cam/ata/ata_pmp.c Mon Oct 21 10:09:48 2013 (r256842)
+++ head/sys/cam/ata/ata_pmp.c Mon Oct 21 12:00:26 2013 (r256843)
@@ -293,7 +293,7 @@ pmpasync(void *callback_arg, u_int32_t c
status = cam_periph_alloc(pmpregister, pmponinvalidate,
pmpcleanup, pmpstart,
"pmp", CAM_PERIPH_BIO,
- cgd->ccb_h.path, pmpasync,
+ path, pmpasync,
AC_FOUND_DEVICE, cgd);
if (status != CAM_REQ_CMP
@@ -318,13 +318,17 @@ pmpasync(void *callback_arg, u_int32_t c
if (code == AC_SENT_BDR || code == AC_BUS_RESET)
softc->found = 0; /* We have to reset everything. */
if (softc->state == PMP_STATE_NORMAL) {
- if (softc->pm_pid == 0x37261095 ||
- softc->pm_pid == 0x38261095)
- softc->state = PMP_STATE_PM_QUIRKS_1;
- else
- softc->state = PMP_STATE_PRECONFIG;
- cam_periph_acquire(periph);
- xpt_schedule(periph, CAM_PRIORITY_DEV);
+ if (cam_periph_acquire(periph) == CAM_REQ_CMP) {
+ if (softc->pm_pid == 0x37261095 ||
+ softc->pm_pid == 0x38261095)
+ softc->state = PMP_STATE_PM_QUIRKS_1;
+ else
+ softc->state = PMP_STATE_PRECONFIG;
+ xpt_schedule(periph, CAM_PRIORITY_DEV);
+ } else {
+ pmprelease(periph, softc->found);
+ xpt_release_boot();
+ }
} else
softc->restart = 1;
break;
Modified: head/sys/cam/ata/ata_xpt.c
==============================================================================
--- head/sys/cam/ata/ata_xpt.c Mon Oct 21 10:09:48 2013 (r256842)
+++ head/sys/cam/ata/ata_xpt.c Mon Oct 21 12:00:26 2013 (r256843)
@@ -182,7 +182,7 @@ static struct cam_ed *
static void ata_device_transport(struct cam_path *path);
static void ata_get_transfer_settings(struct ccb_trans_settings *cts);
static void ata_set_transfer_settings(struct ccb_trans_settings *cts,
- struct cam_ed *device,
+ struct cam_path *path,
int async_update);
static void ata_dev_async(u_int32_t async_code,
struct cam_eb *bus,
@@ -249,6 +249,7 @@ proberegister(struct cam_periph *periph,
return (status);
}
CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe started\n"));
+ ata_device_transport(periph->path);
probeschedule(periph);
return(CAM_REQ_CMP);
}
@@ -1320,6 +1321,7 @@ ata_scan_bus(struct cam_periph *periph,
struct cam_path *path;
ata_scan_bus_info *scan_info;
union ccb *work_ccb, *reset_ccb;
+ struct mtx *mtx;
cam_status status;
CAM_DEBUG(request_ccb->ccb_h.path, CAM_DEBUG_TRACE,
@@ -1395,11 +1397,14 @@ ata_scan_bus(struct cam_periph *periph,
xpt_done(request_ccb);
break;
}
+ mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path);
goto scan_next;
case XPT_SCAN_LUN:
work_ccb = request_ccb;
/* Reuse the same CCB to query if a device was really found */
scan_info = (ata_scan_bus_info *)work_ccb->ccb_h.ppriv_ptr0;
+ mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path);
+ mtx_lock(mtx);
/* If there is PMP... */
if ((scan_info->cpi->hba_inquiry & PI_SATAPM) &&
(scan_info->counter == scan_info->cpi->max_target)) {
@@ -1428,6 +1433,7 @@ ata_scan_bus(struct cam_periph *periph,
((scan_info->cpi->hba_inquiry & PI_SATAPM) ?
0 : scan_info->cpi->max_target)) {
done:
+ mtx_unlock(mtx);
xpt_free_ccb(work_ccb);
xpt_free_ccb((union ccb *)scan_info->cpi);
request_ccb = scan_info->request_ccb;
@@ -1444,6 +1450,8 @@ scan_next:
scan_info->request_ccb->ccb_h.path_id,
scan_info->counter, 0);
if (status != CAM_REQ_CMP) {
+ if (request_ccb->ccb_h.func_code == XPT_SCAN_LUN)
+ mtx_unlock(mtx);
printf("xpt_scan_bus: xpt_create_path failed"
" with status %#x, bus scan halted\n",
status);
@@ -1459,9 +1467,15 @@ scan_next:
scan_info->request_ccb->ccb_h.pinfo.priority);
work_ccb->ccb_h.func_code = XPT_SCAN_LUN;
work_ccb->ccb_h.cbfcnp = ata_scan_bus;
+ work_ccb->ccb_h.flags |= CAM_UNLOCKED;
work_ccb->ccb_h.ppriv_ptr0 = scan_info;
work_ccb->crcn.flags = scan_info->request_ccb->crcn.flags;
+ mtx_unlock(mtx);
+ if (request_ccb->ccb_h.func_code == XPT_SCAN_LUN)
+ mtx = NULL;
xpt_action(work_ccb);
+ if (mtx != NULL)
+ mtx_lock(mtx);
break;
default:
break;
@@ -1476,6 +1490,7 @@ ata_scan_lun(struct cam_periph *periph,
cam_status status;
struct cam_path *new_path;
struct cam_periph *old_periph;
+ int lock;
CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_scan_lun\n"));
@@ -1510,10 +1525,14 @@ ata_scan_lun(struct cam_periph *periph,
}
xpt_setup_ccb(&request_ccb->ccb_h, new_path, CAM_PRIORITY_XPT);
request_ccb->ccb_h.cbfcnp = xptscandone;
+ request_ccb->ccb_h.flags |= CAM_UNLOCKED;
request_ccb->ccb_h.func_code = XPT_SCAN_LUN;
request_ccb->crcn.flags = flags;
}
+ lock = (xpt_path_owned(path) == 0);
+ if (lock)
+ xpt_path_lock(path);
if ((old_periph = cam_periph_find(path, "aprobe")) != NULL) {
if ((old_periph->flags & CAM_PERIPH_INVALID) == 0) {
probe_softc *softc;
@@ -1540,6 +1559,8 @@ ata_scan_lun(struct cam_periph *periph,
xpt_done(request_ccb);
}
}
+ if (lock)
+ xpt_path_unlock(path);
}
static void
@@ -1553,7 +1574,6 @@ xptscandone(struct cam_periph *periph, u
static struct cam_ed *
ata_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
{
- struct cam_path path;
struct ata_quirk_entry *quirk;
struct cam_ed *device;
@@ -1574,22 +1594,6 @@ ata_alloc_device(struct cam_eb *bus, str
device->queue_flags = 0;
device->serial_num = NULL;
device->serial_num_len = 0;
-
- /*
- * XXX should be limited by number of CCBs this bus can
- * do.
- */
- bus->sim->max_ccbs += device->ccbq.devq_openings;
- if (lun_id != CAM_LUN_WILDCARD) {
- xpt_compile_path(&path,
- NULL,
- bus->path_id,
- target->target_id,
- lun_id);
- ata_device_transport(&path);
- xpt_release_path(&path);
- }
-
return (device);
}
@@ -1712,15 +1716,8 @@ ata_dev_advinfo(union ccb *start_ccb)
start_ccb->ccb_h.status = CAM_REQ_CMP;
if (cdai->flags & CDAI_FLAG_STORE) {
- int owned;
-
- owned = mtx_owned(start_ccb->ccb_h.path->bus->sim->mtx);
- if (owned == 0)
- mtx_lock(start_ccb->ccb_h.path->bus->sim->mtx);
xpt_async(AC_ADVINFO_CHANGED, start_ccb->ccb_h.path,
(void *)(uintptr_t)cdai->buftype);
- if (owned == 0)
- mtx_unlock(start_ccb->ccb_h.path->bus->sim->mtx);
}
}
@@ -1732,7 +1729,7 @@ ata_action(union ccb *start_ccb)
case XPT_SET_TRAN_SETTINGS:
{
ata_set_transfer_settings(&start_ccb->cts,
- start_ccb->ccb_h.path->device,
+ start_ccb->ccb_h.path,
/*async_update*/FALSE);
break;
}
@@ -1791,11 +1788,9 @@ ata_get_transfer_settings(struct ccb_tra
struct ccb_trans_settings_ata *ata;
struct ccb_trans_settings_scsi *scsi;
struct cam_ed *device;
- struct cam_sim *sim;
device = cts->ccb_h.path->device;
- sim = cts->ccb_h.path->bus->sim;
- (*(sim->sim_action))(sim, (union ccb *)cts);
+ xpt_action_default((union ccb *)cts);
if (cts->protocol == PROTO_UNKNOWN ||
cts->protocol == PROTO_UNSPECIFIED) {
@@ -1832,17 +1827,17 @@ ata_get_transfer_settings(struct ccb_tra
}
static void
-ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed
*device,
+ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_path
*path,
int async_update)
{
struct ccb_pathinq cpi;
struct ccb_trans_settings_ata *ata;
struct ccb_trans_settings_scsi *scsi;
- struct cam_sim *sim;
struct ata_params *ident_data;
struct scsi_inquiry_data *inq_data;
+ struct cam_ed *device;
- if (device == NULL) {
+ if (path == NULL || (device = path->device) == NULL) {
cts->ccb_h.status = CAM_PATH_INVALID;
xpt_done((union ccb *)cts);
return;
@@ -1859,14 +1854,14 @@ ata_set_transfer_settings(struct ccb_tra
cts->protocol_version = device->protocol_version;
if (cts->protocol != device->protocol) {
- xpt_print(cts->ccb_h.path, "Uninitialized Protocol %x:%x?\n",
+ xpt_print(path, "Uninitialized Protocol %x:%x?\n",
cts->protocol, device->protocol);
cts->protocol = device->protocol;
}
if (cts->protocol_version > device->protocol_version) {
if (bootverbose) {
- xpt_print(cts->ccb_h.path, "Down reving Protocol "
+ xpt_print(path, "Down reving Protocol "
"Version from %d to %d?\n", cts->protocol_version,
device->protocol_version);
}
@@ -1884,21 +1879,20 @@ ata_set_transfer_settings(struct ccb_tra
cts->transport_version = device->transport_version;
if (cts->transport != device->transport) {
- xpt_print(cts->ccb_h.path, "Uninitialized Transport %x:%x?\n",
+ xpt_print(path, "Uninitialized Transport %x:%x?\n",
cts->transport, device->transport);
cts->transport = device->transport;
}
if (cts->transport_version > device->transport_version) {
if (bootverbose) {
- xpt_print(cts->ccb_h.path, "Down reving Transport "
+ xpt_print(path, "Down reving Transport "
"Version from %d to %d?\n", cts->transport_version,
device->transport_version);
}
cts->transport_version = device->transport_version;
}
- sim = cts->ccb_h.path->bus->sim;
ident_data = &device->ident_data;
inq_data = &device->inq_data;
if (cts->protocol == PROTO_ATA)
@@ -1909,7 +1903,7 @@ ata_set_transfer_settings(struct ccb_tra
scsi = &cts->proto_specific.scsi;
else
scsi = NULL;
- xpt_setup_ccb(&cpi.ccb_h, cts->ccb_h.path, CAM_PRIORITY_NONE);
+ xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE);
cpi.ccb_h.func_code = XPT_PATH_INQ;
xpt_action((union ccb *)&cpi);
@@ -1953,11 +1947,11 @@ ata_set_transfer_settings(struct ccb_tra
device->tag_delay_count = CAM_TAG_DELAY_COUNT;
device->flags |= CAM_DEV_TAG_AFTER_COUNT;
} else if (nowt && !newt)
- xpt_stop_tags(cts->ccb_h.path);
+ xpt_stop_tags(path);
}
if (async_update == FALSE)
- (*(sim->sim_action))(sim, (union ccb *)cts);
+ xpt_action_default((union ccb *)cts);
}
/*
@@ -2014,10 +2008,14 @@ ata_dev_async(u_int32_t async_code, stru
xpt_release_device(device);
} else if (async_code == AC_TRANSFER_NEG) {
struct ccb_trans_settings *settings;
+ struct cam_path path;
settings = (struct ccb_trans_settings *)async_arg;
- ata_set_transfer_settings(settings, device,
+ xpt_compile_path(&path, NULL, bus->path_id, target->target_id,
+ device->lun_id);
+ ata_set_transfer_settings(settings, &path,
/*async_update*/TRUE);
+ xpt_release_path(&path);
}
}
@@ -2030,7 +2028,7 @@ ata_announce_periph(struct cam_periph *p
u_int speed;
u_int mb;
- mtx_assert(periph->sim->mtx, MA_OWNED);
+ cam_periph_assert(periph, MA_OWNED);
xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL);
cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
Modified: head/sys/cam/cam_ccb.h
==============================================================================
--- head/sys/cam/cam_ccb.h Mon Oct 21 10:09:48 2013 (r256842)
+++ head/sys/cam/cam_ccb.h Mon Oct 21 12:00:26 2013 (r256843)
@@ -104,7 +104,9 @@ typedef enum {
CAM_SEND_SENSE = 0x08000000,/* Send sense data with status */
CAM_TERM_IO = 0x10000000,/* Terminate I/O Message sup. */
CAM_DISCONNECT = 0x20000000,/* Disconnects are mandatory */
- CAM_SEND_STATUS = 0x40000000 /* Send status after data phase */
+ CAM_SEND_STATUS = 0x40000000,/* Send status after data phase */
+
+ CAM_UNLOCKED = 0x80000000 /* Call callback without lock. */
} ccb_flags;
typedef enum {
@@ -151,6 +153,9 @@ typedef enum {
/* Device statistics (error counts, etc.) */
XPT_DEV_ADVINFO = 0x0e,
/* Get/Set Device advanced information */
+ XPT_ASYNC = 0x0f | XPT_FC_QUEUED | XPT_FC_USER_CCB
+ | XPT_FC_XPT_ONLY,
+ /* Asynchronous event */
/* SCSI Control Functions: 0x10->0x1F */
XPT_ABORT = 0x10,
/* Abort the specified CCB */
@@ -1154,6 +1159,16 @@ struct ccb_dev_advinfo {
};
/*
+ * CCB for sending async events
+ */
+struct ccb_async {
+ struct ccb_hdr ccb_h;
+ uint32_t async_code;
+ off_t async_arg_size;
+ void *async_arg_ptr;
+};
+
+/*
* Union of all CCB types for kernel space allocation. This union should
* never be used for manipulating CCBs - its only use is for the allocation
* and deallocation of raw CCB space and is the return type of xpt_ccb_alloc
@@ -1192,6 +1207,7 @@ union ccb {
struct ccb_debug cdbg;
struct ccb_ataio ataio;
struct ccb_dev_advinfo cdai;
+ struct ccb_async casync;
};
__BEGIN_DECLS
Modified: head/sys/cam/cam_periph.c
==============================================================================
--- head/sys/cam/cam_periph.c Mon Oct 21 10:09:48 2013 (r256842)
+++ head/sys/cam/cam_periph.c Mon Oct 21 12:00:26 2013 (r256843)
@@ -196,12 +196,12 @@ cam_periph_alloc(periph_ctor_t *periph_c
path_id = xpt_path_path_id(path);
target_id = xpt_path_target_id(path);
lun_id = xpt_path_lun_id(path);
- cam_init_pinfo(&periph->pinfo);
periph->periph_start = periph_start;
periph->periph_dtor = periph_dtor;
periph->periph_oninval = periph_oninvalidate;
periph->type = type;
periph->periph_name = name;
+ periph->scheduled_priority = CAM_PRIORITY_NONE;
periph->immediate_priority = CAM_PRIORITY_NONE;
periph->refcount = 1; /* Dropped by invalidation. */
periph->sim = sim;
@@ -298,7 +298,7 @@ cam_periph_find(struct cam_path *path, c
TAILQ_FOREACH(periph, &(*p_drv)->units, unit_links) {
if (xpt_path_comp(periph->path, path) == 0) {
xpt_unlock_buses();
- mtx_assert(periph->sim->mtx, MA_OWNED);
+ cam_periph_assert(periph, MA_OWNED);
return(periph);
}
}
@@ -379,7 +379,7 @@ void
cam_periph_release_locked_buses(struct cam_periph *periph)
{
- mtx_assert(periph->sim->mtx, MA_OWNED);
+ cam_periph_assert(periph, MA_OWNED);
KASSERT(periph->refcount >= 1, ("periph->refcount >= 1"));
if (--periph->refcount == 0)
camperiphfree(periph);
@@ -400,16 +400,16 @@ cam_periph_release_locked(struct cam_per
void
cam_periph_release(struct cam_periph *periph)
{
- struct cam_sim *sim;
+ struct mtx *mtx;
if (periph == NULL)
return;
- sim = periph->sim;
- mtx_assert(sim->mtx, MA_NOTOWNED);
- mtx_lock(sim->mtx);
+ cam_periph_assert(periph, MA_NOTOWNED);
+ mtx = cam_periph_mtx(periph);
+ mtx_lock(mtx);
cam_periph_release_locked(periph);
- mtx_unlock(sim->mtx);
+ mtx_unlock(mtx);
}
int
@@ -427,10 +427,10 @@ cam_periph_hold(struct cam_periph *perip
if (cam_periph_acquire(periph) != CAM_REQ_CMP)
return (ENXIO);
- mtx_assert(periph->sim->mtx, MA_OWNED);
+ cam_periph_assert(periph, MA_OWNED);
while ((periph->flags & CAM_PERIPH_LOCKED) != 0) {
periph->flags |= CAM_PERIPH_LOCK_WANTED;
- if ((error = mtx_sleep(periph, periph->sim->mtx, priority,
+ if ((error = cam_periph_sleep(periph, periph, priority,
"caplck", 0)) != 0) {
cam_periph_release_locked(periph);
return (error);
@@ -449,7 +449,7 @@ void
cam_periph_unhold(struct cam_periph *periph)
{
- mtx_assert(periph->sim->mtx, MA_OWNED);
+ cam_periph_assert(periph, MA_OWNED);
periph->flags &= ~CAM_PERIPH_LOCKED;
if ((periph->flags & CAM_PERIPH_LOCK_WANTED) != 0) {
@@ -577,7 +577,7 @@ void
cam_periph_invalidate(struct cam_periph *periph)
{
- mtx_assert(periph->sim->mtx, MA_OWNED);
+ cam_periph_assert(periph, MA_OWNED);
/*
* We only call this routine the first time a peripheral is
* invalidated.
@@ -600,7 +600,9 @@ camperiphfree(struct cam_periph *periph)
{
struct periph_driver **p_drv;
- mtx_assert(periph->sim->mtx, MA_OWNED);
+ cam_periph_assert(periph, MA_OWNED);
+ KASSERT(periph->periph_allocating == 0, ("%s%d: freed while allocating",
+ periph->periph_name, periph->unit_number));
for (p_drv = periph_drivers; *p_drv != NULL; p_drv++) {
if (strcmp((*p_drv)->driver_name, periph->periph_name) == 0)
break;
@@ -947,40 +949,14 @@ cam_periph_unmapmem(union ccb *ccb, stru
PRELE(curproc);
}
-union ccb *
-cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
-{
- struct ccb_hdr *ccb_h;
-
- mtx_assert(periph->sim->mtx, MA_OWNED);
- CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("entering cdgetccb\n"));
-
- while (SLIST_FIRST(&periph->ccb_list) == NULL) {
- if (periph->immediate_priority > priority)
- periph->immediate_priority = priority;
- xpt_schedule(periph, priority);
- if ((SLIST_FIRST(&periph->ccb_list) != NULL)
- && (SLIST_FIRST(&periph->ccb_list)->pinfo.priority ==
priority))
- break;
- mtx_assert(periph->sim->mtx, MA_OWNED);
- mtx_sleep(&periph->ccb_list, periph->sim->mtx, PRIBIO, "cgticb",
- 0);
- }
-
- ccb_h = SLIST_FIRST(&periph->ccb_list);
- SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
- return ((union ccb *)ccb_h);
-}
-
void
cam_periph_ccbwait(union ccb *ccb)
{
- struct cam_sim *sim;
- sim = xpt_path_sim(ccb->ccb_h.path);
if ((ccb->ccb_h.pinfo.index != CAM_UNQUEUED_INDEX)
|| ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INPROG))
- mtx_sleep(&ccb->ccb_h.cbfcnp, sim->mtx, PRIBIO, "cbwait", 0);
+ xpt_path_sleep(ccb->ccb_h.path, &ccb->ccb_h.cbfcnp, PRIBIO,
+ "cbwait", 0);
}
int
@@ -1045,6 +1021,14 @@ cam_periph_ioctl(struct cam_periph *peri
return(error);
}
+static void
+cam_periph_done(struct cam_periph *periph, union ccb *done_ccb)
+{
+
+ /* Caller will release the CCB */
+ wakeup(&done_ccb->ccb_h.cbfcnp);
+}
+
int
cam_periph_runccb(union ccb *ccb,
int (*error_routine)(union ccb *ccb,
@@ -1053,12 +1037,9 @@ cam_periph_runccb(union ccb *ccb,
cam_flags camflags, u_int32_t sense_flags,
struct devstat *ds)
{
- struct cam_sim *sim;
int error;
- error = 0;
- sim = xpt_path_sim(ccb->ccb_h.path);
- mtx_assert(sim->mtx, MA_OWNED);
+ xpt_path_assert(ccb->ccb_h.path, MA_OWNED);
/*
* If the user has supplied a stats structure, and if we understand
@@ -1068,6 +1049,7 @@ cam_periph_runccb(union ccb *ccb,
ccb->ccb_h.func_code == XPT_ATA_IO))
devstat_start_transaction(ds, NULL);
+ ccb->ccb_h.cbfcnp = cam_periph_done;
xpt_action(ccb);
do {
Modified: head/sys/cam/cam_periph.h
==============================================================================
--- head/sys/cam/cam_periph.h Mon Oct 21 10:09:48 2013 (r256842)
+++ head/sys/cam/cam_periph.h Mon Oct 21 12:00:26 2013 (r256843)
@@ -35,6 +35,7 @@
#include <cam/cam_sim.h>
#ifdef _KERNEL
+#include <sys/taskqueue.h>
#include <cam/cam_xpt.h>
@@ -103,7 +104,6 @@ typedef cam_status periph_ctor_t (struct
typedef void periph_oninv_t (struct cam_periph *periph);
typedef void periph_dtor_t (struct cam_periph *periph);
struct cam_periph {
- cam_pinfo pinfo;
periph_start_t *periph_start;
periph_oninv_t *periph_oninval;
periph_dtor_t *periph_dtor;
@@ -120,15 +120,20 @@ struct cam_periph {
#define CAM_PERIPH_INVALID 0x08
#define CAM_PERIPH_NEW_DEV_FOUND 0x10
#define CAM_PERIPH_RECOVERY_INPROG 0x20
+#define CAM_PERIPH_RUN_TASK 0x40
#define CAM_PERIPH_FREE 0x80
#define CAM_PERIPH_ANNOUNCED 0x100
- u_int32_t immediate_priority;
+ uint32_t scheduled_priority;
+ uint32_t immediate_priority;
+ int periph_allocating;
+ int periph_allocated;
u_int32_t refcount;
SLIST_HEAD(, ccb_hdr) ccb_list; /* For "immediate" requests */
SLIST_ENTRY(cam_periph) periph_links;
TAILQ_ENTRY(cam_periph) unit_links;
ac_callback_t *deferred_callback;
ac_code deferred_ac;
+ struct task periph_run_task;
};
#define CAM_PERIPH_MAXMAPS 2
@@ -185,30 +190,26 @@ void cam_periph_freeze_after_event(stru
int cam_periph_error(union ccb *ccb, cam_flags camflags,
u_int32_t sense_flags, union ccb *save_ccb);
-static __inline void
-cam_periph_lock(struct cam_periph *periph)
+static __inline struct mtx *
+cam_periph_mtx(struct cam_periph *periph)
{
- mtx_lock(periph->sim->mtx);
+ return (xpt_path_mtx(periph->path));
}
-static __inline void
-cam_periph_unlock(struct cam_periph *periph)
-{
- mtx_unlock(periph->sim->mtx);
-}
+#define cam_periph_owned(periph) \
+ mtx_owned(xpt_path_mtx((periph)->path))
-static __inline int
-cam_periph_owned(struct cam_periph *periph)
-{
- return (mtx_owned(periph->sim->mtx));
-}
+#define cam_periph_lock(periph)
\
+ mtx_lock(xpt_path_mtx((periph)->path))
-static __inline int
-cam_periph_sleep(struct cam_periph *periph, void *chan, int priority,
- const char *wmesg, int timo)
-{
- return (msleep(chan, periph->sim->mtx, priority, wmesg, timo));
-}
+#define cam_periph_unlock(periph) \
+ mtx_unlock(xpt_path_mtx((periph)->path))
+
+#define cam_periph_assert(periph, what)
\
+ mtx_assert(xpt_path_mtx((periph)->path), (what))
+
+#define cam_periph_sleep(periph, chan, priority, wmesg, timo) \
+ xpt_path_sleep((periph)->path, (chan), (priority), (wmesg), (timo))
static inline struct cam_periph *
cam_periph_acquire_first(struct periph_driver *driver)
@@ -230,7 +231,7 @@ cam_periph_acquire_next(struct cam_perip
{
struct cam_periph *periph = pperiph;
- mtx_assert(pperiph->sim->mtx, MA_NOTOWNED);
+ cam_periph_assert(pperiph, MA_NOTOWNED);
xpt_lock_buses();
do {
periph = TAILQ_NEXT(periph, unit_links);
Modified: head/sys/cam/cam_queue.c
==============================================================================
--- head/sys/cam/cam_queue.c Mon Oct 21 10:09:48 2013 (r256842)
+++ head/sys/cam/cam_queue.c Mon Oct 21 12:00:26 2013 (r256843)
@@ -220,27 +220,30 @@ cam_devq_alloc(int devices, int openings
}
if (cam_devq_init(devq, devices, openings) != 0) {
free(devq, M_CAMDEVQ);
- return (NULL);
+ return (NULL);
}
-
return (devq);
}
int
cam_devq_init(struct cam_devq *devq, int devices, int openings)
{
+
bzero(devq, sizeof(*devq));
+ mtx_init(&devq->send_mtx, "CAM queue lock", NULL, MTX_DEF);
if (camq_init(&devq->send_queue, devices) != 0)
return (1);
devq->send_openings = openings;
- devq->send_active = 0;
- return (0);
+ devq->send_active = 0;
+ return (0);
}
void
cam_devq_free(struct cam_devq *devq)
{
+
camq_fini(&devq->send_queue);
+ mtx_destroy(&devq->send_mtx);
free(devq, M_CAMDEVQ);
}
@@ -286,6 +289,7 @@ cam_ccbq_resize(struct cam_ccbq *ccbq, i
int delta;
delta = new_size - (ccbq->dev_active + ccbq->dev_openings);
+ ccbq->total_openings += delta;
ccbq->devq_openings += delta;
ccbq->dev_openings += delta;
@@ -303,6 +307,7 @@ cam_ccbq_init(struct cam_ccbq *ccbq, int
if (camq_init(&ccbq->queue,
imax(64, 1 << fls(openings + openings / 2))) != 0)
return (1);
+ ccbq->total_openings = openings;
ccbq->devq_openings = openings;
ccbq->dev_openings = openings;
return (0);
Modified: head/sys/cam/cam_queue.h
==============================================================================
--- head/sys/cam/cam_queue.h Mon Oct 21 10:09:48 2013 (r256842)
+++ head/sys/cam/cam_queue.h Mon Oct 21 12:00:26 2013 (r256843)
@@ -33,6 +33,8 @@
#ifdef _KERNEL
+#include <sys/lock.h>
+#include <sys/mutex.h>
#include <sys/queue.h>
#include <cam/cam.h>
@@ -59,8 +61,8 @@ struct cam_ccbq {
struct camq queue;
struct ccb_hdr_tailq queue_extra_head;
int queue_extra_entries;
+ int total_openings;
int devq_openings;
- int devq_allocating;
int dev_openings;
int dev_active;
int held;
@@ -69,9 +71,10 @@ struct cam_ccbq {
struct cam_ed;
struct cam_devq {
- struct camq send_queue;
- int send_openings;
- int send_active;
+ struct mtx send_mtx;
+ struct camq send_queue;
+ int send_openings;
+ int send_active;
};
Modified: head/sys/cam/cam_sim.c
==============================================================================
--- head/sys/cam/cam_sim.c Mon Oct 21 10:09:48 2013 (r256842)
+++ head/sys/cam/cam_sim.c Mon Oct 21 12:00:26 2013 (r256843)
@@ -87,7 +87,6 @@ cam_sim_alloc(sim_action_func sim_action
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
