Author: mav
Date: Thu Jan 26 21:00:49 2017
New Revision: 312841
URL: https://svnweb.freebsd.org/changeset/base/312841
Log:
MFC r311804: Rewrite CTL statistics in more simple and scalable way.
Instead of collecting statistics for each combination of ports and logical
units, that consumed ~45KB per LU with present number of ports, collect
separate statistics for every port and every logical unit separately, that
consume only 176 bytes per each single LU/port. This reduces struct
ctl_lun size down to just 6KB.
Also new IOCTL API/ABI does not hardcode number of LUs/ports, and should
allow handling of very large quantities.
Old API is still enabled in stable branches for compatibility reasons.
Modified:
stable/10/sys/cam/ctl/ctl.c
stable/10/sys/cam/ctl/ctl_backend.h
stable/10/sys/cam/ctl/ctl_frontend.c
stable/10/sys/cam/ctl/ctl_frontend.h
stable/10/sys/cam/ctl/ctl_ioctl.h
stable/10/sys/cam/ctl/ctl_private.h
stable/10/usr.bin/ctlstat/ctlstat.8
stable/10/usr.bin/ctlstat/ctlstat.c
Directory Properties:
stable/10/ (props changed)
Modified: stable/10/sys/cam/ctl/ctl.c
==============================================================================
--- stable/10/sys/cam/ctl/ctl.c Thu Jan 26 20:59:36 2017 (r312840)
+++ stable/10/sys/cam/ctl/ctl.c Thu Jan 26 21:00:49 2017 (r312841)
@@ -1,7 +1,7 @@
/*-
* Copyright (c) 2003-2009 Silicon Graphics International Corp.
* Copyright (c) 2012 The FreeBSD Foundation
- * Copyright (c) 2015 Alexander Motin <m...@freebsd.org>
+ * Copyright (c) 2014-2017 Alexander Motin <m...@freebsd.org>
* All rights reserved.
*
* Portions of this software were developed by Edward Tomasz Napierala
@@ -2558,6 +2558,7 @@ ctl_ioctl(struct cdev *dev, u_long cmd,
struct thread *td)
{
struct ctl_softc *softc = dev->si_drv1;
+ struct ctl_port *port;
struct ctl_lun *lun;
int retval;
@@ -2769,6 +2770,7 @@ ctl_ioctl(struct cdev *dev, u_long cmd,
#endif /* CTL_IO_DELAY */
break;
}
+#ifdef CTL_LEGACY_STATS
case CTL_GETSTATS: {
struct ctl_stats *stats = (struct ctl_stats *)addr;
int i;
@@ -2781,26 +2783,26 @@ ctl_ioctl(struct cdev *dev, u_long cmd,
stats->status = CTL_SS_OK;
stats->fill_len = 0;
STAILQ_FOREACH(lun, &softc->lun_list, links) {
- if (stats->fill_len + sizeof(lun->stats) >
+ if (stats->fill_len + sizeof(lun->legacy_stats) >
stats->alloc_len) {
stats->status = CTL_SS_NEED_MORE_SPACE;
break;
}
- retval = copyout(&lun->stats, &stats->lun_stats[i++],
- sizeof(lun->stats));
+ retval = copyout(&lun->legacy_stats,
&stats->lun_stats[i++],
+ sizeof(lun->legacy_stats));
if (retval != 0)
break;
- stats->fill_len += sizeof(lun->stats);
+ stats->fill_len += sizeof(lun->legacy_stats);
}
stats->num_luns = softc->num_luns;
-#ifdef CTL_TIME_IO
- stats->flags = CTL_STATS_FLAG_TIME_VALID;
-#else
stats->flags = CTL_STATS_FLAG_NONE;
+#ifdef CTL_TIME_IO
+ stats->flags |= CTL_STATS_FLAG_TIME_VALID;
#endif
getnanouptime(&stats->timestamp);
break;
}
+#endif /* CTL_LEGACY_STATS */
case CTL_ERROR_INJECT: {
struct ctl_error_desc *err_desc, *new_err_desc;
@@ -3388,6 +3390,72 @@ ctl_ioctl(struct cdev *dev, u_long cmd,
ctl_isc_announce_port(port);
break;
}
+ case CTL_GET_LUN_STATS: {
+ struct ctl_get_io_stats *stats = (struct ctl_get_io_stats
*)addr;
+ int i;
+
+ /*
+ * XXX KDM no locking here. If the LUN list changes,
+ * things can blow up.
+ */
+ i = 0;
+ stats->status = CTL_SS_OK;
+ stats->fill_len = 0;
+ STAILQ_FOREACH(lun, &softc->lun_list, links) {
+ if (lun->lun < stats->first_item)
+ continue;
+ if (stats->fill_len + sizeof(lun->stats) >
+ stats->alloc_len) {
+ stats->status = CTL_SS_NEED_MORE_SPACE;
+ break;
+ }
+ retval = copyout(&lun->stats, &stats->stats[i++],
+ sizeof(lun->stats));
+ if (retval != 0)
+ break;
+ stats->fill_len += sizeof(lun->stats);
+ }
+ stats->num_items = softc->num_luns;
+ stats->flags = CTL_STATS_FLAG_NONE;
+#ifdef CTL_TIME_IO
+ stats->flags |= CTL_STATS_FLAG_TIME_VALID;
+#endif
+ getnanouptime(&stats->timestamp);
+ break;
+ }
+ case CTL_GET_PORT_STATS: {
+ struct ctl_get_io_stats *stats = (struct ctl_get_io_stats
*)addr;
+ int i;
+
+ /*
+ * XXX KDM no locking here. If the LUN list changes,
+ * things can blow up.
+ */
+ i = 0;
+ stats->status = CTL_SS_OK;
+ stats->fill_len = 0;
+ STAILQ_FOREACH(port, &softc->port_list, links) {
+ if (port->targ_port < stats->first_item)
+ continue;
+ if (stats->fill_len + sizeof(port->stats) >
+ stats->alloc_len) {
+ stats->status = CTL_SS_NEED_MORE_SPACE;
+ break;
+ }
+ retval = copyout(&port->stats, &stats->stats[i++],
+ sizeof(port->stats));
+ if (retval != 0)
+ break;
+ stats->fill_len += sizeof(port->stats);
+ }
+ stats->num_items = softc->num_ports;
+ stats->flags = CTL_STATS_FLAG_NONE;
+#ifdef CTL_TIME_IO
+ stats->flags |= CTL_STATS_FLAG_TIME_VALID;
+#endif
+ getnanouptime(&stats->timestamp);
+ break;
+ }
default: {
/* XXX KDM should we fix this? */
#if 0
@@ -4382,7 +4450,7 @@ ctl_alloc_lun(struct ctl_softc *ctl_soft
struct scsi_vpd_id_descriptor *desc;
struct scsi_vpd_id_t10 *t10id;
const char *eui, *naa, *scsiname, *uuid, *vendor, *value;
- int lun_number, i, lun_malloced;
+ int lun_number, lun_malloced;
int devidlen, idlen1, idlen2 = 0, len;
if (be_lun == NULL)
@@ -4604,13 +4672,16 @@ ctl_alloc_lun(struct ctl_softc *ctl_soft
ctl_softc->num_luns++;
/* Setup statistics gathering */
- lun->stats.device_type = be_lun->lun_type;
- lun->stats.lun_number = lun_number;
- lun->stats.blocksize = be_lun->blocksize;
+#ifdef CTL_LEGACY_STATS
+ lun->legacy_stats.device_type = be_lun->lun_type;
+ lun->legacy_stats.lun_number = lun_number;
+ lun->legacy_stats.blocksize = be_lun->blocksize;
if (be_lun->blocksize == 0)
- lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
- for (i = 0;i < CTL_MAX_PORTS;i++)
- lun->stats.ports[i].targ_port = i;
+ lun->legacy_stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
+ for (len = 0; len < CTL_MAX_PORTS; len++)
+ lun->legacy_stats.ports[len].targ_port = len;
+#endif /* CTL_LEGACY_STATS */
+ lun->stats.item = lun_number;
mtx_unlock(&ctl_softc->ctl_lock);
@@ -6676,9 +6747,7 @@ ctl_sap_log_sense_handler(struct ctl_scs
{
struct ctl_lun *lun = CTL_LUN(ctsio);
struct stat_page *data;
- uint64_t rn, wn, rb, wb;
- struct bintime rt, wt;
- int i;
+ struct bintime *t;
data = (struct stat_page *)page_index->page_data;
@@ -6686,28 +6755,21 @@ ctl_sap_log_sense_handler(struct ctl_scs
data->sap.hdr.param_control = SLP_LBIN;
data->sap.hdr.param_len = sizeof(struct scsi_log_stat_and_perf) -
sizeof(struct scsi_log_param_header);
- rn = wn = rb = wb = 0;
- bintime_clear(&rt);
- bintime_clear(&wt);
- for (i = 0; i < CTL_MAX_PORTS; i++) {
- rn += lun->stats.ports[i].operations[CTL_STATS_READ];
- wn += lun->stats.ports[i].operations[CTL_STATS_WRITE];
- rb += lun->stats.ports[i].bytes[CTL_STATS_READ];
- wb += lun->stats.ports[i].bytes[CTL_STATS_WRITE];
- bintime_add(&rt, &lun->stats.ports[i].time[CTL_STATS_READ]);
- bintime_add(&wt, &lun->stats.ports[i].time[CTL_STATS_WRITE]);
- }
- scsi_u64to8b(rn, data->sap.read_num);
- scsi_u64to8b(wn, data->sap.write_num);
- if (lun->stats.blocksize > 0) {
- scsi_u64to8b(wb / lun->stats.blocksize,
- data->sap.recvieved_lba);
- scsi_u64to8b(rb / lun->stats.blocksize,
- data->sap.transmitted_lba);
+ scsi_u64to8b(lun->stats.operations[CTL_STATS_READ],
+ data->sap.read_num);
+ scsi_u64to8b(lun->stats.operations[CTL_STATS_WRITE],
+ data->sap.write_num);
+ if (lun->be_lun->blocksize > 0) {
+ scsi_u64to8b(lun->stats.bytes[CTL_STATS_WRITE] /
+ lun->be_lun->blocksize, data->sap.recvieved_lba);
+ scsi_u64to8b(lun->stats.bytes[CTL_STATS_READ] /
+ lun->be_lun->blocksize, data->sap.transmitted_lba);
}
- scsi_u64to8b((uint64_t)rt.sec * 1000 + rt.frac / (UINT64_MAX / 1000),
+ t = &lun->stats.time[CTL_STATS_READ];
+ scsi_u64to8b((uint64_t)t->sec * 1000 + t->frac / (UINT64_MAX / 1000),
data->sap.read_int);
- scsi_u64to8b((uint64_t)wt.sec * 1000 + wt.frac / (UINT64_MAX / 1000),
+ t = &lun->stats.time[CTL_STATS_WRITE];
+ scsi_u64to8b((uint64_t)t->sec * 1000 + t->frac / (UINT64_MAX / 1000),
data->sap.write_int);
scsi_u64to8b(0, data->sap.weighted_num);
scsi_u64to8b(0, data->sap.weighted_int);
@@ -13044,13 +13106,13 @@ static void
ctl_process_done(union ctl_io *io)
{
struct ctl_softc *softc = CTL_SOFTC(io);
+ struct ctl_port *port = CTL_PORT(io);
struct ctl_lun *lun = CTL_LUN(io);
void (*fe_done)(union ctl_io *io);
union ctl_ha_msg msg;
- uint32_t targ_port = io->io_hdr.nexus.targ_port;
CTL_DEBUG_PRINT(("ctl_process_done\n"));
- fe_done = softc->ctl_ports[targ_port]->fe_done;
+ fe_done = port->fe_done;
#ifdef CTL_TIME_IO
if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
@@ -13153,11 +13215,13 @@ ctl_process_done(union ctl_io *io)
*/
if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS &&
io->io_hdr.io_type == CTL_IO_SCSI) {
-#ifdef CTL_TIME_IO
- struct bintime cur_bt;
-#endif
int type;
+#ifdef CTL_TIME_IO
+ struct bintime bt;
+ getbinuptime(&bt);
+ bintime_sub(&bt, &io->io_hdr.start_bt);
+#endif
if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
CTL_FLAG_DATA_IN)
type = CTL_STATS_READ;
@@ -13167,18 +13231,38 @@ ctl_process_done(union ctl_io *io)
else
type = CTL_STATS_NO_IO;
- lun->stats.ports[targ_port].bytes[type] +=
+#ifdef CTL_LEGACY_STATS
+ uint32_t targ_port = port->targ_port;
+ lun->legacy_stats.ports[targ_port].bytes[type] +=
io->scsiio.kern_total_len;
- lun->stats.ports[targ_port].operations[type]++;
+ lun->legacy_stats.ports[targ_port].operations[type] ++;
+ lun->legacy_stats.ports[targ_port].num_dmas[type] +=
+ io->io_hdr.num_dmas;
#ifdef CTL_TIME_IO
- bintime_add(&lun->stats.ports[targ_port].dma_time[type],
+ bintime_add(&lun->legacy_stats.ports[targ_port].dma_time[type],
&io->io_hdr.dma_bt);
- getbinuptime(&cur_bt);
- bintime_sub(&cur_bt, &io->io_hdr.start_bt);
- bintime_add(&lun->stats.ports[targ_port].time[type], &cur_bt);
+ bintime_add(&lun->legacy_stats.ports[targ_port].time[type],
+ &bt);
#endif
- lun->stats.ports[targ_port].num_dmas[type] +=
- io->io_hdr.num_dmas;
+#endif /* CTL_LEGACY_STATS */
+
+ lun->stats.bytes[type] += io->scsiio.kern_total_len;
+ lun->stats.operations[type] ++;
+ lun->stats.dmas[type] += io->io_hdr.num_dmas;
+#ifdef CTL_TIME_IO
+ bintime_add(&lun->stats.dma_time[type], &io->io_hdr.dma_bt);
+ bintime_add(&lun->stats.time[type], &bt);
+#endif
+
+ mtx_lock(&port->port_lock);
+ port->stats.bytes[type] += io->scsiio.kern_total_len;
+ port->stats.operations[type] ++;
+ port->stats.dmas[type] += io->io_hdr.num_dmas;
+#ifdef CTL_TIME_IO
+ bintime_add(&port->stats.dma_time[type], &io->io_hdr.dma_bt);
+ bintime_add(&port->stats.time[type], &bt);
+#endif
+ mtx_unlock(&port->port_lock);
}
/*
Modified: stable/10/sys/cam/ctl/ctl_backend.h
==============================================================================
--- stable/10/sys/cam/ctl/ctl_backend.h Thu Jan 26 20:59:36 2017
(r312840)
+++ stable/10/sys/cam/ctl/ctl_backend.h Thu Jan 26 21:00:49 2017
(r312841)
@@ -1,6 +1,6 @@
/*-
* Copyright (c) 2003 Silicon Graphics International Corp.
- * Copyright (c) 2014-2015 Alexander Motin <m...@freebsd.org>
+ * Copyright (c) 2014-2017 Alexander Motin <m...@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -40,54 +40,7 @@
#ifndef _CTL_BACKEND_H_
#define _CTL_BACKEND_H_
-/*
- * XXX KDM move this to another header file?
- */
-#define CTL_BE_NAME_LEN 32
-
-/*
- * The ID_REQ flag is used to say that the caller has requested a
- * particular LUN ID in the req_lun_id field. If we cannot allocate that
- * LUN ID, the ctl_add_lun() call will fail.
- *
- * The STOPPED flag tells us that the LUN should default to the powered
- * off state. It will return 0x04,0x02 until it is powered up. ("Logical
- * unit not ready, initializing command required.")
- *
- * The NO_MEDIA flag tells us that the LUN has no media inserted.
- *
- * The PRIMARY flag tells us that this LUN is registered as a Primary LUN
- * which is accessible via the Master shelf controller in an HA. This flag
- * being set indicates a Primary LUN. This flag being reset represents a
- * Secondary LUN controlled by the Secondary controller in an HA
- * configuration. Flag is applicable at this time to T_DIRECT types.
- *
- * The SERIAL_NUM flag tells us that the serial_num field is filled in and
- * valid for use in SCSI INQUIRY VPD page 0x80.
- *
- * The DEVID flag tells us that the device_id field is filled in and
- * valid for use in SCSI INQUIRY VPD page 0x83.
- *
- * The DEV_TYPE flag tells us that the device_type field is filled in.
- *
- * The EJECTED flag tells us that the removable LUN has tray open.
- *
- * The UNMAP flag tells us that this LUN supports UNMAP.
- *
- * The OFFLINE flag tells us that this LUN can not access backing store.
- */
-typedef enum {
- CTL_LUN_FLAG_ID_REQ = 0x01,
- CTL_LUN_FLAG_STOPPED = 0x02,
- CTL_LUN_FLAG_NO_MEDIA = 0x04,
- CTL_LUN_FLAG_PRIMARY = 0x08,
- CTL_LUN_FLAG_SERIAL_NUM = 0x10,
- CTL_LUN_FLAG_DEVID = 0x20,
- CTL_LUN_FLAG_DEV_TYPE = 0x40,
- CTL_LUN_FLAG_UNMAP = 0x80,
- CTL_LUN_FLAG_EJECTED = 0x100,
- CTL_LUN_FLAG_READONLY = 0x200
-} ctl_backend_lun_flags;
+#include <cam/ctl/ctl_ioctl.h>
typedef enum {
CTL_LUN_SERSEQ_OFF,
Modified: stable/10/sys/cam/ctl/ctl_frontend.c
==============================================================================
--- stable/10/sys/cam/ctl/ctl_frontend.c Thu Jan 26 20:59:36 2017
(r312840)
+++ stable/10/sys/cam/ctl/ctl_frontend.c Thu Jan 26 21:00:49 2017
(r312841)
@@ -1,5 +1,6 @@
/*-
* Copyright (c) 2003 Silicon Graphics International Corp.
+ * Copyright (c) 2014-2017 Alexander Motin <m...@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -192,13 +193,14 @@ error:
mtx_unlock(&softc->ctl_lock);
return (retval);
}
+ port->targ_port = port_num;
port->ctl_pool_ref = pool;
-
if (port->options.stqh_first == NULL)
STAILQ_INIT(&port->options);
+ port->stats.item = port_num;
+ mtx_init(&port->port_lock, "CTL port", NULL, MTX_DEF);
mtx_lock(&softc->ctl_lock);
- port->targ_port = port_num;
STAILQ_INSERT_TAIL(&port->frontend->port_list, port, fe_links);
for (tport = NULL, nport = STAILQ_FIRST(&softc->port_list);
nport != NULL && nport->targ_port < port_num;
@@ -218,17 +220,11 @@ int
ctl_port_deregister(struct ctl_port *port)
{
struct ctl_softc *softc = port->ctl_softc;
- struct ctl_io_pool *pool;
- int retval, i;
-
- retval = 0;
-
- pool = (struct ctl_io_pool *)port->ctl_pool_ref;
+ struct ctl_io_pool *pool = (struct ctl_io_pool *)port->ctl_pool_ref;
+ int i;
- if (port->targ_port == -1) {
- retval = 1;
- goto bailout;
- }
+ if (port->targ_port == -1)
+ return (1);
mtx_lock(&softc->ctl_lock);
STAILQ_REMOVE(&softc->port_list, port, ctl_port, links);
@@ -251,9 +247,9 @@ ctl_port_deregister(struct ctl_port *por
for (i = 0; i < port->max_initiators; i++)
free(port->wwpn_iid[i].name, M_CTL);
free(port->wwpn_iid, M_CTL);
+ mtx_destroy(&port->port_lock);
-bailout:
- return (retval);
+ return (0);
}
void
Modified: stable/10/sys/cam/ctl/ctl_frontend.h
==============================================================================
--- stable/10/sys/cam/ctl/ctl_frontend.h Thu Jan 26 20:59:36 2017
(r312840)
+++ stable/10/sys/cam/ctl/ctl_frontend.h Thu Jan 26 21:00:49 2017
(r312841)
@@ -1,5 +1,6 @@
/*-
* Copyright (c) 2003 Silicon Graphics International Corp.
+ * Copyright (c) 2014-2017 Alexander Motin <m...@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -39,6 +40,8 @@
#ifndef _CTL_FRONTEND_H_
#define _CTL_FRONTEND_H_
+#include <cam/ctl/ctl_ioctl.h>
+
typedef enum {
CTL_PORT_STATUS_NONE = 0x00,
CTL_PORT_STATUS_ONLINE = 0x01,
@@ -243,6 +246,8 @@ struct ctl_port {
struct ctl_devid *port_devid; /* passed to CTL */
struct ctl_devid *target_devid; /* passed to CTL */
struct ctl_devid *init_devid; /* passed to CTL */
+ struct ctl_io_stats stats; /* used by CTL */
+ struct mtx port_lock; /* used by CTL */
STAILQ_ENTRY(ctl_port) fe_links; /* used by CTL */
STAILQ_ENTRY(ctl_port) links; /* used by CTL */
};
Modified: stable/10/sys/cam/ctl/ctl_ioctl.h
==============================================================================
--- stable/10/sys/cam/ctl/ctl_ioctl.h Thu Jan 26 20:59:36 2017
(r312840)
+++ stable/10/sys/cam/ctl/ctl_ioctl.h Thu Jan 26 21:00:49 2017
(r312841)
@@ -1,6 +1,7 @@
/*-
* Copyright (c) 2003 Silicon Graphics International Corp.
* Copyright (c) 2011 Spectra Logic Corporation
+ * Copyright (c) 2014-2017 Alexander Motin <m...@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -80,6 +81,9 @@
/* Hopefully this won't conflict with new misc devices that pop up */
#define CTL_MINOR 225
+/* Legacy statistics accumulated for every port for every LU. */
+#define CTL_LEGACY_STATS 1
+
typedef enum {
CTL_DELAY_TYPE_NONE,
CTL_DELAY_TYPE_CONT,
@@ -117,6 +121,18 @@ typedef enum {
#define CTL_STATS_NUM_TYPES 3
typedef enum {
+ CTL_SS_OK,
+ CTL_SS_NEED_MORE_SPACE,
+ CTL_SS_ERROR
+} ctl_stats_status;
+
+typedef enum {
+ CTL_STATS_FLAG_NONE = 0x00,
+ CTL_STATS_FLAG_TIME_VALID = 0x01
+} ctl_stats_flags;
+
+#ifdef CTL_LEGACY_STATS
+typedef enum {
CTL_LUN_STATS_NO_BLOCKSIZE = 0x01
} ctl_lun_stats_flags;
@@ -137,17 +153,6 @@ struct ctl_lun_io_stats {
struct ctl_lun_io_port_stats ports[CTL_MAX_PORTS];
};
-typedef enum {
- CTL_SS_OK,
- CTL_SS_NEED_MORE_SPACE,
- CTL_SS_ERROR
-} ctl_stats_status;
-
-typedef enum {
- CTL_STATS_FLAG_NONE = 0x00,
- CTL_STATS_FLAG_TIME_VALID = 0x01
-} ctl_stats_flags;
-
struct ctl_stats {
int alloc_len; /* passed to kernel */
struct ctl_lun_io_stats *lun_stats; /* passed to/from kernel */
@@ -157,6 +162,27 @@ struct ctl_stats {
ctl_stats_flags flags; /* passed to userland */
struct timespec timestamp; /* passed to userland */
};
+#endif /* CTL_LEGACY_STATS */
+
+struct ctl_io_stats {
+ uint32_t item;
+ uint64_t bytes[CTL_STATS_NUM_TYPES];
+ uint64_t operations[CTL_STATS_NUM_TYPES];
+ uint64_t dmas[CTL_STATS_NUM_TYPES];
+ struct bintime time[CTL_STATS_NUM_TYPES];
+ struct bintime dma_time[CTL_STATS_NUM_TYPES];
+};
+
+struct ctl_get_io_stats {
+ struct ctl_io_stats *stats; /* passed to/from kernel */
+ size_t alloc_len; /* passed to kernel */
+ size_t fill_len; /* passed to userland */
+ int first_item; /* passed to kernel */
+ int num_items; /* passed to userland */
+ ctl_stats_status status; /* passed to userland */
+ ctl_stats_flags flags; /* passed to userland */
+ struct timespec timestamp; /* passed to userland */
+};
/*
* The types of errors that can be injected:
@@ -342,12 +368,54 @@ typedef enum {
CTL_LUNREQ_MODIFY,
} ctl_lunreq_type;
+/*
+ * The ID_REQ flag is used to say that the caller has requested a
+ * particular LUN ID in the req_lun_id field. If we cannot allocate that
+ * LUN ID, the ctl_add_lun() call will fail.
+ *
+ * The STOPPED flag tells us that the LUN should default to the powered
+ * off state. It will return 0x04,0x02 until it is powered up. ("Logical
+ * unit not ready, initializing command required.")
+ *
+ * The NO_MEDIA flag tells us that the LUN has no media inserted.
+ *
+ * The PRIMARY flag tells us that this LUN is registered as a Primary LUN
+ * which is accessible via the Master shelf controller in an HA. This flag
+ * being set indicates a Primary LUN. This flag being reset represents a
+ * Secondary LUN controlled by the Secondary controller in an HA
+ * configuration. Flag is applicable at this time to T_DIRECT types.
+ *
+ * The SERIAL_NUM flag tells us that the serial_num field is filled in and
+ * valid for use in SCSI INQUIRY VPD page 0x80.
+ *
+ * The DEVID flag tells us that the device_id field is filled in and
+ * valid for use in SCSI INQUIRY VPD page 0x83.
+ *
+ * The DEV_TYPE flag tells us that the device_type field is filled in.
+ *
+ * The EJECTED flag tells us that the removable LUN has tray open.
+ *
+ * The UNMAP flag tells us that this LUN supports UNMAP.
+ *
+ * The OFFLINE flag tells us that this LUN can not access backing store.
+ */
+typedef enum {
+ CTL_LUN_FLAG_ID_REQ = 0x01,
+ CTL_LUN_FLAG_STOPPED = 0x02,
+ CTL_LUN_FLAG_NO_MEDIA = 0x04,
+ CTL_LUN_FLAG_PRIMARY = 0x08,
+ CTL_LUN_FLAG_SERIAL_NUM = 0x10,
+ CTL_LUN_FLAG_DEVID = 0x20,
+ CTL_LUN_FLAG_DEV_TYPE = 0x40,
+ CTL_LUN_FLAG_UNMAP = 0x80,
+ CTL_LUN_FLAG_EJECTED = 0x100,
+ CTL_LUN_FLAG_READONLY = 0x200
+} ctl_backend_lun_flags;
/*
* LUN creation parameters:
*
- * flags: Various LUN flags, see ctl_backend.h for a
- * description of the flag values and meanings.
+ * flags: Various LUN flags, see above.
*
* device_type: The SCSI device type. e.g. 0 for Direct Access,
* 3 for Processor, etc. Only certain backends may
@@ -465,6 +533,7 @@ union ctl_lunreq_data {
* kern_be_args: For kernel use only.
*/
struct ctl_lun_req {
+#define CTL_BE_NAME_LEN 32
char backend[CTL_BE_NAME_LEN];
ctl_lunreq_type reqtype;
union ctl_lunreq_data reqdata;
@@ -761,6 +830,8 @@ struct ctl_lun_map {
#define CTL_PORT_REQ _IOWR(CTL_MINOR, 0x26, struct ctl_req)
#define CTL_PORT_LIST _IOWR(CTL_MINOR, 0x27, struct
ctl_lun_list)
#define CTL_LUN_MAP _IOW(CTL_MINOR, 0x28, struct
ctl_lun_map)
+#define CTL_GET_LUN_STATS _IOWR(CTL_MINOR, 0x29, struct
ctl_get_io_stats)
+#define CTL_GET_PORT_STATS _IOWR(CTL_MINOR, 0x2a, struct
ctl_get_io_stats)
#endif /* _CTL_IOCTL_H_ */
Modified: stable/10/sys/cam/ctl/ctl_private.h
==============================================================================
--- stable/10/sys/cam/ctl/ctl_private.h Thu Jan 26 20:59:36 2017
(r312840)
+++ stable/10/sys/cam/ctl/ctl_private.h Thu Jan 26 21:00:49 2017
(r312841)
@@ -1,6 +1,6 @@
/*-
* Copyright (c) 2003, 2004, 2005, 2008 Silicon Graphics International Corp.
- * Copyright (c) 2014-2015 Alexander Motin <m...@freebsd.org>
+ * Copyright (c) 2014-2017 Alexander Motin <m...@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -404,7 +404,10 @@ struct ctl_lun {
struct callout ie_callout; /* INTERVAL TIMER */
struct ctl_mode_pages mode_pages;
struct ctl_log_pages log_pages;
- struct ctl_lun_io_stats stats;
+#ifdef CTL_LEGACY_STATS
+ struct ctl_lun_io_stats legacy_stats;
+#endif /* CTL_LEGACY_STATS */
+ struct ctl_io_stats stats;
uint32_t res_idx;
uint32_t pr_generation;
uint64_t *pr_keys[CTL_MAX_PORTS];
Modified: stable/10/usr.bin/ctlstat/ctlstat.8
==============================================================================
--- stable/10/usr.bin/ctlstat/ctlstat.8 Thu Jan 26 20:59:36 2017
(r312840)
+++ stable/10/usr.bin/ctlstat/ctlstat.8 Thu Jan 26 21:00:49 2017
(r312841)
@@ -34,7 +34,7 @@
.\" $Id: //depot/users/kenm/FreeBSD-test2/usr.bin/ctlstat/ctlstat.8#2 $
.\" $FreeBSD$
.\"
-.Dd September 21, 2015
+.Dd January 9, 2017
.Dt CTLSTAT 8
.Os
.Sh NAME
@@ -118,5 +118,6 @@ every 10 seconds.
.Xr ctld 8 ,
.Xr iostat 8
.Sh AUTHORS
-.An Ken Merry Aq k...@freebsd.org
-.An Will Andrews Aq w...@freebsd.org
+.An Ken Merry Aq Mt k...@freebsd.org
+.An Will Andrews Aq Mt w...@freebsd.org
+.An Alexander Motin Aq Mt m...@freebsd.org
Modified: stable/10/usr.bin/ctlstat/ctlstat.c
==============================================================================
--- stable/10/usr.bin/ctlstat/ctlstat.c Thu Jan 26 20:59:36 2017
(r312840)
+++ stable/10/usr.bin/ctlstat/ctlstat.c Thu Jan 26 21:00:49 2017
(r312841)
@@ -1,5 +1,6 @@
/*-
* Copyright (c) 2004, 2008, 2009 Silicon Graphics International Corp.
+ * Copyright (c) 2017 Alexander Motin <m...@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -66,17 +67,17 @@ __FBSDID("$FreeBSD$");
#include <cam/ctl/ctl_ioctl.h>
/*
- * The default amount of space we allocate for LUN storage space. We
- * dynamically allocate more if needed.
+ * The default amount of space we allocate for stats storage space.
+ * We dynamically allocate more if needed.
*/
-#define CTL_STAT_NUM_LUNS 30
+#define CTL_STAT_NUM_ITEMS 256
/*
* The default number of LUN selection bits we allocate. This is large
* because we don't currently increase it if the user specifies a LUN
* number of 1024 or larger.
*/
-#define CTL_STAT_LUN_BITS 1024L
+#define CTL_STAT_BITS 1024L
static const char *ctlstat_opts = "Cc:Ddhjl:n:p:tw:";
static const char *ctlstat_usage = "Usage: ctlstat [-CDdjht] [-l lunnum]"
@@ -101,31 +102,32 @@ typedef enum {
#define CTLSTAT_FLAG_FIRST_RUN (1 << 2)
#define CTLSTAT_FLAG_TOTALS (1 << 3)
#define CTLSTAT_FLAG_DMA_TIME (1 << 4)
-#define CTLSTAT_FLAG_LUN_TIME_VALID (1 << 5)
-#define CTLSTAT_FLAG_LUN_MASK (1 << 6)
-#define CTLSTAT_FLAG_PORT_MASK (1 << 7)
+#define CTLSTAT_FLAG_TIME_VALID (1 << 5)
+#define CTLSTAT_FLAG_MASK (1 << 6)
+#define CTLSTAT_FLAG_LUNS (1 << 7)
+#define CTLSTAT_FLAG_PORTS (1 << 8)
#define F_CPU(ctx) ((ctx)->flags & CTLSTAT_FLAG_CPU)
#define F_HDR(ctx) ((ctx)->flags & CTLSTAT_FLAG_HEADER)
#define F_FIRST(ctx) ((ctx)->flags & CTLSTAT_FLAG_FIRST_RUN)
#define F_TOTALS(ctx) ((ctx)->flags & CTLSTAT_FLAG_TOTALS)
#define F_DMA(ctx) ((ctx)->flags & CTLSTAT_FLAG_DMA_TIME)
-#define F_LUNVAL(ctx) ((ctx)->flags & CTLSTAT_FLAG_LUN_TIME_VALID)
-#define F_LUNMASK(ctx) ((ctx)->flags & CTLSTAT_FLAG_LUN_MASK)
-#define F_PORTMASK(ctx) ((ctx)->flags & CTLSTAT_FLAG_PORT_MASK)
+#define F_TIMEVAL(ctx) ((ctx)->flags & CTLSTAT_FLAG_TIME_VALID)
+#define F_MASK(ctx) ((ctx)->flags & CTLSTAT_FLAG_MASK)
+#define F_LUNS(ctx) ((ctx)->flags & CTLSTAT_FLAG_LUNS)
+#define F_PORTS(ctx) ((ctx)->flags & CTLSTAT_FLAG_PORTS)
struct ctlstat_context {
ctlstat_mode_types mode;
int flags;
- struct ctl_lun_io_stats *cur_lun_stats, *prev_lun_stats,
- *tmp_lun_stats;
- struct ctl_lun_io_stats cur_total_stats[3], prev_total_stats[3];
+ struct ctl_io_stats *cur_stats, *prev_stats;
+ struct ctl_io_stats cur_total_stats[3], prev_total_stats[3];
struct timespec cur_time, prev_time;
struct ctl_cpu_stats cur_cpu, prev_cpu;
uint64_t cur_total_jiffies, prev_total_jiffies;
uint64_t cur_idle, prev_idle;
- bitstr_t bit_decl(lun_mask, CTL_STAT_LUN_BITS);
- bitstr_t bit_decl(port_mask, CTL_MAX_PORTS);
- int num_luns;
+ bitstr_t bit_decl(item_mask, CTL_STAT_BITS);
+ int cur_items, prev_items;
+ int cur_alloc, prev_alloc;
int numdevs;
int header_interval;
};
@@ -135,12 +137,11 @@ struct ctlstat_context {
#endif
static void usage(int error);
-static int getstats(int fd, int *num_luns, struct ctl_lun_io_stats
**xlun_stats,
- struct timespec *cur_time, int *lun_time_valid);
+static int getstats(int fd, int *alloc_items, int *num_items,
+ struct ctl_io_stats **xstats, struct timespec *cur_time, int *time_valid);
static int getcpu(struct ctl_cpu_stats *cpu_stats);
-static void compute_stats(struct ctlstat_context *ctx,
- struct ctl_lun_io_stats *cur_stats,
- struct ctl_lun_io_stats *prev_stats,
+static void compute_stats(struct ctl_io_stats *cur_stats,
+ struct ctl_io_stats *prev_stats,
long double etime, long double *mbsec,
long double *kb_per_transfer,
long double *transfers_per_second,
@@ -155,64 +156,55 @@ usage(int error)
}
static int
-getstats(int fd, int *num_luns, struct ctl_lun_io_stats **xlun_stats,
+getstats(int fd, int *alloc_items, int *num_items, struct ctl_io_stats **stats,
struct timespec *cur_time, int *flags)
{
- struct ctl_lun_io_stats *lun_stats;
- struct ctl_stats stats;
- int more_space_count;
+ struct ctl_get_io_stats get_stats;
+ int more_space_count = 0;
- more_space_count = 0;
-
- if (*num_luns == 0)
- *num_luns = CTL_STAT_NUM_LUNS;
-
- lun_stats = *xlun_stats;
+ if (*alloc_items == 0)
+ *alloc_items = CTL_STAT_NUM_ITEMS;
retry:
+ if (*stats == NULL)
+ *stats = malloc(sizeof(**stats) * *alloc_items);
- if (lun_stats == NULL) {
- lun_stats = (struct ctl_lun_io_stats *)malloc(
- sizeof(*lun_stats) * *num_luns);
- }
-
- memset(&stats, 0, sizeof(stats));
- stats.alloc_len = *num_luns * sizeof(*lun_stats);
- memset(lun_stats, 0, stats.alloc_len);
- stats.lun_stats = lun_stats;
-
- if (ioctl(fd, CTL_GETSTATS, &stats) == -1)
- err(1, "error returned from CTL_GETSTATS ioctl");
+ memset(&get_stats, 0, sizeof(get_stats));
+ get_stats.alloc_len = *alloc_items * sizeof(**stats);
+ memset(*stats, 0, get_stats.alloc_len);
+ get_stats.stats = *stats;
+
+ if (ioctl(fd, (*flags & CTLSTAT_FLAG_PORTS) ? CTL_GET_PORT_STATS :
+ CTL_GET_LUN_STATS, &get_stats) == -1)
+ err(1, "CTL_GET_*_STATS ioctl returned error");
- switch (stats.status) {
+ switch (get_stats.status) {
case CTL_SS_OK:
break;
case CTL_SS_ERROR:
- err(1, "CTL_SS_ERROR returned from CTL_GETSTATS ioctl");
+ err(1, "CTL_GET_*_STATS ioctl returned CTL_SS_ERROR");
break;
case CTL_SS_NEED_MORE_SPACE:
- if (more_space_count > 0) {
- errx(1, "CTL_GETSTATS returned NEED_MORE_SPACE again");
- }
- *num_luns = stats.num_luns;
- free(lun_stats);
- lun_stats = NULL;
+ if (more_space_count >= 2)
+ errx(1, "CTL_GET_*_STATS returned NEED_MORE_SPACE
again");
+ *alloc_items = get_stats.num_items * 5 / 4;
+ free(*stats);
+ *stats = NULL;
more_space_count++;
goto retry;
break; /* NOTREACHED */
default:
- errx(1, "unknown status %d returned from CTL_GETSTATS ioctl",
- stats.status);
+ errx(1, "CTL_GET_*_STATS ioctl returned unknown status %d",
+ get_stats.status);
break;
}
- *xlun_stats = lun_stats;
- *num_luns = stats.num_luns;
- cur_time->tv_sec = stats.timestamp.tv_sec;
- cur_time->tv_nsec = stats.timestamp.tv_nsec;
- if (stats.flags & CTL_STATS_FLAG_TIME_VALID)
- *flags |= CTLSTAT_FLAG_LUN_TIME_VALID;
+ *num_items = get_stats.fill_len / sizeof(**stats);
+ cur_time->tv_sec = get_stats.timestamp.tv_sec;
+ cur_time->tv_nsec = get_stats.timestamp.tv_nsec;
+ if (get_stats.flags & CTL_STATS_FLAG_TIME_VALID)
+ *flags |= CTLSTAT_FLAG_TIME_VALID;
else
- *flags &= ~CTLSTAT_FLAG_LUN_TIME_VALID;
+ *flags &= ~CTLSTAT_FLAG_TIME_VALID;
return (0);
}
@@ -240,14 +232,13 @@ getcpu(struct ctl_cpu_stats *cpu_stats)
}
static void
-compute_stats(struct ctlstat_context *ctx, struct ctl_lun_io_stats *cur_stats,
- struct ctl_lun_io_stats *prev_stats, long double etime,
+compute_stats(struct ctl_io_stats *cur_stats,
+ struct ctl_io_stats *prev_stats, long double etime,
long double *mbsec, long double *kb_per_transfer,
long double *transfers_per_second, long double *ms_per_transfer,
long double *ms_per_dma, long double *dmas_per_second)
{
uint64_t total_bytes = 0, total_operations = 0, total_dmas = 0;
- uint32_t port;
struct bintime total_time_bt, total_dma_bt;
struct timespec total_time_ts, total_dma_ts;
int i;
@@ -256,31 +247,18 @@ compute_stats(struct ctlstat_context *ct
bzero(&total_dma_bt, sizeof(total_dma_bt));
bzero(&total_time_ts, sizeof(total_time_ts));
bzero(&total_dma_ts, sizeof(total_dma_ts));
- for (port = 0; port < CTL_MAX_PORTS; port++) {
- if (F_PORTMASK(ctx) &&
- bit_test(ctx->port_mask, port) == 0)
- continue;
- for (i = 0; i < CTL_STATS_NUM_TYPES; i++) {
- total_bytes += cur_stats->ports[port].bytes[i];
- total_operations +=
- cur_stats->ports[port].operations[i];
- total_dmas += cur_stats->ports[port].num_dmas[i];
- bintime_add(&total_time_bt,
- &cur_stats->ports[port].time[i]);
- bintime_add(&total_dma_bt,
- &cur_stats->ports[port].dma_time[i]);
- if (prev_stats != NULL) {
- total_bytes -=
- prev_stats->ports[port].bytes[i];
- total_operations -=
- prev_stats->ports[port].operations[i];
- total_dmas -=
- prev_stats->ports[port].num_dmas[i];
- bintime_sub(&total_time_bt,
- &prev_stats->ports[port].time[i]);
- bintime_sub(&total_dma_bt,
- &prev_stats->ports[port].dma_time[i]);
- }
+ for (i = 0; i < CTL_STATS_NUM_TYPES; i++) {
+ total_bytes += cur_stats->bytes[i];
+ total_operations += cur_stats->operations[i];
+ total_dmas += cur_stats->dmas[i];
+ bintime_add(&total_time_bt, &cur_stats->time[i]);
+ bintime_add(&total_dma_bt, &cur_stats->dma_time[i]);
+ if (prev_stats != NULL) {
+ total_bytes -= prev_stats->bytes[i];
+ total_operations -= prev_stats->operations[i];
+ total_dmas -= prev_stats->dmas[i];
+ bintime_sub(&total_time_bt, &prev_stats->time[i]);
+ bintime_sub(&total_dma_bt, &prev_stats->dma_time[i]);
}
}
@@ -340,35 +318,25 @@ compute_stats(struct ctlstat_context *ct
static const char *iotypes[] = {"NO IO", "READ", "WRITE"};
static void
-ctlstat_dump(struct ctlstat_context *ctx) {
- int iotype, lun, port;
- struct ctl_lun_io_stats *stats = ctx->cur_lun_stats;
+ctlstat_dump(struct ctlstat_context *ctx)
+{
+ int iotype, i;
+ struct ctl_io_stats *stats = ctx->cur_stats;
- for (lun = 0; lun < ctx->num_luns;lun++) {
- if (F_LUNMASK(ctx) && bit_test(ctx->lun_mask, lun) == 0)
+ for (i = 0; i < ctx->cur_items;i++) {
+ if (F_MASK(ctx) && bit_test(ctx->item_mask, i) == 0)
continue;
- printf("lun %d\n", lun);
- for (port = 0; port < CTL_MAX_PORTS; port++) {
- if (F_PORTMASK(ctx) &&
- bit_test(ctx->port_mask, port) == 0)
- continue;
- printf(" port %d\n",
- stats[lun].ports[port].targ_port);
- for (iotype = 0; iotype < CTL_STATS_NUM_TYPES;
- iotype++) {
- printf(" io type %d (%s)\n", iotype,
- iotypes[iotype]);
- printf(" bytes %ju\n", (uintmax_t)
- stats[lun].ports[port].bytes[iotype]);
- printf(" operations %ju\n", (uintmax_t)
- stats[lun].ports[port].operations[iotype]);
- PRINT_BINTIME(" io time",
- stats[lun].ports[port].time[iotype]);
- printf(" num dmas %ju\n", (uintmax_t)
- stats[lun].ports[port].num_dmas[iotype]);
- PRINT_BINTIME(" dma time",
- stats[lun].ports[port].dma_time[iotype]);
- }
+ printf("%s %d\n", F_PORTS(ctx) ? "port" : "lun", stats[i].item);
+ for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) {
+ printf(" io type %d (%s)\n", iotype, iotypes[iotype]);
+ printf(" bytes %ju\n", (uintmax_t)
+ stats[i].bytes[iotype]);
+ printf(" operations %ju\n", (uintmax_t)
+ stats[i].operations[iotype]);
+ printf(" dmas %ju\n", (uintmax_t)
+ stats[i].dmas[iotype]);
+ PRINT_BINTIME(" io time", stats[i].time[iotype]);
+ PRINT_BINTIME(" dma time", stats[i].dma_time[iotype]);
}
}
}
@@ -378,63 +346,49 @@ ctlstat_dump(struct ctlstat_context *ctx
(uintmax_t)(((bt).frac >> 32) * 1000000 >> 32))
static void
ctlstat_json(struct ctlstat_context *ctx) {
- int iotype, lun, port;
- struct ctl_lun_io_stats *stats = ctx->cur_lun_stats;
+ int iotype, i;
+ struct ctl_io_stats *stats = ctx->cur_stats;
- printf("{\"luns\":[");
- for (lun = 0; lun < ctx->num_luns; lun++) {
- if (F_LUNMASK(ctx) && bit_test(ctx->lun_mask, lun) == 0)
+ printf("{\"%s\":[", F_PORTS(ctx) ? "ports" : "luns");
+ for (i = 0; i < ctx->cur_items; i++) {
+ if (F_MASK(ctx) && bit_test(ctx->item_mask, i) == 0)
continue;
- printf("{\"ports\":[");
- for (port = 0; port < CTL_MAX_PORTS;port++) {
- if (F_PORTMASK(ctx) &&
- bit_test(ctx->port_mask, port) == 0)
- continue;
- printf("{\"num\":%d,\"io\":[",
- stats[lun].ports[port].targ_port);
- for (iotype = 0; iotype < CTL_STATS_NUM_TYPES;
- iotype++) {
- printf("{\"type\":\"%s\",", iotypes[iotype]);
- printf("\"bytes\":%ju,", (uintmax_t)stats[
- lun].ports[port].bytes[iotype]);
- printf("\"operations\":%ju,", (uintmax_t)stats[
- lun].ports[port].operations[iotype]);
- JSON_BINTIME("io time",
- stats[lun].ports[port].time[iotype]);
- JSON_BINTIME("dma time",
- stats[lun].ports[port].dma_time[iotype]);
- printf("\"num dmas\":%ju}", (uintmax_t)
- stats[lun].ports[port].num_dmas[iotype]);
- if (iotype < (CTL_STATS_NUM_TYPES - 1))
- printf(","); /* continue io array */
- }
- printf("]}"); /* close port */
- if (port < (CTL_MAX_PORTS - 1))
- printf(","); /* continue port array */
+ printf("{\"num\":%d,\"io\":[",
+ stats[i].item);
+ for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) {
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
