kpriv/linux-hardened
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

xen/blkfront: split per device io_lock

Browse source 
After patch "xen/blkfront: separate per ring information out of device
info", per-ring data is protected by a per-device lock ('io_lock').

This is not a good way and will effect the scalability, so introduce a
per-ring lock ('ring_lock').

The old 'io_lock' is renamed to 'dev_lock' which protects the ->grants list and
->persistent_gnts_c which are shared by all rings.

Note that in 'blkfront_probe' the 'blkfront_info' is setup via kzalloc
so setting ->persistent_gnts_c to zero is not needed.

Signed-off-by: Bob Liu <bob.liu@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
This commit is contained in:
Bob Liu 2015-11-14 11:12:13 +08:00 committed by Konrad Rzeszutek Wilk
parent 3df0e50599
commit 11659569f7
1 changed files with 47 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

73
drivers/block/xen-blkfront.c
View file

@ -125,6 +125,8 @@ MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the
* depending on how many hardware queues/rings to be used. * depending on how many hardware queues/rings to be used.
*/ */
struct blkfront_ring_info { struct blkfront_ring_info {
/* Lock to protect data in every ring buffer. */
spinlock_t ring_lock;
struct blkif_front_ring ring; struct blkif_front_ring ring;
unsigned int ring_ref[XENBUS_MAX_RING_GRANTS]; unsigned int ring_ref[XENBUS_MAX_RING_GRANTS];
unsigned int evtchn, irq; unsigned int evtchn, irq;
@ -143,7 +145,6 @@ struct blkfront_ring_info {
*/ */
struct blkfront_info struct blkfront_info
{ {
spinlock_t io_lock;
struct mutex mutex; struct mutex mutex;
struct xenbus_device *xbdev; struct xenbus_device *xbdev;
struct gendisk *gd; struct gendisk *gd;
@ -153,6 +154,11 @@ struct blkfront_info
/* Number of pages per ring buffer. */ /* Number of pages per ring buffer. */
unsigned int nr_ring_pages; unsigned int nr_ring_pages;
struct request_queue *rq; struct request_queue *rq;
/*
* Lock to protect info->grants list and persistent_gnts_c shared by all
* rings.
*/
spinlock_t dev_lock;
struct list_head grants; struct list_head grants;
unsigned int persistent_gnts_c; unsigned int persistent_gnts_c;
unsigned int feature_flush; unsigned int feature_flush;
@ -258,7 +264,9 @@ static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num)
} }
gnt_list_entry->gref = GRANT_INVALID_REF; gnt_list_entry->gref = GRANT_INVALID_REF;
spin_lock_irq(&info->dev_lock);
list_add(&gnt_list_entry->node, &info->grants); list_add(&gnt_list_entry->node, &info->grants);
spin_unlock_irq(&info->dev_lock);
i++; i++;
} }
@ -267,7 +275,9 @@ static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num)
out_of_memory: out_of_memory:
list_for_each_entry_safe(gnt_list_entry, n, list_for_each_entry_safe(gnt_list_entry, n,
&info->grants, node) { &info->grants, node) {
spin_lock_irq(&info->dev_lock);
list_del(&gnt_list_entry->node); list_del(&gnt_list_entry->node);
spin_unlock_irq(&info->dev_lock);
if (info->feature_persistent) if (info->feature_persistent)
__free_page(gnt_list_entry->page); __free_page(gnt_list_entry->page);
kfree(gnt_list_entry); kfree(gnt_list_entry);
@ -280,7 +290,9 @@ out_of_memory:
static struct grant *get_free_grant(struct blkfront_info *info) static struct grant *get_free_grant(struct blkfront_info *info)
{ {
struct grant *gnt_list_entry; struct grant *gnt_list_entry;
unsigned long flags;
spin_lock_irqsave(&info->dev_lock, flags);
BUG_ON(list_empty(&info->grants)); BUG_ON(list_empty(&info->grants));
gnt_list_entry = list_first_entry(&info->grants, struct grant, gnt_list_entry = list_first_entry(&info->grants, struct grant,
node); node);
@ -288,6 +300,7 @@ static struct grant *get_free_grant(struct blkfront_info *info)
if (gnt_list_entry->gref != GRANT_INVALID_REF) if (gnt_list_entry->gref != GRANT_INVALID_REF)
info->persistent_gnts_c--; info->persistent_gnts_c--;
spin_unlock_irqrestore(&info->dev_lock, flags);
return gnt_list_entry; return gnt_list_entry;
} }
@ -757,11 +770,11 @@ static inline bool blkif_request_flush_invalid(struct request *req,
static int blkif_queue_rq(struct blk_mq_hw_ctx *hctx, static int blkif_queue_rq(struct blk_mq_hw_ctx *hctx,
const struct blk_mq_queue_data *qd) const struct blk_mq_queue_data *qd)
{ {
unsigned long flags;
struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)hctx->driver_data; struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)hctx->driver_data;
struct blkfront_info *info = rinfo->dev_info;
blk_mq_start_request(qd->rq); blk_mq_start_request(qd->rq);
spin_lock_irq(&info->io_lock); spin_lock_irqsave(&rinfo->ring_lock, flags);
if (RING_FULL(&rinfo->ring)) if (RING_FULL(&rinfo->ring))
goto out_busy; goto out_busy;
@ -772,15 +785,15 @@ static int blkif_queue_rq(struct blk_mq_hw_ctx *hctx,
goto out_busy; goto out_busy;
flush_requests(rinfo); flush_requests(rinfo);
spin_unlock_irq(&info->io_lock); spin_unlock_irqrestore(&rinfo->ring_lock, flags);
return BLK_MQ_RQ_QUEUE_OK; return BLK_MQ_RQ_QUEUE_OK;
out_err: out_err:
spin_unlock_irq(&info->io_lock); spin_unlock_irqrestore(&rinfo->ring_lock, flags);
return BLK_MQ_RQ_QUEUE_ERROR; return BLK_MQ_RQ_QUEUE_ERROR;
out_busy: out_busy:
spin_unlock_irq(&info->io_lock); spin_unlock_irqrestore(&rinfo->ring_lock, flags);
blk_mq_stop_hw_queue(hctx); blk_mq_stop_hw_queue(hctx);
return BLK_MQ_RQ_QUEUE_BUSY; return BLK_MQ_RQ_QUEUE_BUSY;
} }
@ -1082,21 +1095,28 @@ static void xlvbd_release_gendisk(struct blkfront_info *info)
info->gd = NULL; info->gd = NULL;
} }
/* Must be called with io_lock holded */ /* Already hold rinfo->ring_lock. */
static void kick_pending_request_queues(struct blkfront_ring_info *rinfo) static inline void kick_pending_request_queues_locked(struct blkfront_ring_info *rinfo)
{ {
if (!RING_FULL(&rinfo->ring)) if (!RING_FULL(&rinfo->ring))
blk_mq_start_stopped_hw_queues(rinfo->dev_info->rq, true); blk_mq_start_stopped_hw_queues(rinfo->dev_info->rq, true);
} }
static void kick_pending_request_queues(struct blkfront_ring_info *rinfo)
{
unsigned long flags;
spin_lock_irqsave(&rinfo->ring_lock, flags);
kick_pending_request_queues_locked(rinfo);
spin_unlock_irqrestore(&rinfo->ring_lock, flags);
}
static void blkif_restart_queue(struct work_struct *work) static void blkif_restart_queue(struct work_struct *work)
{ {
struct blkfront_ring_info *rinfo = container_of(work, struct blkfront_ring_info, work); struct blkfront_ring_info *rinfo = container_of(work, struct blkfront_ring_info, work);
spin_lock_irq(&rinfo->dev_info->io_lock);
if (rinfo->dev_info->connected == BLKIF_STATE_CONNECTED) if (rinfo->dev_info->connected == BLKIF_STATE_CONNECTED)
kick_pending_request_queues(rinfo); kick_pending_request_queues(rinfo);
spin_unlock_irq(&rinfo->dev_info->io_lock);
} }
static void blkif_free_ring(struct blkfront_ring_info *rinfo) static void blkif_free_ring(struct blkfront_ring_info *rinfo)
@ -1188,7 +1208,6 @@ static void blkif_free(struct blkfront_info *info, int suspend)
unsigned int i; unsigned int i;
/* Prevent new requests being issued until we fix things up. */ /* Prevent new requests being issued until we fix things up. */
spin_lock_irq(&info->io_lock);
info->connected = suspend ? info->connected = suspend ?
BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED; BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED;
/* No more blkif_request(). */ /* No more blkif_request(). */
@ -1196,6 +1215,7 @@ static void blkif_free(struct blkfront_info *info, int suspend)
blk_mq_stop_hw_queues(info->rq); blk_mq_stop_hw_queues(info->rq);
/* Remove all persistent grants */ /* Remove all persistent grants */
spin_lock_irq(&info->dev_lock);
if (!list_empty(&info->grants)) { if (!list_empty(&info->grants)) {
list_for_each_entry_safe(persistent_gnt, n, list_for_each_entry_safe(persistent_gnt, n,
&info->grants, node) { &info->grants, node) {
@ -1211,6 +1231,7 @@ static void blkif_free(struct blkfront_info *info, int suspend)
} }
} }
BUG_ON(info->persistent_gnts_c != 0); BUG_ON(info->persistent_gnts_c != 0);
spin_unlock_irq(&info->dev_lock);
for (i = 0; i < info->nr_rings; i++) for (i = 0; i < info->nr_rings; i++)
blkif_free_ring(&info->rinfo[i]); blkif_free_ring(&info->rinfo[i]);
@ -1218,7 +1239,6 @@ static void blkif_free(struct blkfront_info *info, int suspend)
kfree(info->rinfo); kfree(info->rinfo);
info->rinfo = NULL; info->rinfo = NULL;
info->nr_rings = 0; info->nr_rings = 0;
spin_unlock_irq(&info->io_lock);
} }
struct copy_from_grant { struct copy_from_grant {
@ -1253,6 +1273,7 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_ring_info *ri
int i = 0; int i = 0;
struct scatterlist *sg; struct scatterlist *sg;
int num_sg, num_grant; int num_sg, num_grant;
unsigned long flags;
struct blkfront_info *info = rinfo->dev_info; struct blkfront_info *info = rinfo->dev_info;
struct copy_from_grant data = { struct copy_from_grant data = {
.s = s, .s = s,
@ -1291,8 +1312,10 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_ring_info *ri
if (!info->feature_persistent) if (!info->feature_persistent)
pr_alert_ratelimited("backed has not unmapped grant: %u\n", pr_alert_ratelimited("backed has not unmapped grant: %u\n",
s->grants_used[i]->gref); s->grants_used[i]->gref);
spin_lock_irqsave(&info->dev_lock, flags);
list_add(&s->grants_used[i]->node, &info->grants); list_add(&s->grants_used[i]->node, &info->grants);
info->persistent_gnts_c++; info->persistent_gnts_c++;
spin_unlock_irqrestore(&info->dev_lock, flags);
} else { } else {
/* /*
* If the grant is not mapped by the backend we end the * If the grant is not mapped by the backend we end the
@ -1302,7 +1325,9 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_ring_info *ri
*/ */
gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL); gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL);
s->grants_used[i]->gref = GRANT_INVALID_REF; s->grants_used[i]->gref = GRANT_INVALID_REF;
spin_lock_irqsave(&info->dev_lock, flags);
list_add_tail(&s->grants_used[i]->node, &info->grants); list_add_tail(&s->grants_used[i]->node, &info->grants);
spin_unlock_irqrestore(&info->dev_lock, flags);
} }
} }
if (s->req.operation == BLKIF_OP_INDIRECT) { if (s->req.operation == BLKIF_OP_INDIRECT) {
@ -1311,8 +1336,10 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_ring_info *ri
if (!info->feature_persistent) if (!info->feature_persistent)
pr_alert_ratelimited("backed has not unmapped grant: %u\n", pr_alert_ratelimited("backed has not unmapped grant: %u\n",
s->indirect_grants[i]->gref); s->indirect_grants[i]->gref);
spin_lock_irqsave(&info->dev_lock, flags);
list_add(&s->indirect_grants[i]->node, &info->grants); list_add(&s->indirect_grants[i]->node, &info->grants);
info->persistent_gnts_c++; info->persistent_gnts_c++;
spin_unlock_irqrestore(&info->dev_lock, flags);
} else { } else {
struct page *indirect_page; struct page *indirect_page;
@ -1326,7 +1353,9 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_ring_info *ri
list_add(&indirect_page->lru, &rinfo->indirect_pages); list_add(&indirect_page->lru, &rinfo->indirect_pages);
} }
s->indirect_grants[i]->gref = GRANT_INVALID_REF; s->indirect_grants[i]->gref = GRANT_INVALID_REF;
spin_lock_irqsave(&info->dev_lock, flags);
list_add_tail(&s->indirect_grants[i]->node, &info->grants); list_add_tail(&s->indirect_grants[i]->node, &info->grants);
spin_unlock_irqrestore(&info->dev_lock, flags);
} }
} }
} }
@ -1342,13 +1371,10 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
struct blkfront_info *info = rinfo->dev_info; struct blkfront_info *info = rinfo->dev_info;
int error; int error;
spin_lock_irqsave(&info->io_lock, flags); if (unlikely(info->connected != BLKIF_STATE_CONNECTED))
if (unlikely(info->connected != BLKIF_STATE_CONNECTED)) {
spin_unlock_irqrestore(&info->io_lock, flags);
return IRQ_HANDLED; return IRQ_HANDLED;
}
spin_lock_irqsave(&rinfo->ring_lock, flags);
again: again:
rp = rinfo->ring.sring->rsp_prod; rp = rinfo->ring.sring->rsp_prod;
rmb(); /* Ensure we see queued responses up to 'rp'. */ rmb(); /* Ensure we see queued responses up to 'rp'. */
@ -1439,9 +1465,9 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id)
} else } else
rinfo->ring.sring->rsp_event = i + 1; rinfo->ring.sring->rsp_event = i + 1;
kick_pending_request_queues(rinfo); kick_pending_request_queues_locked(rinfo);
spin_unlock_irqrestore(&info->io_lock, flags); spin_unlock_irqrestore(&rinfo->ring_lock, flags);
return IRQ_HANDLED; return IRQ_HANDLED;
} }
@ -1690,14 +1716,14 @@ static int blkfront_probe(struct xenbus_device *dev,
INIT_LIST_HEAD(&rinfo->indirect_pages); INIT_LIST_HEAD(&rinfo->indirect_pages);
rinfo->dev_info = info; rinfo->dev_info = info;
INIT_WORK(&rinfo->work, blkif_restart_queue); INIT_WORK(&rinfo->work, blkif_restart_queue);
spin_lock_init(&rinfo->ring_lock);
} }
mutex_init(&info->mutex); mutex_init(&info->mutex);
spin_lock_init(&info->io_lock); spin_lock_init(&info->dev_lock);
info->xbdev = dev; info->xbdev = dev;
info->vdevice = vdevice; info->vdevice = vdevice;
INIT_LIST_HEAD(&info->grants); INIT_LIST_HEAD(&info->grants);
info->persistent_gnts_c = 0;
info->connected = BLKIF_STATE_DISCONNECTED; info->connected = BLKIF_STATE_DISCONNECTED;
/* Front end dir is a number, which is used as the id. */ /* Front end dir is a number, which is used as the id. */
@ -1790,8 +1816,6 @@ static int blkif_recover(struct blkfront_info *info)
} }
xenbus_switch_state(info->xbdev, XenbusStateConnected); xenbus_switch_state(info->xbdev, XenbusStateConnected);
spin_lock_irq(&info->io_lock);
/* Now safe for us to use the shared ring */ /* Now safe for us to use the shared ring */
info->connected = BLKIF_STATE_CONNECTED; info->connected = BLKIF_STATE_CONNECTED;
@ -1809,7 +1833,6 @@ static int blkif_recover(struct blkfront_info *info)
BUG_ON(req->nr_phys_segments > segs); BUG_ON(req->nr_phys_segments > segs);
blk_mq_requeue_request(req); blk_mq_requeue_request(req);
} }
spin_unlock_irq(&info->io_lock);
blk_mq_kick_requeue_list(info->rq); blk_mq_kick_requeue_list(info->rq);
while ((bio = bio_list_pop(&bio_list)) != NULL) { while ((bio = bio_list_pop(&bio_list)) != NULL) {
@ -2158,11 +2181,9 @@ static void blkfront_connect(struct blkfront_info *info)
xenbus_switch_state(info->xbdev, XenbusStateConnected); xenbus_switch_state(info->xbdev, XenbusStateConnected);
/* Kick pending requests. */ /* Kick pending requests. */
spin_lock_irq(&info->io_lock);
info->connected = BLKIF_STATE_CONNECTED; info->connected = BLKIF_STATE_CONNECTED;
for (i = 0; i < info->nr_rings; i++) for (i = 0; i < info->nr_rings; i++)
kick_pending_request_queues(&info->rinfo[i]); kick_pending_request_queues(&info->rinfo[i]);
spin_unlock_irq(&info->io_lock);
add_disk(info->gd); add_disk(info->gd);