drbd: allow petabyte storage on 64bit arch
Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com> Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
This commit is contained in:
parent
19f843aa08
commit
4b0715f096
5 changed files with 140 additions and 83 deletions
|
@ -37,10 +37,46 @@
|
|||
* convention:
|
||||
* function name drbd_bm_... => used elsewhere, "public".
|
||||
* function name bm_... => internal to implementation, "private".
|
||||
*/
|
||||
|
||||
* Note that since find_first_bit returns int, at the current granularity of
|
||||
* the bitmap (4KB per byte), this implementation "only" supports up to
|
||||
* 1<<(32+12) == 16 TB...
|
||||
|
||||
/*
|
||||
* LIMITATIONS:
|
||||
* We want to support >= peta byte of backend storage, while for now still using
|
||||
* a granularity of one bit per 4KiB of storage.
|
||||
* 1 << 50 bytes backend storage (1 PiB)
|
||||
* 1 << (50 - 12) bits needed
|
||||
* 38 --> we need u64 to index and count bits
|
||||
* 1 << (38 - 3) bitmap bytes needed
|
||||
* 35 --> we still need u64 to index and count bytes
|
||||
* (that's 32 GiB of bitmap for 1 PiB storage)
|
||||
* 1 << (35 - 2) 32bit longs needed
|
||||
* 33 --> we'd even need u64 to index and count 32bit long words.
|
||||
* 1 << (35 - 3) 64bit longs needed
|
||||
* 32 --> we could get away with a 32bit unsigned int to index and count
|
||||
* 64bit long words, but I rather stay with unsigned long for now.
|
||||
* We probably should neither count nor point to bytes or long words
|
||||
* directly, but either by bitnumber, or by page index and offset.
|
||||
* 1 << (35 - 12)
|
||||
* 22 --> we need that much 4KiB pages of bitmap.
|
||||
* 1 << (22 + 3) --> on a 64bit arch,
|
||||
* we need 32 MiB to store the array of page pointers.
|
||||
*
|
||||
* Because I'm lazy, and because the resulting patch was too large, too ugly
|
||||
* and still incomplete, on 32bit we still "only" support 16 TiB (minus some),
|
||||
* (1 << 32) bits * 4k storage.
|
||||
*
|
||||
|
||||
* bitmap storage and IO:
|
||||
* Bitmap is stored little endian on disk, and is kept little endian in
|
||||
* core memory. Currently we still hold the full bitmap in core as long
|
||||
* as we are "attached" to a local disk, which at 32 GiB for 1PiB storage
|
||||
* seems excessive.
|
||||
*
|
||||
* We plan to reduce the amount of in-core bitmap pages by pageing them in
|
||||
* and out against their on-disk location as necessary, but need to make
|
||||
* sure we don't cause too much meta data IO, and must not deadlock in
|
||||
* tight memory situations. This needs some more work.
|
||||
*/
|
||||
|
||||
/*
|
||||
|
@ -56,13 +92,9 @@
|
|||
struct drbd_bitmap {
|
||||
struct page **bm_pages;
|
||||
spinlock_t bm_lock;
|
||||
/* WARNING unsigned long bm_*:
|
||||
* 32bit number of bit offset is just enough for 512 MB bitmap.
|
||||
* it will blow up if we make the bitmap bigger...
|
||||
* not that it makes much sense to have a bitmap that large,
|
||||
* rather change the granularity to 16k or 64k or something.
|
||||
* (that implies other problems, however...)
|
||||
*/
|
||||
|
||||
/* see LIMITATIONS: above */
|
||||
|
||||
unsigned long bm_set; /* nr of set bits; THINK maybe atomic_t? */
|
||||
unsigned long bm_bits;
|
||||
size_t bm_words;
|
||||
|
@ -517,43 +549,39 @@ static void bm_set_surplus(struct drbd_bitmap *b)
|
|||
bm_unmap(p_addr);
|
||||
}
|
||||
|
||||
/* you better not modify the bitmap while this is running,
|
||||
* or its results will be stale */
|
||||
static unsigned long bm_count_bits(struct drbd_bitmap *b)
|
||||
{
|
||||
unsigned long *p_addr, *bm, offset = 0;
|
||||
unsigned long *p_addr;
|
||||
unsigned long bits = 0;
|
||||
unsigned long i, do_now;
|
||||
unsigned long words;
|
||||
unsigned long mask = (1UL << (b->bm_bits & BITS_PER_LONG_MASK)) -1;
|
||||
int idx, last_page, i, last_word;
|
||||
|
||||
/* due to 64bit alignment, the last long on a 32bit arch
|
||||
* may be not used at all. The last used long will likely
|
||||
* be only partially used, always. Don't count those bits,
|
||||
* but mask them out. */
|
||||
words = (b->bm_bits + BITS_PER_LONG - 1) >> LN2_BPL;
|
||||
/* because of the "extra long to catch oob access" we allocate in
|
||||
* drbd_bm_resize, bm_number_of_pages -1 is not necessarily the page
|
||||
* containing the last _relevant_ bitmap word */
|
||||
last_page = bm_bit_to_page_idx(b, b->bm_bits-1);
|
||||
|
||||
while (offset < words) {
|
||||
i = do_now = min_t(size_t, words-offset, LWPP);
|
||||
p_addr = __bm_map_pidx(b, bm_word_to_page_idx(b, offset), KM_USER0);
|
||||
bm = p_addr + MLPP(offset);
|
||||
while (i--) {
|
||||
bits += hweight_long(*bm++);
|
||||
}
|
||||
offset += do_now;
|
||||
if (offset == words) {
|
||||
/* last word may only be partially used,
|
||||
* see also bm_clear_surplus. */
|
||||
i = (1UL << (b->bm_bits & (BITS_PER_LONG-1))) -1;
|
||||
if (i) {
|
||||
bits -= hweight_long(p_addr[do_now-1] & ~i);
|
||||
p_addr[do_now-1] &= i;
|
||||
}
|
||||
/* 32bit arch, may have an unused padding long */
|
||||
if (words != b->bm_words)
|
||||
p_addr[do_now] = 0;
|
||||
}
|
||||
/* all but last page */
|
||||
for (idx = 0; idx < last_page; idx++) {
|
||||
p_addr = __bm_map_pidx(b, idx, KM_USER0);
|
||||
for (i = 0; i < LWPP; i++)
|
||||
bits += hweight_long(p_addr[i]);
|
||||
__bm_unmap(p_addr, KM_USER0);
|
||||
cond_resched();
|
||||
}
|
||||
|
||||
/* last (or only) page */
|
||||
last_word = ((b->bm_bits - 1) & BITS_PER_PAGE_MASK) >> LN2_BPL;
|
||||
p_addr = __bm_map_pidx(b, idx, KM_USER0);
|
||||
for (i = 0; i < last_word; i++)
|
||||
bits += hweight_long(p_addr[i]);
|
||||
p_addr[last_word] &= cpu_to_lel(mask);
|
||||
bits += hweight_long(p_addr[last_word]);
|
||||
/* 32bit arch, may have an unused padding long */
|
||||
if (BITS_PER_LONG == 32 && (last_word & 1) == 0)
|
||||
p_addr[last_word+1] = 0;
|
||||
__bm_unmap(p_addr, KM_USER0);
|
||||
return bits;
|
||||
}
|
||||
|
||||
|
@ -564,8 +592,6 @@ static void bm_memset(struct drbd_bitmap *b, size_t offset, int c, size_t len)
|
|||
unsigned int idx;
|
||||
size_t do_now, end;
|
||||
|
||||
#define BM_SECTORS_PER_BIT (BM_BLOCK_SIZE/512)
|
||||
|
||||
end = offset + len;
|
||||
|
||||
if (end > b->bm_words) {
|
||||
|
@ -645,8 +671,14 @@ int drbd_bm_resize(struct drbd_conf *mdev, sector_t capacity, int set_new_bits)
|
|||
words = ALIGN(bits, 64) >> LN2_BPL;
|
||||
|
||||
if (get_ldev(mdev)) {
|
||||
D_ASSERT((u64)bits <= (((u64)mdev->ldev->md.md_size_sect-MD_BM_OFFSET) << 12));
|
||||
u64 bits_on_disk = ((u64)mdev->ldev->md.md_size_sect-MD_BM_OFFSET) << 12;
|
||||
put_ldev(mdev);
|
||||
if (bits > bits_on_disk) {
|
||||
dev_info(DEV, "bits = %lu\n", bits);
|
||||
dev_info(DEV, "bits_on_disk = %llu\n", bits_on_disk);
|
||||
err = -ENOSPC;
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
/* one extra long to catch off by one errors */
|
||||
|
@ -1113,9 +1145,12 @@ int drbd_bm_write_lazy(struct drbd_conf *mdev, unsigned upper_idx) __must_hold(l
|
|||
* @mdev: DRBD device.
|
||||
* @idx: bitmap page index
|
||||
*
|
||||
* We don't want to special case on logical_block_size of the underlaying
|
||||
* device, so we submit PAGE_SIZE aligned pieces containing the requested enr.
|
||||
* We don't want to special case on logical_block_size of the backend device,
|
||||
* so we submit PAGE_SIZE aligned pieces.
|
||||
* Note that on "most" systems, PAGE_SIZE is 4k.
|
||||
*
|
||||
* In case this becomes an issue on systems with larger PAGE_SIZE,
|
||||
* we may want to change this again to write 4k aligned 4k pieces.
|
||||
*/
|
||||
int drbd_bm_write_page(struct drbd_conf *mdev, unsigned int idx) __must_hold(local)
|
||||
{
|
||||
|
@ -1144,52 +1179,57 @@ int drbd_bm_write_page(struct drbd_conf *mdev, unsigned int idx) __must_hold(loc
|
|||
|
||||
/* NOTE
|
||||
* find_first_bit returns int, we return unsigned long.
|
||||
* should not make much difference anyways, but ...
|
||||
* For this to work on 32bit arch with bitnumbers > (1<<32),
|
||||
* we'd need to return u64, and get a whole lot of other places
|
||||
* fixed where we still use unsigned long.
|
||||
*
|
||||
* this returns a bit number, NOT a sector!
|
||||
*/
|
||||
#define BPP_MASK ((1UL << (PAGE_SHIFT+3)) - 1)
|
||||
static unsigned long __bm_find_next(struct drbd_conf *mdev, unsigned long bm_fo,
|
||||
const int find_zero_bit, const enum km_type km)
|
||||
{
|
||||
struct drbd_bitmap *b = mdev->bitmap;
|
||||
unsigned long i = -1UL;
|
||||
unsigned long *p_addr;
|
||||
unsigned long bit_offset; /* bit offset of the mapped page. */
|
||||
unsigned long bit_offset;
|
||||
unsigned i;
|
||||
|
||||
|
||||
if (bm_fo > b->bm_bits) {
|
||||
dev_err(DEV, "bm_fo=%lu bm_bits=%lu\n", bm_fo, b->bm_bits);
|
||||
bm_fo = DRBD_END_OF_BITMAP;
|
||||
} else {
|
||||
while (bm_fo < b->bm_bits) {
|
||||
/* bit offset of the first bit in the page */
|
||||
bit_offset = bm_fo & ~BPP_MASK;
|
||||
bit_offset = bm_fo & ~BITS_PER_PAGE_MASK;
|
||||
p_addr = __bm_map_pidx(b, bm_bit_to_page_idx(b, bm_fo), km);
|
||||
|
||||
if (find_zero_bit)
|
||||
i = generic_find_next_zero_le_bit(p_addr, PAGE_SIZE*8, bm_fo & BPP_MASK);
|
||||
i = generic_find_next_zero_le_bit(p_addr,
|
||||
PAGE_SIZE*8, bm_fo & BITS_PER_PAGE_MASK);
|
||||
else
|
||||
i = generic_find_next_le_bit(p_addr, PAGE_SIZE*8, bm_fo & BPP_MASK);
|
||||
i = generic_find_next_le_bit(p_addr,
|
||||
PAGE_SIZE*8, bm_fo & BITS_PER_PAGE_MASK);
|
||||
|
||||
__bm_unmap(p_addr, km);
|
||||
if (i < PAGE_SIZE*8) {
|
||||
i = bit_offset + i;
|
||||
if (i >= b->bm_bits)
|
||||
bm_fo = bit_offset + i;
|
||||
if (bm_fo >= b->bm_bits)
|
||||
break;
|
||||
goto found;
|
||||
}
|
||||
bm_fo = bit_offset + PAGE_SIZE*8;
|
||||
}
|
||||
i = -1UL;
|
||||
bm_fo = DRBD_END_OF_BITMAP;
|
||||
}
|
||||
found:
|
||||
return i;
|
||||
return bm_fo;
|
||||
}
|
||||
|
||||
static unsigned long bm_find_next(struct drbd_conf *mdev,
|
||||
unsigned long bm_fo, const int find_zero_bit)
|
||||
{
|
||||
struct drbd_bitmap *b = mdev->bitmap;
|
||||
unsigned long i = -1UL;
|
||||
unsigned long i = DRBD_END_OF_BITMAP;
|
||||
|
||||
ERR_IF(!b) return i;
|
||||
ERR_IF(!b->bm_pages) return i;
|
||||
|
@ -1267,9 +1307,9 @@ static int __bm_change_bits_to(struct drbd_conf *mdev, const unsigned long s,
|
|||
last_page_nr = page_nr;
|
||||
}
|
||||
if (val)
|
||||
c += (0 == generic___test_and_set_le_bit(bitnr & BPP_MASK, p_addr));
|
||||
c += (0 == generic___test_and_set_le_bit(bitnr & BITS_PER_PAGE_MASK, p_addr));
|
||||
else
|
||||
c -= (0 != generic___test_and_clear_le_bit(bitnr & BPP_MASK, p_addr));
|
||||
c -= (0 != generic___test_and_clear_le_bit(bitnr & BITS_PER_PAGE_MASK, p_addr));
|
||||
}
|
||||
if (p_addr)
|
||||
__bm_unmap(p_addr, km);
|
||||
|
@ -1418,7 +1458,7 @@ int drbd_bm_test_bit(struct drbd_conf *mdev, const unsigned long bitnr)
|
|||
bm_print_lock_info(mdev);
|
||||
if (bitnr < b->bm_bits) {
|
||||
p_addr = bm_map_pidx(b, bm_bit_to_page_idx(b, bitnr));
|
||||
i = generic_test_le_bit(bitnr & BPP_MASK, p_addr) ? 1 : 0;
|
||||
i = generic_test_le_bit(bitnr & BITS_PER_PAGE_MASK, p_addr) ? 1 : 0;
|
||||
bm_unmap(p_addr);
|
||||
} else if (bitnr == b->bm_bits) {
|
||||
i = -1;
|
||||
|
@ -1517,13 +1557,15 @@ int drbd_bm_e_weight(struct drbd_conf *mdev, unsigned long enr)
|
|||
return count;
|
||||
}
|
||||
|
||||
/* set all bits covered by the AL-extent al_enr */
|
||||
/* Set all bits covered by the AL-extent al_enr.
|
||||
* Returns number of bits changed. */
|
||||
unsigned long drbd_bm_ALe_set_all(struct drbd_conf *mdev, unsigned long al_enr)
|
||||
{
|
||||
struct drbd_bitmap *b = mdev->bitmap;
|
||||
unsigned long *p_addr, *bm;
|
||||
unsigned long weight;
|
||||
int count, s, e, i, do_now;
|
||||
unsigned long s, e;
|
||||
int count, i, do_now;
|
||||
ERR_IF(!b) return 0;
|
||||
ERR_IF(!b->bm_pages) return 0;
|
||||
|
||||
|
@ -1552,7 +1594,7 @@ unsigned long drbd_bm_ALe_set_all(struct drbd_conf *mdev, unsigned long al_enr)
|
|||
if (e == b->bm_words)
|
||||
b->bm_set -= bm_clear_surplus(b);
|
||||
} else {
|
||||
dev_err(DEV, "start offset (%d) too large in drbd_bm_ALe_set_all\n", s);
|
||||
dev_err(DEV, "start offset (%lu) too large in drbd_bm_ALe_set_all\n", s);
|
||||
}
|
||||
weight = b->bm_set - weight;
|
||||
spin_unlock_irq(&b->bm_lock);
|
||||
|
|
|
@ -1003,9 +1003,9 @@ struct drbd_conf {
|
|||
struct hlist_head *tl_hash;
|
||||
unsigned int tl_hash_s;
|
||||
|
||||
/* blocks to sync in this run [unit BM_BLOCK_SIZE] */
|
||||
/* blocks to resync in this run [unit BM_BLOCK_SIZE] */
|
||||
unsigned long rs_total;
|
||||
/* number of sync IOs that failed in this run */
|
||||
/* number of resync blocks that failed in this run */
|
||||
unsigned long rs_failed;
|
||||
/* Syncer's start time [unit jiffies] */
|
||||
unsigned long rs_start;
|
||||
|
@ -1399,7 +1399,9 @@ struct bm_extent {
|
|||
* you should use 64bit OS for that much storage, anyways. */
|
||||
#define DRBD_MAX_SECTORS_FLEX BM_BIT_TO_SECT(0xffff7fff)
|
||||
#else
|
||||
#define DRBD_MAX_SECTORS_FLEX BM_BIT_TO_SECT(0x1LU << 32)
|
||||
/* we allow up to 1 PiB now on 64bit architecture with "flexible" meta data */
|
||||
#define DRBD_MAX_SECTORS_FLEX (1UL << 51)
|
||||
/* corresponds to (1UL << 38) bits right now. */
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
@ -1419,11 +1421,15 @@ extern int drbd_bm_resize(struct drbd_conf *mdev, sector_t sectors, int set_new
|
|||
extern void drbd_bm_cleanup(struct drbd_conf *mdev);
|
||||
extern void drbd_bm_set_all(struct drbd_conf *mdev);
|
||||
extern void drbd_bm_clear_all(struct drbd_conf *mdev);
|
||||
/* set/clear/test only a few bits at a time */
|
||||
extern int drbd_bm_set_bits(
|
||||
struct drbd_conf *mdev, unsigned long s, unsigned long e);
|
||||
extern int drbd_bm_clear_bits(
|
||||
struct drbd_conf *mdev, unsigned long s, unsigned long e);
|
||||
/* bm_set_bits variant for use while holding drbd_bm_lock */
|
||||
extern int drbd_bm_count_bits(
|
||||
struct drbd_conf *mdev, const unsigned long s, const unsigned long e);
|
||||
/* bm_set_bits variant for use while holding drbd_bm_lock,
|
||||
* may process the whole bitmap in one go */
|
||||
extern void _drbd_bm_set_bits(struct drbd_conf *mdev,
|
||||
const unsigned long s, const unsigned long e);
|
||||
extern int drbd_bm_test_bit(struct drbd_conf *mdev, unsigned long bitnr);
|
||||
|
@ -1436,6 +1442,8 @@ extern unsigned long drbd_bm_ALe_set_all(struct drbd_conf *mdev,
|
|||
extern size_t drbd_bm_words(struct drbd_conf *mdev);
|
||||
extern unsigned long drbd_bm_bits(struct drbd_conf *mdev);
|
||||
extern sector_t drbd_bm_capacity(struct drbd_conf *mdev);
|
||||
|
||||
#define DRBD_END_OF_BITMAP (~(unsigned long)0)
|
||||
extern unsigned long drbd_bm_find_next(struct drbd_conf *mdev, unsigned long bm_fo);
|
||||
/* bm_find_next variants for use while you hold drbd_bm_lock() */
|
||||
extern unsigned long _drbd_bm_find_next(struct drbd_conf *mdev, unsigned long bm_fo);
|
||||
|
@ -1452,8 +1460,6 @@ extern void drbd_bm_get_lel(struct drbd_conf *mdev, size_t offset,
|
|||
|
||||
extern void drbd_bm_lock(struct drbd_conf *mdev, char *why);
|
||||
extern void drbd_bm_unlock(struct drbd_conf *mdev);
|
||||
|
||||
extern int drbd_bm_count_bits(struct drbd_conf *mdev, const unsigned long s, const unsigned long e);
|
||||
/* drbd_main.c */
|
||||
|
||||
extern struct kmem_cache *drbd_request_cache;
|
||||
|
@ -2158,10 +2164,8 @@ extern int _get_ldev_if_state(struct drbd_conf *mdev, enum drbd_disk_state mins)
|
|||
static inline void drbd_get_syncer_progress(struct drbd_conf *mdev,
|
||||
unsigned long *bits_left, unsigned int *per_mil_done)
|
||||
{
|
||||
/*
|
||||
* this is to break it at compile time when we change that
|
||||
* (we may feel 4TB maximum storage per drbd is not enough)
|
||||
*/
|
||||
/* this is to break it at compile time when we change that, in case we
|
||||
* want to support more than (1<<32) bits on a 32bit arch. */
|
||||
typecheck(unsigned long, mdev->rs_total);
|
||||
|
||||
/* note: both rs_total and rs_left are in bits, i.e. in
|
||||
|
@ -2186,10 +2190,19 @@ static inline void drbd_get_syncer_progress(struct drbd_conf *mdev,
|
|||
*bits_left, mdev->rs_total, mdev->rs_failed);
|
||||
*per_mil_done = 0;
|
||||
} else {
|
||||
/* make sure the calculation happens in long context */
|
||||
unsigned long tmp = 1000UL -
|
||||
(*bits_left >> 10)*1000UL
|
||||
/ ((mdev->rs_total >> 10) + 1UL);
|
||||
/* Make sure the division happens in long context.
|
||||
* We allow up to one petabyte storage right now,
|
||||
* at a granularity of 4k per bit that is 2**38 bits.
|
||||
* After shift right and multiplication by 1000,
|
||||
* this should still fit easily into a 32bit long,
|
||||
* so we don't need a 64bit division on 32bit arch.
|
||||
* Note: currently we don't support such large bitmaps on 32bit
|
||||
* arch anyways, but no harm done to be prepared for it here.
|
||||
*/
|
||||
unsigned int shift = mdev->rs_total >= (1ULL << 32) ? 16 : 10;
|
||||
unsigned long left = *bits_left >> shift;
|
||||
unsigned long total = 1UL + (mdev->rs_total >> shift);
|
||||
unsigned long tmp = 1000UL - left * 1000UL/total;
|
||||
*per_mil_done = tmp;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -527,17 +527,19 @@ static void drbd_md_set_sector_offsets(struct drbd_conf *mdev,
|
|||
}
|
||||
}
|
||||
|
||||
/* input size is expected to be in KB */
|
||||
char *ppsize(char *buf, unsigned long long size)
|
||||
{
|
||||
/* Needs 9 bytes at max. */
|
||||
/* Needs 9 bytes at max including trailing NUL:
|
||||
* -1ULL ==> "16384 EB" */
|
||||
static char units[] = { 'K', 'M', 'G', 'T', 'P', 'E' };
|
||||
int base = 0;
|
||||
while (size >= 10000) {
|
||||
while (size >= 10000 && base < sizeof(units)-1) {
|
||||
/* shift + round */
|
||||
size = (size >> 10) + !!(size & (1<<9));
|
||||
base++;
|
||||
}
|
||||
sprintf(buf, "%lu %cB", (long)size, units[base]);
|
||||
sprintf(buf, "%u %cB", (unsigned)size, units[base]);
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
|
|
@ -91,9 +91,9 @@ static void drbd_syncer_progress(struct drbd_conf *mdev, struct seq_file *seq)
|
|||
seq_printf(seq, "sync'ed:");
|
||||
seq_printf(seq, "%3u.%u%% ", res / 10, res % 10);
|
||||
|
||||
/* if more than 1 GB display in MB */
|
||||
if (mdev->rs_total > 0x100000L)
|
||||
seq_printf(seq, "(%lu/%lu)M\n\t",
|
||||
/* if more than a few GB, display in MB */
|
||||
if (mdev->rs_total > (4UL << (30 - BM_BLOCK_SHIFT)))
|
||||
seq_printf(seq, "(%lu/%lu)M",
|
||||
(unsigned long) Bit2KB(rs_left >> 10),
|
||||
(unsigned long) Bit2KB(mdev->rs_total >> 10));
|
||||
else
|
||||
|
|
|
@ -577,7 +577,7 @@ next_sector:
|
|||
size = BM_BLOCK_SIZE;
|
||||
bit = drbd_bm_find_next(mdev, mdev->bm_resync_fo);
|
||||
|
||||
if (bit == -1UL) {
|
||||
if (bit == DRBD_END_OF_BITMAP) {
|
||||
mdev->bm_resync_fo = drbd_bm_bits(mdev);
|
||||
mdev->resync_work.cb = w_resync_inactive;
|
||||
put_ldev(mdev);
|
||||
|
|
Loading…
Reference in a new issue