mm: write iovec cleanup

Hide some of the open-coded nr_segs tests into the iovec helpers.  This is all
to simplify generic_file_buffered_write, because that gets more complex in the
next patch.

Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Nick Piggin 2007-10-16 01:24:58 -07:00 committed by Linus Torvalds
parent eb2be18931
commit 4a9e5ef1f4
3 changed files with 77 additions and 96 deletions

View file

@ -1823,12 +1823,7 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
/* /*
* handle partial DIO write. Adjust cur_iov if needed. * handle partial DIO write. Adjust cur_iov if needed.
*/ */
if (likely(nr_segs == 1)) filemap_set_next_iovec(&cur_iov, nr_segs, &iov_offset, written);
buf = iov->iov_base + written;
else {
filemap_set_next_iovec(&cur_iov, &iov_offset, written);
buf = cur_iov->iov_base + iov_offset;
}
do { do {
struct page *page; struct page *page;
@ -1838,6 +1833,7 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
size_t bytes; /* Bytes to write to page */ size_t bytes; /* Bytes to write to page */
size_t copied; /* Bytes copied from user */ size_t copied; /* Bytes copied from user */
buf = cur_iov->iov_base + iov_offset;
offset = (pos & (PAGE_CACHE_SIZE - 1)); offset = (pos & (PAGE_CACHE_SIZE - 1));
index = pos >> PAGE_CACHE_SHIFT; index = pos >> PAGE_CACHE_SHIFT;
bytes = PAGE_CACHE_SIZE - offset; bytes = PAGE_CACHE_SIZE - offset;
@ -1869,13 +1865,10 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
if (unlikely(status)) if (unlikely(status))
goto fs_write_aop_error; goto fs_write_aop_error;
if (likely(nr_segs == 1)) copied = filemap_copy_from_user(page, offset,
copied = filemap_copy_from_user(page, offset, cur_iov, nr_segs, iov_offset, bytes);
buf, bytes);
else
copied = filemap_copy_from_user_iovec(page, offset,
cur_iov, iov_offset, bytes);
flush_dcache_page(page); flush_dcache_page(page);
status = a_ops->commit_write(file, page, offset, offset+bytes); status = a_ops->commit_write(file, page, offset, offset+bytes);
if (unlikely(status < 0 || status == AOP_TRUNCATED_PAGE)) if (unlikely(status < 0 || status == AOP_TRUNCATED_PAGE))
goto fs_write_aop_error; goto fs_write_aop_error;
@ -1886,20 +1879,11 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
if (unlikely(status > 0)) /* filesystem did partial write */ if (unlikely(status > 0)) /* filesystem did partial write */
copied = status; copied = status;
if (likely(copied > 0)) { written += copied;
written += copied; count -= copied;
count -= copied; pos += copied;
pos += copied; filemap_set_next_iovec(&cur_iov, nr_segs, &iov_offset, copied);
buf += copied;
if (unlikely(nr_segs > 1)) {
filemap_set_next_iovec(&cur_iov,
&iov_offset, copied);
if (count)
buf = cur_iov->iov_base + iov_offset;
} else {
iov_offset += copied;
}
}
unlock_page(page); unlock_page(page);
mark_page_accessed(page); mark_page_accessed(page);
page_cache_release(page); page_cache_release(page);

View file

@ -22,82 +22,82 @@ __filemap_copy_from_user_iovec_inatomic(char *vaddr,
/* /*
* Copy as much as we can into the page and return the number of bytes which * Copy as much as we can into the page and return the number of bytes which
* were sucessfully copied. If a fault is encountered then clear the page * were sucessfully copied. If a fault is encountered then return the number of
* out to (offset+bytes) and return the number of bytes which were copied. * bytes which were copied.
*
* NOTE: For this to work reliably we really want copy_from_user_inatomic_nocache
* to *NOT* zero any tail of the buffer that it failed to copy. If it does,
* and if the following non-atomic copy succeeds, then there is a small window
* where the target page contains neither the data before the write, nor the
* data after the write (it contains zero). A read at this time will see
* data that is inconsistent with any ordering of the read and the write.
* (This has been detected in practice).
*/ */
static inline size_t static inline size_t
filemap_copy_from_user(struct page *page, unsigned long offset, filemap_copy_from_user_atomic(struct page *page, unsigned long offset,
const char __user *buf, unsigned bytes) const struct iovec *iov, unsigned long nr_segs,
{ size_t base, size_t bytes)
char *kaddr;
int left;
kaddr = kmap_atomic(page, KM_USER0);
left = __copy_from_user_inatomic_nocache(kaddr + offset, buf, bytes);
kunmap_atomic(kaddr, KM_USER0);
if (left != 0) {
/* Do it the slow way */
kaddr = kmap(page);
left = __copy_from_user_nocache(kaddr + offset, buf, bytes);
kunmap(page);
}
return bytes - left;
}
/*
* This has the same sideeffects and return value as filemap_copy_from_user().
* The difference is that on a fault we need to memset the remainder of the
* page (out to offset+bytes), to emulate filemap_copy_from_user()'s
* single-segment behaviour.
*/
static inline size_t
filemap_copy_from_user_iovec(struct page *page, unsigned long offset,
const struct iovec *iov, size_t base, size_t bytes)
{ {
char *kaddr; char *kaddr;
size_t copied; size_t copied;
kaddr = kmap_atomic(page, KM_USER0); kaddr = kmap_atomic(page, KM_USER0);
copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset, iov, if (likely(nr_segs == 1)) {
base, bytes); int left;
kunmap_atomic(kaddr, KM_USER0); char __user *buf = iov->iov_base + base;
if (copied != bytes) { left = __copy_from_user_inatomic_nocache(kaddr + offset,
kaddr = kmap(page); buf, bytes);
copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset, iov, copied = bytes - left;
base, bytes); } else {
if (bytes - copied) copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset,
memset(kaddr + offset + copied, 0, bytes - copied); iov, base, bytes);
kunmap(page);
} }
kunmap_atomic(kaddr, KM_USER0);
return copied;
}
/*
* This has the same sideeffects and return value as
* filemap_copy_from_user_atomic().
* The difference is that it attempts to resolve faults.
*/
static inline size_t
filemap_copy_from_user(struct page *page, unsigned long offset,
const struct iovec *iov, unsigned long nr_segs,
size_t base, size_t bytes)
{
char *kaddr;
size_t copied;
kaddr = kmap(page);
if (likely(nr_segs == 1)) {
int left;
char __user *buf = iov->iov_base + base;
left = __copy_from_user_nocache(kaddr + offset, buf, bytes);
copied = bytes - left;
} else {
copied = __filemap_copy_from_user_iovec_inatomic(kaddr + offset,
iov, base, bytes);
}
kunmap(page);
return copied; return copied;
} }
static inline void static inline void
filemap_set_next_iovec(const struct iovec **iovp, size_t *basep, size_t bytes) filemap_set_next_iovec(const struct iovec **iovp, unsigned long nr_segs,
size_t *basep, size_t bytes)
{ {
const struct iovec *iov = *iovp; if (likely(nr_segs == 1)) {
size_t base = *basep; *basep += bytes;
} else {
const struct iovec *iov = *iovp;
size_t base = *basep;
while (bytes) { while (bytes) {
int copy = min(bytes, iov->iov_len - base); int copy = min(bytes, iov->iov_len - base);
bytes -= copy; bytes -= copy;
base += copy; base += copy;
if (iov->iov_len == base) { if (iov->iov_len == base) {
iov++; iov++;
base = 0; base = 0;
}
} }
*iovp = iov;
*basep = base;
} }
*iovp = iov;
*basep = base;
} }
#endif #endif

View file

@ -15,7 +15,6 @@
#include <linux/rmap.h> #include <linux/rmap.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include "filemap.h"
/* /*
* We do use our own empty page to avoid interference with other users * We do use our own empty page to avoid interference with other users
@ -288,6 +287,7 @@ __xip_file_write(struct file *filp, const char __user *buf,
unsigned long index; unsigned long index;
unsigned long offset; unsigned long offset;
size_t copied; size_t copied;
char *kaddr;
offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */ offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
index = pos >> PAGE_CACHE_SHIFT; index = pos >> PAGE_CACHE_SHIFT;
@ -295,14 +295,6 @@ __xip_file_write(struct file *filp, const char __user *buf,
if (bytes > count) if (bytes > count)
bytes = count; bytes = count;
/*
* Bring in the user page that we will copy from _first_.
* Otherwise there's a nasty deadlock on copying from the
* same page as we're writing to, without it being marked
* up-to-date.
*/
fault_in_pages_readable(buf, bytes);
page = a_ops->get_xip_page(mapping, page = a_ops->get_xip_page(mapping,
index*(PAGE_SIZE/512), 0); index*(PAGE_SIZE/512), 0);
if (IS_ERR(page) && (PTR_ERR(page) == -ENODATA)) { if (IS_ERR(page) && (PTR_ERR(page) == -ENODATA)) {
@ -319,8 +311,13 @@ __xip_file_write(struct file *filp, const char __user *buf,
break; break;
} }
copied = filemap_copy_from_user(page, offset, buf, bytes); fault_in_pages_readable(buf, bytes);
kaddr = kmap_atomic(page, KM_USER0);
copied = bytes -
__copy_from_user_inatomic_nocache(kaddr, buf, bytes);
kunmap_atomic(kaddr, KM_USER0);
flush_dcache_page(page); flush_dcache_page(page);
if (likely(copied > 0)) { if (likely(copied > 0)) {
status = copied; status = copied;