// SPDX-License-Identifier: GPL-2.0-only
/*
* Userfaultfd unit tests.
*
* Copyright (C) 2015-2023 Red Hat, Inc.
*/
#include "uffd-common.h"
#include <linux/fs.h>
#include <sys/uio.h>
#include "../../../../mm/gup_test.h"
#ifdef __NR_userfaultfd
/* The unit test doesn't need a large or random size, make it 32MB for now */
#define UFFD_TEST_MEM_SIZE (32UL << 20)
#define MEM_ANON BIT_ULL(0)
#define MEM_SHMEM BIT_ULL(1)
#define MEM_SHMEM_PRIVATE BIT_ULL(2)
#define MEM_HUGETLB BIT_ULL(3)
#define MEM_HUGETLB_PRIVATE BIT_ULL(4)
#define MEM_ALL (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \
MEM_HUGETLB | MEM_HUGETLB_PRIVATE)
#define ALIGN_UP(x, align_to) \
((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
struct mem_type {
const char *name;
unsigned int mem_flag;
uffd_test_ops_t *mem_ops;
bool shared;
};
typedef struct mem_type mem_type_t;
mem_type_t mem_types[] = {
{
.name = "anon",
.mem_flag = MEM_ANON,
.mem_ops = &anon_uffd_test_ops,
.shared = false,
},
{
.name = "shmem",
.mem_flag = MEM_SHMEM,
.mem_ops = &shmem_uffd_test_ops,
.shared = true,
},
{
.name = "shmem-private",
.mem_flag = MEM_SHMEM_PRIVATE,
.mem_ops = &shmem_uffd_test_ops,
.shared = false,
},
{
.name = "hugetlb",
.mem_flag = MEM_HUGETLB,
.mem_ops = &hugetlb_uffd_test_ops,
.shared = true,
},
{
.name = "hugetlb-private",
.mem_flag = MEM_HUGETLB_PRIVATE,
.mem_ops = &hugetlb_uffd_test_ops,
.shared = false,
},
};
/* Arguments to be passed over to each uffd unit test */
struct uffd_test_args {
mem_type_t *mem_type;
};
typedef struct uffd_test_args uffd_test_args_t;
/* Returns: UFFD_TEST_* */
typedef void (*uffd_test_fn)(uffd_global_test_opts_t *, uffd_test_args_t *);
typedef struct {
const char *name;
uffd_test_fn uffd_fn;
unsigned int mem_targets;
uint64_t uffd_feature_required;
uffd_test_case_ops_t *test_case_ops;
} uffd_test_case_t;
static char current_test[256];
static void uffd_test_pass(void)
{
ksft_test_result_pass("%s\n", current_test);
}
#define uffd_test_start(...) do { \
snprintf(current_test, sizeof(current_test), __VA_ARGS__); \
} while (0)
#define uffd_test_fail(fmt, ...) do { \
ksft_print_msg("failed reason: [" fmt "]\n", ##__VA_ARGS__); \
ksft_test_result_fail("%s\n", current_test); \
} while (0)
static void uffd_test_skip(const char *message)
{
ksft_test_result_skip("%s (%s)\n", current_test, message);
}
static void test_uffd_api(bool use_dev)
{
const uint64_t expected_ioctls =
BIT_ULL(_UFFDIO_REGISTER) |
BIT_ULL(_UFFDIO_UNREGISTER) |
BIT_ULL(_UFFDIO_API);
struct uffdio_api uffdio_api;
int uffd;
uffd_test_start("UFFDIO_API (with %s)",
use_dev ? "/dev/userfaultfd" : "syscall");
if (use_dev)
uffd = uffd_open_dev(UFFD_FLAGS);
else
uffd = uffd_open_sys(UFFD_FLAGS);
if (uffd < 0) {
uffd_test_skip("cannot open userfaultfd handle");
return;
}
/* Test wrong UFFD_API */
uffdio_api.api = 0xab;
uffdio_api.features = 0;
if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
uffd_test_fail("UFFDIO_API should fail with wrong api but didn't");
goto out;
}
/* Test wrong feature bit */
uffdio_api.api = UFFD_API;
uffdio_api.features = BIT_ULL(63);
if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't");
goto out;
}
/* Test normal UFFDIO_API */
uffdio_api.api = UFFD_API;
uffdio_api.features = 0;
if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {
uffd_test_fail("UFFDIO_API should succeed but failed");
goto out;
}
/* Verify returned fd-level ioctls bitmask */
if ((uffdio_api.ioctls & expected_ioctls) != expected_ioctls) {
uffd_test_fail("UFFDIO_API missing expected ioctls: "
"got=0x%"PRIx64", expected=0x%"PRIx64,
(uint64_t)uffdio_api.ioctls,
expected_ioctls);
goto out;
}
/* Test double requests of UFFDIO_API with a random feature set */
uffdio_api.features = BIT_ULL(0);
if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
uffd_test_fail("UFFDIO_API should reject initialized uffd");
goto out;
}
uffd_test_pass();
out:
close(uffd);
}
static bool uffd_feature_supported(uffd_test_case_t *test)
{
uint64_t features;
if (uffd_get_features(&features))
return false;
return (features & test->uffd_feature_required) ==
test->uffd_feature_required;
}
static int pagemap_open(void)
{
int fd = open("/proc/self/pagemap", O_RDONLY);
if (fd < 0)
err("open pagemap");
return fd;
}
/* This macro let __LINE__ works in err() */
#define pagemap_check_wp(value, wp) do { \
if (!!(value & PM_UFFD_WP) != wp) \
err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
} while (0)
typedef struct {
uffd_global_test_opts_t *gopts;
int child_uffd;
} fork_event_args;
static void *fork_event_consumer(void *data)
{
fork_event_args *args = data;
struct uffd_msg msg = { 0 };
args->gopts->ready_for_fork = true;
/* Read until a full msg received */
while (uffd_read_msg(args->gopts, &msg));
if (msg.event != UFFD_EVENT_FORK)
err("wrong message: %u\n", msg.event);
/* Just to be properly freed later */
args->child_uffd = msg.arg.fork.ufd;
return NULL;
}
typedef struct {
int gup_fd;
bool pinned;
} pin_args;
/*
* Returns 0 if succeed, <0 for errors. pin_pages() needs to be paired
* with unpin_pages(). Currently it needs to be RO longterm pin to satisfy
* all needs of the test cases (e.g., trigger unshare, trigger fork() early
* CoW, etc.).
*/
static int pin_pages(pin_args *args, void *buffer, size_t size)
{
struct pin_longterm_test test = {
.addr = (uintptr_t)buffer,
.size = size,
/* Read-only pins */
.flags = 0,
};
if (args->pinned)
err("already pinned");
args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
if (args->gup_fd < 0)
return -errno;
if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {
/* Even if gup_test existed, can be an old gup_test / kernel */
close(args->gup_fd);
return -errno;
}
args->pinned = true;
return 0;
}
static void unpin_pages(pin_args *args)
{
if (!args->pinned)
err("unpin without pin first");
if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))
err("PIN_LONGTERM_TEST_STOP");
close(args->gup_fd);
args->pinned = false;
}
static int pagemap_test_fork(uffd_global_test_opts_t *gopts, bool with_event, bool test_pin)
{
fork_event_args args = { .gopts = gopts, .child_uffd = -1 };
pthread_t thread;
pid_t child;
uint64_t value;
int fd, result;
/* Prepare a thread to resolve EVENT_FORK */
if (with_event) {
gopts->ready_for_fork = false;
if (pthread_create(&thread, NULL, fork_event_consumer, &args))
err("pthread_create()");
while (!gopts->ready_for_fork)
; /* Wait for the poll_thread to start executing before forking */
}
child = fork();
if (!child) {
/* Open the pagemap fd of the child itself */
pin_args args = {};
fd = pagemap_open();
if (test_pin && pin_pages(&args, gopts->area_dst, gopts->page_size))
/*
* Normally when reach here we have pinned in
* previous tests, so shouldn't fail anymore
*/
err("pin page failed in child");
value = pagemap_get_entry(fd, gopts->area_dst);
/*
* After fork(), we should handle uffd-wp bit differently:
*
* (1) when with EVENT_FORK, it should persist
* (2) when without EVENT_FORK, it should be dropped
*/
pagemap_check_wp(value, with_event);
if (test_pin)
unpin_pages(&args);
/* Succeed */
_exit(0);
}
waitpid(child, &result, 0);
if (with_event) {
if (pthread_join(thread, NULL))
err("pthread_join()");
if (args.child_uffd < 0)
err("Didn't receive child uffd");
close(args.child_uffd);
}
return result;
}
static void uffd_wp_unpopulated_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
uint64_t value;
int pagemap_fd;
if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
false, true, false))
err("register failed");
pagemap_fd = pagemap_open();
/* Test applying pte marker to anon unpopulated */
wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
pagemap_check_wp(value, true);
/* Test unprotect on anon pte marker */
wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, false);
value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
pagemap_check_wp(value, false);
/* Test zap on anon marker */
wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED))
err("madvise(MADV_DONTNEED) failed");
value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
pagemap_check_wp(value, false);
/* Test fault in after marker removed */
*gopts->area_dst = 1;
value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
pagemap_check_wp(value, false);
/* Drop it to make pte none again */
if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED))
err("madvise(MADV_DONTNEED) failed");
/* Test read-zero-page upon pte marker */
wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
*(volatile char *)gopts->area_dst;
/* Drop it to make pte none again */
if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED))
err("madvise(MADV_DONTNEED) failed");
uffd_test_pass();
}
static void uffd_wp_fork_test_common(uffd_global_test_opts_t *gopts, uffd_test_args_t *args,
bool with_event)
{
int pagemap_fd;
uint64_t value;
if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
false, true, false))
err("register failed");
pagemap_fd = pagemap_open();
/* Touch the page */
*gopts->area_dst = 1;
wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
pagemap_check_wp(value, true);
if (pagemap_test_fork(gopts, with_event, false)) {
uffd_test_fail("Detected %s uffd-wp bit in child in present pte",
with_event ? "missing" : "stall");
goto out;
}
/*
* This is an attempt for zapping the pgtable so as to test the
* markers.
*
* For private mappings, PAGEOUT will only work on exclusive ptes
* (PM_MMAP_EXCLUSIVE) which we should satisfy.
*
* For shared, PAGEOUT may not work. Use DONTNEED instead which
* plays a similar role of zapping (rather than freeing the page)
* to expose pte markers.
*/
if (args->mem_type->shared) {
if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED))
err("MADV_DONTNEED");
} else {
/*
* NOTE: ignore retval because private-hugetlb doesn't yet
* support swapping, so it could fail.
*/
madvise(gopts->area_dst, gopts->page_size, MADV_PAGEOUT);
}
/* Uffd-wp should persist even swapped out */
value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
pagemap_check_wp(value, true);
if (pagemap_test_fork(gopts, with_event, false)) {
uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",
with_event ? "missing" : "stall");
goto out;
}
/* Unprotect; this tests swap pte modifications */
wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, false);
value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
pagemap_check_wp(value, false);
/* Fault in the page from disk */
*gopts->area_dst = 2;
value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
pagemap_check_wp(value, false);
uffd_test_pass();
out:
if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size))
err("unregister failed");
close(pagemap_fd);
}
static void uffd_wp_fork_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
uffd_wp_fork_test_common(gopts, args, false);
}
static void uffd_wp_fork_with_event_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
uffd_wp_fork_test_common(gopts, args, true);
}
static void uffd_wp_fork_pin_test_common(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args,
bool with_event)
{
int pagemap_fd;
pin_args pin_args = {};
if (uffd_register(gopts->uffd, gopts->area_dst, gopts->page_size, false, true, false))
err("register failed");
pagemap_fd = pagemap_open();
/* Touch the page */
*gopts->area_dst = 1;
wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
/*
* 1. First pin, then fork(). This tests fork() special path when
* doing early CoW if the page is private.
*/
if (pin_pages(&pin_args, gopts->area_dst, gopts->page_size)) {
uffd_test_skip("Possibly CONFIG_GUP_TEST missing "
"or unprivileged");
close(pagemap_fd);
uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size);
return;
}
if (pagemap_test_fork(gopts, with_event, false)) {
uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",
with_event ? "missing" : "stall");
unpin_pages(&pin_args);
goto out;
}
unpin_pages(&pin_args);
/*
* 2. First fork(), then pin (in the child, where test_pin==true).
* This tests COR, aka, page unsharing on private memories.
*/
if (pagemap_test_fork(gopts, with_event, true)) {
uffd_test_fail("Detected %s uffd-wp bit when RO pin",
with_event ? "missing" : "stall");
goto out;
}
uffd_test_pass();
out:
if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size))
err("register failed");
close(pagemap_fd);
}
static void uffd_wp_fork_pin_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
uffd_wp_fork_pin_test_common(gopts, args, false);
}
static void uffd_wp_fork_pin_with_event_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
uffd_wp_fork_pin_test_common(gopts, args, true);
}
static void check_memory_contents(uffd_global_test_opts_t *gopts, char *p)
{
unsigned long i, j;
uint8_t expected_byte;
for (i = 0; i < gopts->nr_pages; ++i) {
expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));
for (j = 0; j < gopts->page_size; j++) {
uint8_t v = *(uint8_t *)(p + (i * gopts->page_size) + j);
if (v != expected_byte)
err("unexpected page contents");
}
}
}
static void uffd_minor_test_common(uffd_global_test_opts_t *gopts, bool test_collapse, bool test_wp)
{
unsigned long p;
pthread_t uffd_mon;
char c = '\0';
struct uffd_args args = { 0 };
args.gopts = gopts;
/*
* NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing
* both do not make much sense.
*/
assert(!(test_collapse && test_wp));
if (uffd_register(gopts->uffd, gopts->area_dst_alias, gopts->nr_pages * gopts->page_size,
/* NOTE! MADV_COLLAPSE may not work with uffd-wp */
false, test_wp, true))
err("register failure");
/*
* After registering with UFFD, populate the non-UFFD-registered side of
* the shared mapping. This should *not* trigger any UFFD minor faults.
*/
for (p = 0; p < gopts->nr_pages; ++p)
memset(gopts->area_dst + (p * gopts->page_size), p % ((uint8_t)-1),
gopts->page_size);
args.apply_wp = test_wp;
if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
err("uffd_poll_thread create");
/*
* Read each of the pages back using the UFFD-registered mapping. We
* expect that the first time we touch a page, it will result in a minor
* fault. uffd_poll_thread will resolve the fault by bit-flipping the
* page's contents, and then issuing a CONTINUE ioctl.
*/
check_memory_contents(gopts, gopts->area_dst_alias);
if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
err("pipe write");
if (pthread_join(uffd_mon, NULL))
err("join() failed");
if (test_collapse) {
if (madvise(gopts->area_dst_alias, gopts->nr_pages * gopts->page_size,
MADV_COLLAPSE)) {
/* It's fine to fail for this one... */
uffd_test_skip("MADV_COLLAPSE failed");
return;
}
uffd_test_ops->check_pmd_mapping(gopts,
gopts->area_dst,
gopts->nr_pages * gopts->page_size /
read_pmd_pagesize());
/*
* This won't cause uffd-fault - it purely just makes sure there
* was no corruption.
*/
check_memory_contents(gopts, gopts->area_dst_alias);
}
if (args.missing_faults != 0 || args.minor_faults != gopts->nr_pages)
uffd_test_fail("stats check error");
else
uffd_test_pass();
}
void uffd_minor_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
uffd_minor_test_common(gopts, false, false);
}
void uffd_minor_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
uffd_minor_test_common(gopts, false, true);
}
void uffd_minor_collapse_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
uffd_minor_test_common(gopts, true, false);
}
static int uffd_register_rwp(int uffd, void *addr, uint64_t len)
{
struct uffdio_register reg = {
.range = { .start = (unsigned long)addr, .len = len },
.mode = UFFDIO_REGISTER_MODE_RWP,
};
if (ioctl(uffd, UFFDIO_REGISTER, ®) == -1)
return -errno;
return 0;
}
static void rwprotect_range(int uffd, __u64 start, __u64 len, bool protect)
{
struct uffdio_rwprotect rwp = {
.range = { .start = start, .len = len },
.mode = protect ? UFFDIO_RWPROTECT_MODE_RWP : 0,
};
if (ioctl(uffd, UFFDIO_RWPROTECT, &rwp))
err("UFFDIO_RWPROTECT failed");
}
static void set_async_mode(int uffd, bool enable)
{
struct uffdio_set_mode mode = { };
if (enable)
mode.enable = UFFD_FEATURE_RWP_ASYNC;
else
mode.disable = UFFD_FEATURE_RWP_ASYNC;
if (ioctl(uffd, UFFDIO_SET_MODE, &mode))
err("UFFDIO_SET_MODE failed");
}
/*
* Test async RWP faults on anonymous memory.
* Populate pages, register MODE_RWP with RWP_ASYNC,
* RW-protect, re-access, verify content preserved and no faults delivered.
*/
static void uffd_rwp_async_test(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args)
{
unsigned long nr_pages = gopts->nr_pages;
unsigned long page_size = gopts->page_size;
unsigned long p;
/* Populate all pages with known content */
for (p = 0; p < nr_pages; p++)
memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size);
/* Register MODE_RWP */
if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
nr_pages * page_size))
err("register failure");
/* RW-protect all pages (sets protnone) */
rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
nr_pages * page_size, true);
/* Access all pages — should auto-resolve, no faults */
for (p = 0; p < nr_pages; p++) {
unsigned char *page = (unsigned char *)gopts->area_dst +
p * page_size;
unsigned char expected = p % 255 + 1;
if (page[0] != expected) {
uffd_test_fail("page %lu content mismatch: %u != %u",
p, page[0], expected);
return;
}
}
uffd_test_pass();
}
/*
* Fault handler for RWP — unprotect the page via UFFDIO_RWPROTECT.
*/
static void uffd_handle_rwp_fault(uffd_global_test_opts_t *gopts,
struct uffd_msg *msg,
struct uffd_args *uargs)
{
if (!(msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_RWP))
err("expected RWP fault, got 0x%llx",
msg->arg.pagefault.flags);
rwprotect_range(gopts->uffd, msg->arg.pagefault.address,
gopts->page_size, false);
uargs->minor_faults++;
}
/*
* Test sync RWP faults on anonymous memory.
* Populate pages, register MODE_RWP (sync), RW-protect,
* access from worker thread, verify fault delivered, UFFDIO_RWPROTECT resolves.
*/
static void uffd_rwp_sync_test(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args)
{
unsigned long nr_pages = gopts->nr_pages;
unsigned long page_size = gopts->page_size;
pthread_t uffd_mon;
struct uffd_args uargs = { };
bool failed = false;
char c = '\0';
unsigned long p;
uargs.gopts = gopts;
uargs.handle_fault = uffd_handle_rwp_fault;
/* Populate all pages */
for (p = 0; p < nr_pages; p++)
memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size);
/* Register MODE_RWP */
if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
nr_pages * page_size))
err("register failure");
/* RW-protect all pages */
rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
nr_pages * page_size, true);
/* Start fault handler thread */
if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs))
err("uffd_poll_thread create");
/* Access all pages — triggers sync RWP faults, handler unprotects */
for (p = 0; p < nr_pages; p++) {
unsigned char *page = (unsigned char *)gopts->area_dst +
p * page_size;
if (page[0] != (p % 255 + 1)) {
uffd_test_fail("page %lu content mismatch", p);
failed = true;
goto out;
}
}
out:
/*
* Stop the handler before reading minor_faults: the last fault
* resolution rwprotect_range()s before incrementing the counter,
* so the main thread can race ahead of the increment.
*/
if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
err("pipe write");
if (pthread_join(uffd_mon, NULL))
err("join() failed");
if (failed)
return;
if (uargs.minor_faults == 0)
uffd_test_fail("expected RWP faults, got 0");
else
uffd_test_pass();
}
/*
* Test PAGEMAP_SCAN working-set discovery via the "hot" (accessed) scan.
*
* The working-set primitive is to find pages that were accessed: scan for
* PAGE_IS_ACCESSED, which is set once an access clears the protnone+uffd
* marker. A VMM treats every access fault as "hot" (RWP here; MINOR/MISSING
* for non-resident pages) and reclaims the rest from the backing file.
*
* We deliberately do NOT use an inverted "cold" scan: that only sees
* VMA-resident ptes, so for a file mapping it misses cached-but-unmapped (and
* never-faulted, pre-populated) pages, which are pte_none and thus invisible.
* Hot tracking + file-level reclaim covers them; a cold pte scan cannot.
*/
static void uffd_rwp_pagemap_test(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args)
{
unsigned long nr_pages = gopts->nr_pages;
unsigned long page_size = gopts->page_size;
unsigned long p;
struct page_region regions[16];
struct pm_scan_arg pm_arg;
int pagemap_fd;
long ret;
/* Need at least 4 pages */
if (nr_pages < 4) {
uffd_test_skip("need at least 4 pages");
return;
}
/* Populate all pages */
for (p = 0; p < nr_pages; p++)
memset(gopts->area_dst + p * page_size, 0xab, page_size);
/* Register and RW-protect */
if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
nr_pages * page_size))
err("register failure");
rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
nr_pages * page_size, true);
/* Touch first half of pages to re-activate them (async auto-resolve) */
for (p = 0; p < nr_pages / 2; p++) {
volatile char *page = gopts->area_dst + p * page_size;
(void)*page;
}
uint64_t start = (uint64_t)gopts->area_dst;
uint64_t boundary = start + (nr_pages / 2) * page_size;
uint64_t end = start + nr_pages * page_size;
pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
if (pagemap_fd < 0)
err("open pagemap");
/*
* Hot scan: report the pages that were accessed. PAGE_IS_ACCESSED is
* set once the protnone+uffd marker is cleared (by the access, async
* auto-resolve here). The touched first half must come back as exactly
* one hot region [start, boundary); the untouched second half must not
* appear.
*/
memset(&pm_arg, 0, sizeof(pm_arg));
pm_arg.size = sizeof(pm_arg);
pm_arg.start = start;
pm_arg.end = end;
pm_arg.vec = (uint64_t)regions;
pm_arg.vec_len = ARRAY_SIZE(regions);
pm_arg.category_mask = PAGE_IS_ACCESSED;
pm_arg.return_mask = PAGE_IS_ACCESSED;
ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &pm_arg);
close(pagemap_fd);
if (ret < 0) {
uffd_test_fail("PAGEMAP_SCAN failed: %s", strerror(errno));
return;
}
if (ret != 1 || regions[0].start != start ||
regions[0].end != boundary) {
uffd_test_fail("hot set wrong: got %ld regions [0x%lx,0x%lx), expected 1 [0x%lx,0x%lx)",
ret, (unsigned long)regions[0].start,
(unsigned long)regions[0].end,
(unsigned long)start, (unsigned long)boundary);
return;
}
uffd_test_pass();
}
/*
* Test that RWP protection survives a mprotect(PROT_NONE) ->
* mprotect(PROT_READ|PROT_WRITE) round-trip. The uffd-wp bit on a
* VM_UFFD_RWP VMA must continue to carry PROT_NONE semantics after
* mprotect() changes the base protection; otherwise accesses would
* silently succeed and the pagemap bit would stick without a fault
* ever clearing it.
*/
static void uffd_rwp_mprotect_test(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args)
{
unsigned long nr_pages = gopts->nr_pages;
unsigned long page_size = gopts->page_size;
unsigned long p;
struct page_region regions[16];
struct pm_scan_arg pm_arg;
int pagemap_fd;
uint64_t value;
long ret;
/* Populate all pages */
for (p = 0; p < nr_pages; p++)
memset(gopts->area_dst + p * page_size, 0xab, page_size);
/* Register and RW-protect the whole range */
if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
nr_pages * page_size))
err("register failure");
rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
nr_pages * page_size, true);
/* Round-trip mprotect(): PROT_NONE -> PROT_READ|PROT_WRITE */
if (mprotect(gopts->area_dst, nr_pages * page_size, PROT_NONE))
err("mprotect() PROT_NONE");
if (mprotect(gopts->area_dst, nr_pages * page_size,
PROT_READ | PROT_WRITE))
err("mprotect() PROT_READ|PROT_WRITE");
/*
* The marker must survive the round-trip; if mprotect() dropped it,
* the touches below would not fault and the scan would pass
* vacuously.
*/
pagemap_fd = pagemap_open();
value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
close(pagemap_fd);
if (!(value & PM_UFFD_WP)) {
uffd_test_fail("RWP marker lost across mprotect()");
return;
}
/* Touch every page. Async RWP must auto-resolve each fault. */
for (p = 0; p < nr_pages; p++) {
volatile char *page = gopts->area_dst + p * page_size;
(void)*page;
}
/*
* After touching, no page should remain RW-protected. A stuck
* uffd-wp bit would mean mprotect() silently dropped PROT_NONE and
* the access never faulted.
*/
pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
if (pagemap_fd < 0)
err("open pagemap");
memset(&pm_arg, 0, sizeof(pm_arg));
pm_arg.size = sizeof(pm_arg);
pm_arg.start = (uint64_t)gopts->area_dst;
pm_arg.end = (uint64_t)gopts->area_dst + nr_pages * page_size;
pm_arg.vec = (uint64_t)regions;
pm_arg.vec_len = ARRAY_SIZE(regions);
pm_arg.category_mask = PAGE_IS_ACCESSED;
pm_arg.category_inverted = PAGE_IS_ACCESSED;
pm_arg.return_mask = PAGE_IS_ACCESSED;
ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &pm_arg);
close(pagemap_fd);
if (ret < 0) {
uffd_test_fail("PAGEMAP_SCAN failed: %s", strerror(errno));
return;
}
if (ret != 0) {
uffd_test_fail("expected no cold pages after mprotect()+touch, got %ld regions",
ret);
return;
}
uffd_test_pass();
}
/*
* Test that GUP resolves through protnone PTEs (async mode).
* vmsplice() into a pipe pins user pages via get_user_pages_fast() --
* unlike write(), which goes through copy_from_user() and ordinary
* hardware page faults -- so it exercises gup_can_follow_protnone() on
* the RW-protected PTE. In async mode the kernel auto-restores
* permissions and GUP returns the page.
*/
static void uffd_rwp_gup_test(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args)
{
struct iovec iov;
char buf;
int pipefd[2];
/* Populate first page with known content */
memset(gopts->area_dst, 0xCD, gopts->page_size);
if (uffd_register_rwp(gopts->uffd, gopts->area_dst, gopts->page_size))
err("register failure");
rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
gopts->page_size, true);
if (pipe(pipefd))
err("pipe");
/*
* One byte's worth of iov is enough to GUP the containing page and
* keeps the pipe transfer well under any pipe-capacity limit even on
* hugetlb-backed runs.
*/
iov.iov_base = gopts->area_dst;
iov.iov_len = 1;
if (vmsplice(pipefd[1], &iov, 1, 0) != 1) {
uffd_test_fail("vmsplice from RW-protected page failed: %s",
strerror(errno));
goto out;
}
if (read(pipefd[0], &buf, 1) != 1) {
uffd_test_fail("read from pipe failed");
goto out;
}
if (buf != (char)0xCD) {
uffd_test_fail("content mismatch: got 0x%02x, expected 0xCD",
(unsigned char)buf);
goto out;
}
uffd_test_pass();
out:
close(pipefd[0]);
close(pipefd[1]);
}
/*
* Test runtime toggle between async and sync modes.
* Start in async mode (detection), flip to sync (eviction), verify faults
* block, resolve them, flip back to async.
*/
static void uffd_rwp_async_toggle_test(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args)
{
unsigned long nr_pages = gopts->nr_pages;
unsigned long page_size = gopts->page_size;
struct uffd_args uargs = { };
pthread_t uffd_mon;
char c = '\0';
unsigned long p;
uargs.gopts = gopts;
uargs.handle_fault = uffd_handle_rwp_fault;
/* Populate */
for (p = 0; p < nr_pages; p++)
memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size);
if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
nr_pages * page_size))
err("register failure");
/* Phase 1: async detection — RW-protect, access first half */
rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
nr_pages * page_size, true);
for (p = 0; p < nr_pages / 2; p++) {
volatile char *page = gopts->area_dst + p * page_size;
(void)*page; /* auto-resolves in async mode */
}
/* Phase 2: flip to sync for eviction */
set_async_mode(gopts->uffd, false);
/* Start handler — will receive faults for cold pages */
if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs))
err("uffd_poll_thread create");
/* Access second half (cold pages) — should trigger sync faults */
for (p = nr_pages / 2; p < nr_pages; p++) {
unsigned char *page = (unsigned char *)gopts->area_dst +
p * page_size;
if (page[0] != (p % 255 + 1)) {
uffd_test_fail("page %lu content mismatch", p);
goto out;
}
}
/*
* Stop the handler before reading minor_faults: the last fault
* resolution rwprotect_range()s before incrementing the counter,
* so the main thread can race ahead of the increment. Stopping
* here also makes Phase 3 a clean async-only test -- with the
* handler still running it would silently resolve any sync fault
* the kernel erroneously delivers, masking a regression.
*/
if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
err("pipe write");
if (pthread_join(uffd_mon, NULL))
err("join() failed");
if (uargs.minor_faults == 0) {
uffd_test_fail("expected sync faults, got 0");
return;
}
/* Phase 3: flip back to async */
set_async_mode(gopts->uffd, true);
/* RW-protect and access again — should auto-resolve */
rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
nr_pages * page_size, true);
for (p = 0; p < nr_pages; p++) {
volatile char *page = gopts->area_dst + p * page_size;
(void)*page;
}
uffd_test_pass();
return;
out:
if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
err("pipe write");
if (pthread_join(uffd_mon, NULL))
err("join() failed");
}
/*
* Test that RW-protected pages become accessible after closing uffd.
*/
static void uffd_rwp_close_test(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args)
{
unsigned long nr_pages = gopts->nr_pages;
unsigned long page_size = gopts->page_size;
unsigned long p;
/* Populate */
for (p = 0; p < nr_pages; p++)
memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size);
if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
nr_pages * page_size))
err("register failure");
rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
nr_pages * page_size, true);
/* Close uffd — should restore protnone PTEs */
close(gopts->uffd);
gopts->uffd = -1;
/* All pages should be accessible with original content */
for (p = 0; p < nr_pages; p++) {
unsigned char *page = (unsigned char *)gopts->area_dst +
p * page_size;
unsigned char expected = p % 255 + 1;
if (page[0] != expected) {
uffd_test_fail("page %lu not accessible after close", p);
return;
}
}
uffd_test_pass();
}
/*
* Test that RWP protection is preserved across fork() when
* UFFD_FEATURE_EVENT_FORK is enabled. Without preservation, the child's
* PTEs would lose the uffd-wp marker and RWP-protected accesses would
* silently fall through to do_numa_page().
*/
static void uffd_rwp_fork_test(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args)
{
unsigned long nr_pages = gopts->nr_pages;
unsigned long page_size = gopts->page_size;
int pagemap_fd;
uint64_t value;
if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
nr_pages * page_size))
err("register failed");
/* Populate + RWP-protect */
*gopts->area_dst = 1;
rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
page_size, true);
/* Parent: verify uffd-wp bit is set before fork */
pagemap_fd = pagemap_open();
value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
pagemap_check_wp(value, true);
/*
* Fork with EVENT_FORK: child inherits VM_UFFD_RWP. Child reads
* its own pagemap and must still see the uffd-wp bit set.
*/
if (pagemap_test_fork(gopts, true, false)) {
uffd_test_fail("RWP marker lost in child after fork");
goto out;
}
uffd_test_pass();
out:
close(pagemap_fd);
}
/*
* Test that RWP protection on a pinned anon page is preserved across fork().
* Pinning forces copy_present_page() in the child path, which must restore
* PAGE_NONE on top of the uffd bit. Using async mode, a read in the child
* auto-resolves if — and only if — the PTE was actually protnone+uffd; the
* cleared uffd bit afterward proves the fault path ran.
*/
static void uffd_rwp_fork_pin_test(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args)
{
unsigned long page_size = gopts->page_size;
fork_event_args fevent_args = { .gopts = gopts, .child_uffd = -1 };
pin_args pin_args = {};
int pagemap_fd, status;
pthread_t fevent_thread;
uint64_t value;
pid_t child;
if (uffd_register_rwp(gopts->uffd, gopts->area_dst, page_size))
err("register failed");
/* Populate. */
*gopts->area_dst = 1;
/* RO-longterm pin so fork() takes copy_present_page() for this PTE. */
if (pin_pages(&pin_args, gopts->area_dst, page_size)) {
uffd_test_skip("Possibly CONFIG_GUP_TEST missing or unprivileged");
uffd_unregister(gopts->uffd, gopts->area_dst, page_size);
return;
}
/* RWP-protect: PTE is now PAGE_NONE + uffd bit. */
rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, page_size, true);
pagemap_fd = pagemap_open();
value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
pagemap_check_wp(value, true);
/*
* UFFD_FEATURE_EVENT_FORK is required so the child inherits
* VM_UFFD_RWP and the marker; without it dup_userfaultfd() resets
* the child VMA and the test would pass for the wrong reason.
* dup_userfaultfd() blocks until the EVENT_FORK message is consumed,
* so spawn a reader before the fork().
*/
gopts->ready_for_fork = false;
if (pthread_create(&fevent_thread, NULL, fork_event_consumer,
&fevent_args))
err("pthread_create() for fork event consumer");
while (!gopts->ready_for_fork)
; /* Wait for consumer to start polling. */
child = fork();
if (child < 0)
err("fork");
if (child == 0) {
volatile char c;
int cfd;
/*
* Precondition: the child must have inherited the marker.
* If copy_present_page() dropped it together with PAGE_NONE,
* the read below would succeed without the fault path and
* the after-read check would pass for the wrong reason.
*/
cfd = pagemap_open();
value = pagemap_get_entry(cfd, gopts->area_dst);
if (!(value & PM_UFFD_WP)) {
close(cfd);
_exit(2);
}
/*
* Read the pinned page. Only reaches the fault path if the
* child PTE is protnone + uffd; async mode auto-resolves and
* clears the uffd bit. If copy_present_page() dropped
* PAGE_NONE, the read would silently succeed and the bit
* would still be set.
*/
c = *(volatile char *)gopts->area_dst;
(void)c;
value = pagemap_get_entry(cfd, gopts->area_dst);
close(cfd);
_exit((value & PM_UFFD_WP) ? 1 : 0);
}
if (waitpid(child, &status, 0) < 0)
err("waitpid");
if (pthread_join(fevent_thread, NULL))
err("pthread_join() for fork event consumer");
if (fevent_args.child_uffd >= 0)
close(fevent_args.child_uffd);
unpin_pages(&pin_args);
close(pagemap_fd);
if (uffd_unregister(gopts->uffd, gopts->area_dst, page_size))
err("unregister failed");
if (WIFEXITED(status) && WEXITSTATUS(status) == 2) {
uffd_test_fail("RWP marker not inherited by child");
return;
}
if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
uffd_test_fail("RWP not enforced in child after pinned fork");
return;
}
uffd_test_pass();
}
/*
* A non-exclusive (forked, COW-shared) anon page that is RWP-protected and
* then swapped out must keep tracking across swap-in. On the write that
* swaps it back in, do_swap_page() restores PAGE_NONE and the access retries
* through the RWP fault path, instead of being COWed straight to a fresh
* accessible page -- which would silently drop the marker for a non-exclusive
* folio. Sync mode lets us observe the fault directly: with the bug, the
* write COWs without delivering any RWP fault.
*
* Needs a swap device; skipped if MADV_PAGEOUT cannot evict the page.
*/
static void uffd_rwp_swap_cow_test(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args)
{
unsigned long page_size = gopts->page_size;
struct uffd_args uargs = { };
int pagemap_fd, go[2], i;
pthread_t uffd_mon;
char c = '\0';
pid_t child;
uargs.gopts = gopts;
uargs.handle_fault = uffd_handle_rwp_fault;
if (uffd_register_rwp(gopts->uffd, gopts->area_dst, page_size))
err("register failed");
/* Populate one page (exclusive at this point). */
*gopts->area_dst = 0x11;
/* RWP-protect: PTE becomes PAGE_NONE + uffd bit (still exclusive). */
rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, page_size, true);
/*
* Swap the page out while it is still exclusive: a shared (forked)
* folio does not get reclaimed by MADV_PAGEOUT. Retry, since a hot
* page may just be rotated on the first reclaim pass.
*/
pagemap_fd = pagemap_open();
for (i = 0; i < 100; i++) {
if (madvise(gopts->area_dst, page_size, MADV_PAGEOUT))
err("MADV_PAGEOUT");
if (pagemap_is_swapped(pagemap_fd, gopts->area_dst))
break;
usleep(10000);
}
if (!pagemap_is_swapped(pagemap_fd, gopts->area_dst)) {
uffd_test_skip("MADV_PAGEOUT did not swap the page; is swap enabled?");
close(pagemap_fd);
uffd_unregister(gopts->uffd, gopts->area_dst, page_size);
return;
}
/*
* fork() now: the child duplicates the swap entry, so the slot becomes
* non-exclusive. The child parks (keeping the reference) until the
* parent has faulted the page back in.
*/
if (pipe(go))
err("pipe");
child = fork();
if (child < 0)
err("fork");
if (child == 0) {
close(go[1]);
read(go[0], &c, 1);
_exit(0);
}
close(go[0]);
if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs))
err("uffd_poll_thread create");
/*
* Write the page: swaps it back in (do_swap_page) on a non-exclusive
* folio with FAULT_FLAG_WRITE. The marker must survive and deliver an
* RWP fault rather than COW silently.
*/
*gopts->area_dst = 0x22;
if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
err("pipe write");
if (pthread_join(uffd_mon, NULL))
err("join failed");
if (uargs.minor_faults == 0)
uffd_test_fail("no RWP fault on swapped-in non-exclusive page");
else
uffd_test_pass();
close(pagemap_fd);
if (write(go[1], &c, 1) != 1)
err("child release");
close(go[1]);
waitpid(child, NULL, 0);
}
/*
* WP and RWP share the uffd-wp PTE bit and cannot coexist in the same VMA.
* Registration requesting both modes must be rejected.
*/
static void uffd_rwp_wp_exclusive_test(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args)
{
unsigned long nr_pages = gopts->nr_pages;
unsigned long page_size = gopts->page_size;
struct uffdio_register reg = { };
reg.range.start = (unsigned long)gopts->area_dst;
reg.range.len = nr_pages * page_size;
reg.mode = UFFDIO_REGISTER_MODE_WP | UFFDIO_REGISTER_MODE_RWP;
if (ioctl(gopts->uffd, UFFDIO_REGISTER, ®) == 0) {
uffd_test_fail("register with WP|RWP unexpectedly succeeded");
return;
}
if (errno != EINVAL) {
uffd_test_fail("register with WP|RWP: expected EINVAL, got %d",
errno);
return;
}
uffd_test_pass();
}
static sigjmp_buf jbuf, *sigbuf;
static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
{
if (sig == SIGBUS) {
if (sigbuf)
siglongjmp(*sigbuf, 1);
abort();
}
}
/*
* For non-cooperative userfaultfd test we fork() a process that will
* generate pagefaults, will mremap the area monitored by the
* userfaultfd and at last this process will release the monitored
* area.
* For the anonymous and shared memory the area is divided into two
* parts, the first part is accessed before mremap, and the second
* part is accessed after mremap. Since hugetlbfs does not support
* mremap, the entire monitored area is accessed in a single pass for
* HUGETLB_TEST.
* The release of the pages currently generates event for shmem and
* anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
* for hugetlb.
* For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
* monitored area, generate pagefaults and test that signal is delivered.
* Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
* test robustness use case - we release monitored area, fork a process
* that will generate pagefaults and verify signal is generated.
* This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
* feature. Using monitor thread, verify no userfault events are generated.
*/
static int faulting_process(uffd_global_test_opts_t *gopts, int signal_test, bool wp)
{
unsigned long nr, i;
unsigned long long count;
unsigned long split_nr_pages;
unsigned long lastnr;
struct sigaction act;
volatile unsigned long signalled = 0;
split_nr_pages = (gopts->nr_pages + 1) / 2;
if (signal_test) {
sigbuf = &jbuf;
memset(&act, 0, sizeof(act));
act.sa_sigaction = sighndl;
act.sa_flags = SA_SIGINFO;
if (sigaction(SIGBUS, &act, 0))
err("sigaction");
lastnr = (unsigned long)-1;
}
for (nr = 0; nr < split_nr_pages; nr++) {
volatile int steps = 1;
unsigned long offset = nr * gopts->page_size;
if (signal_test) {
if (sigsetjmp(*sigbuf, 1) != 0) {
if (steps == 1 && nr == lastnr)
err("Signal repeated");
lastnr = nr;
if (signal_test == 1) {
if (steps == 1) {
/* This is a MISSING request */
steps++;
if (copy_page(gopts, offset, wp))
signalled++;
} else {
/* This is a WP request */
assert(steps == 2);
wp_range(gopts->uffd,
(__u64)gopts->area_dst +
offset,
gopts->page_size, false);
}
} else {
signalled++;
continue;
}
}
}
count = *area_count(gopts->area_dst, nr, gopts);
if (count != gopts->count_verify[nr])
err("nr %lu memory corruption %llu %llu\n",
nr, count, gopts->count_verify[nr]);
/*
* Trigger write protection if there is by writing
* the same value back.
*/
*area_count(gopts->area_dst, nr, gopts) = count;
}
if (signal_test)
return signalled != split_nr_pages;
gopts->area_dst = mremap(gopts->area_dst, gopts->nr_pages * gopts->page_size,
gopts->nr_pages * gopts->page_size,
MREMAP_MAYMOVE | MREMAP_FIXED,
gopts->area_src);
if (gopts->area_dst == MAP_FAILED)
err("mremap");
/* Reset area_src since we just clobbered it */
gopts->area_src = NULL;
for (; nr < gopts->nr_pages; nr++) {
count = *area_count(gopts->area_dst, nr, gopts);
if (count != gopts->count_verify[nr]) {
err("nr %lu memory corruption %llu %llu\n",
nr, count, gopts->count_verify[nr]);
}
/*
* Trigger write protection if there is by writing
* the same value back.
*/
*area_count(gopts->area_dst, nr, gopts) = count;
}
uffd_test_ops->release_pages(gopts, gopts->area_dst);
for (nr = 0; nr < gopts->nr_pages; nr++)
for (i = 0; i < gopts->page_size; i++)
if (*(gopts->area_dst + nr * gopts->page_size + i) != 0)
err("page %lu offset %lu is not zero", nr, i);
return 0;
}
static void uffd_sigbus_test_common(uffd_global_test_opts_t *gopts, bool wp)
{
unsigned long userfaults;
pthread_t uffd_mon;
pid_t pid;
int err;
char c = '\0';
struct uffd_args args = { 0 };
args.gopts = gopts;
gopts->ready_for_fork = false;
fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK);
if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
true, wp, false))
err("register failure");
if (faulting_process(gopts, 1, wp))
err("faulting process failed");
uffd_test_ops->release_pages(gopts, gopts->area_dst);
args.apply_wp = wp;
if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
err("uffd_poll_thread create");
while (!gopts->ready_for_fork)
; /* Wait for the poll_thread to start executing before forking */
pid = fork();
if (pid < 0)
err("fork");
if (!pid)
_exit(faulting_process(gopts, 2, wp));
waitpid(pid, &err, 0);
if (err)
err("faulting process failed");
if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
err("pipe write");
if (pthread_join(uffd_mon, (void **)&userfaults))
err("pthread_join()");
if (userfaults)
uffd_test_fail("Signal test failed, userfaults: %ld", userfaults);
else
uffd_test_pass();
}
static void uffd_sigbus_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
uffd_sigbus_test_common(gopts, false);
}
static void uffd_sigbus_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
uffd_sigbus_test_common(gopts, true);
}
static void uffd_events_test_common(uffd_global_test_opts_t *gopts, bool wp)
{
pthread_t uffd_mon;
pid_t pid;
int err;
char c = '\0';
struct uffd_args args = { 0 };
args.gopts = gopts;
gopts->ready_for_fork = false;
fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK);
if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
true, wp, false))
err("register failure");
args.apply_wp = wp;
if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
err("uffd_poll_thread create");
while (!gopts->ready_for_fork)
; /* Wait for the poll_thread to start executing before forking */
pid = fork();
if (pid < 0)
err("fork");
if (!pid)
_exit(faulting_process(gopts, 0, wp));
waitpid(pid, &err, 0);
if (err)
err("faulting process failed");
if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
err("pipe write");
if (pthread_join(uffd_mon, NULL))
err("pthread_join()");
if (args.missing_faults != gopts->nr_pages)
uffd_test_fail("Fault counts wrong");
else
uffd_test_pass();
}
static void uffd_events_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
uffd_events_test_common(gopts, false);
}
static void uffd_events_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
uffd_events_test_common(gopts, true);
}
static void retry_uffdio_zeropage(uffd_global_test_opts_t *gopts,
struct uffdio_zeropage *uffdio_zeropage)
{
uffd_test_ops->alias_mapping(gopts, &uffdio_zeropage->range.start,
uffdio_zeropage->range.len,
0);
if (ioctl(gopts->uffd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
if (uffdio_zeropage->zeropage != -EEXIST)
err("UFFDIO_ZEROPAGE error: %"PRId64,
(int64_t)uffdio_zeropage->zeropage);
} else {
err("UFFDIO_ZEROPAGE error: %"PRId64,
(int64_t)uffdio_zeropage->zeropage);
}
}
static bool do_uffdio_zeropage(uffd_global_test_opts_t *gopts, bool has_zeropage)
{
struct uffdio_zeropage uffdio_zeropage = { 0 };
int ret;
__s64 res;
uffdio_zeropage.range.start = (unsigned long) gopts->area_dst;
uffdio_zeropage.range.len = gopts->page_size;
uffdio_zeropage.mode = 0;
ret = ioctl(gopts->uffd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
res = uffdio_zeropage.zeropage;
if (ret) {
/* real retval in ufdio_zeropage.zeropage */
if (has_zeropage)
err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);
else if (res != -EINVAL)
err("UFFDIO_ZEROPAGE not -EINVAL");
} else if (has_zeropage) {
if (res != gopts->page_size)
err("UFFDIO_ZEROPAGE unexpected size");
else
retry_uffdio_zeropage(gopts, &uffdio_zeropage);
return true;
} else
err("UFFDIO_ZEROPAGE succeeded");
return false;
}
/*
* Registers a range with MISSING mode only for zeropage test. Return true
* if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register()
* because we want to detect .ioctls along the way.
*/
static bool
uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len)
{
uint64_t ioctls = 0;
if (uffd_register_with_ioctls(uffd, addr, len, true,
false, false, &ioctls))
err("zeropage register fail");
return ioctls & (1 << _UFFDIO_ZEROPAGE);
}
/* exercise UFFDIO_ZEROPAGE */
static void uffd_zeropage_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
bool has_zeropage;
int i;
has_zeropage = uffd_register_detect_zeropage(gopts->uffd,
gopts->area_dst,
gopts->page_size);
if (gopts->area_dst_alias)
/* Ignore the retval; we already have it */
uffd_register_detect_zeropage(gopts->uffd, gopts->area_dst_alias, gopts->page_size);
if (do_uffdio_zeropage(gopts, has_zeropage))
for (i = 0; i < gopts->page_size; i++)
if (gopts->area_dst[i] != 0)
err("data non-zero at offset %d\n", i);
if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size))
err("unregister");
if (gopts->area_dst_alias && uffd_unregister(gopts->uffd,
gopts->area_dst_alias,
gopts->page_size))
err("unregister");
uffd_test_pass();
}
static void uffd_register_poison(int uffd, void *addr, uint64_t len)
{
uint64_t ioctls = 0;
uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON);
if (uffd_register_with_ioctls(uffd, addr, len, true,
false, false, &ioctls))
err("poison register fail");
if ((ioctls & expected) != expected)
err("registered area doesn't support COPY and POISON ioctls");
}
static void do_uffdio_poison(uffd_global_test_opts_t *gopts, unsigned long offset)
{
struct uffdio_poison uffdio_poison = { 0 };
int ret;
__s64 res;
uffdio_poison.range.start = (unsigned long) gopts->area_dst + offset;
uffdio_poison.range.len = gopts->page_size;
uffdio_poison.mode = 0;
ret = ioctl(gopts->uffd, UFFDIO_POISON, &uffdio_poison);
res = uffdio_poison.updated;
if (ret)
err("UFFDIO_POISON error: %"PRId64, (int64_t)res);
else if (res != gopts->page_size)
err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res);
}
static void uffd_poison_handle_fault(uffd_global_test_opts_t *gopts,
struct uffd_msg *msg,
struct uffd_args *args)
{
unsigned long offset;
if (msg->event != UFFD_EVENT_PAGEFAULT)
err("unexpected msg event %u", msg->event);
if (msg->arg.pagefault.flags &
(UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR))
err("unexpected fault type %llu", msg->arg.pagefault.flags);
offset = (char *)(unsigned long)msg->arg.pagefault.address - gopts->area_dst;
offset &= ~(gopts->page_size-1);
/* Odd pages -> copy zeroed page; even pages -> poison. */
if (offset & gopts->page_size)
copy_page(gopts, offset, false);
else
do_uffdio_poison(gopts, offset);
}
/* Make sure to cover odd/even, and minimum duplications */
#define UFFD_POISON_TEST_NPAGES 4
static void uffd_poison_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
{
pthread_t uffd_mon;
char c;
struct uffd_args args = { 0 };
struct sigaction act = { 0 };
unsigned long nr_sigbus = 0;
unsigned long nr, poison_pages = UFFD_POISON_TEST_NPAGES;
if (gopts->nr_pages < poison_pages) {
uffd_test_skip("Too less pages for POISON test");
return;
}
args.gopts = gopts;
fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK);
uffd_register_poison(gopts->uffd, gopts->area_dst, poison_pages * gopts->page_size);
memset(gopts->area_src, 0, poison_pages * gopts->page_size);
args.handle_fault = uffd_poison_handle_fault;
if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
err("uffd_poll_thread create");
sigbuf = &jbuf;
act.sa_sigaction = sighndl;
act.sa_flags = SA_SIGINFO;
if (sigaction(SIGBUS, &act, 0))
err("sigaction");
for (nr = 0; nr < poison_pages; ++nr) {
unsigned long offset = nr * gopts->page_size;
const char *bytes = (const char *) gopts->area_dst + offset;
const char *i;
if (sigsetjmp(*sigbuf, 1)) {
/*
* Access below triggered a SIGBUS, which was caught by
* sighndl, which then jumped here. Count this SIGBUS,
* and move on to next page.
*/
++nr_sigbus;
continue;
}
for (i = bytes; i < bytes + gopts->page_size; ++i) {
if (*i)
err("nonzero byte in area_dst (%p) at %p: %u",
gopts->area_dst, i, *i);
}
}
if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
err("pipe write");
if (pthread_join(uffd_mon, NULL))
err("pthread_join()");
if (nr_sigbus != poison_pages / 2)
err("expected to receive %lu SIGBUS, actually received %lu",
poison_pages / 2, nr_sigbus);
uffd_test_pass();
}
static void
uffd_move_handle_fault_common(uffd_global_test_opts_t *gopts,
struct uffd_msg *msg,
struct uffd_args *args,
unsigned long len)
{
unsigned long offset;
if (msg->event != UFFD_EVENT_PAGEFAULT)
err("unexpected msg event %u", msg->event);
if (msg->arg.pagefault.flags &
(UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE))
err("unexpected fault type %llu", msg->arg.pagefault.flags);
offset = (char *)(unsigned long)msg->arg.pagefault.address - gopts->area_dst;
offset &= ~(len-1);
if (move_page(gopts, offset, len))
args->missing_faults++;
}
static void uffd_move_handle_fault(uffd_global_test_opts_t *gopts, struct uffd_msg *msg,
struct uffd_args *args)
{
uffd_move_handle_fault_common(gopts, msg, args, gopts->page_size);
}
static void uffd_move_pmd_handle_fault(uffd_global_test_opts_t *gopts, struct uffd_msg *msg,
struct uffd_args *args)
{
uffd_move_handle_fault_common(gopts, msg, args, read_pmd_pagesize());
}
static void
uffd_move_test_common(uffd_global_test_opts_t *gopts,
uffd_test_args_t *targs,
unsigned long chunk_size,
void (*handle_fault)(struct uffd_global_test_opts *gopts,
struct uffd_msg *msg, struct uffd_args *args)
)
{
unsigned long nr;
pthread_t uffd_mon;
char c = '\0';
unsigned long long count;
struct uffd_args args = { 0 };
char *orig_area_src = NULL, *orig_area_dst = NULL;
unsigned long step_size, step_count;
unsigned long src_offs = 0;
unsigned long dst_offs = 0;
args.gopts = gopts;
/* Prevent source pages from being mapped more than once */
if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_DONTFORK))
err("madvise(MADV_DONTFORK) failure");
if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
true, false, false))
err("register failure");
args.handle_fault = handle_fault;
if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
err("uffd_poll_thread create");
step_size = chunk_size / gopts->page_size;
step_count = gopts->nr_pages / step_size;
if (chunk_size > gopts->page_size) {
char *aligned_src = ALIGN_UP(gopts->area_src, chunk_size);
char *aligned_dst = ALIGN_UP(gopts->area_dst, chunk_size);
if (aligned_src != gopts->area_src || aligned_dst != gopts->area_dst) {
src_offs = (aligned_src - gopts->area_src) / gopts->page_size;
dst_offs = (aligned_dst - gopts->area_dst) / gopts->page_size;
step_count--;
}
orig_area_src = gopts->area_src;
orig_area_dst = gopts->area_dst;
gopts->area_src = aligned_src;
gopts->area_dst = aligned_dst;
}
/*
* Read each of the pages back using the UFFD-registered mapping. We
* expect that the first time we touch a page, it will result in a missing
* fault. uffd_poll_thread will resolve the fault by moving source
* page to destination.
*/
for (nr = 0; nr < step_count * step_size; nr += step_size) {
unsigned long i;
/* Check area_src content */
for (i = 0; i < step_size; i++) {
count = *area_count(gopts->area_src, nr + i, gopts);
if (count != gopts->count_verify[src_offs + nr + i])
err("nr %lu source memory invalid %llu %llu\n",
nr + i, count, gopts->count_verify[src_offs + nr + i]);
}
/* Faulting into area_dst should move the page or the huge page */
for (i = 0; i < step_size; i++) {
count = *area_count(gopts->area_dst, nr + i, gopts);
if (count != gopts->count_verify[dst_offs + nr + i])
err("nr %lu memory corruption %llu %llu\n",
nr, count, gopts->count_verify[dst_offs + nr + i]);
}
/* Re-check area_src content which should be empty */
for (i = 0; i < step_size; i++) {
count = *area_count(gopts->area_src, nr + i, gopts);
if (count != 0)
err("nr %lu move failed %llu %llu\n",
nr, count, gopts->count_verify[src_offs + nr + i]);
}
}
if (chunk_size > gopts->page_size) {
gopts->area_src = orig_area_src;
gopts->area_dst = orig_area_dst;
}
if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
err("pipe write");
if (pthread_join(uffd_mon, NULL))
err("join() failed");
if (args.missing_faults != step_count || args.minor_faults != 0)
uffd_test_fail("stats check error");
else
uffd_test_pass();
}
static void uffd_move_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
{
uffd_move_test_common(gopts, targs, gopts->page_size, uffd_move_handle_fault);
}
static void uffd_move_pmd_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
{
if (madvise(gopts->area_dst, gopts->nr_pages * gopts->page_size, MADV_HUGEPAGE))
err("madvise(MADV_HUGEPAGE) failure");
uffd_move_test_common(gopts, targs, read_pmd_pagesize(),
uffd_move_pmd_handle_fault);
}
static void uffd_move_pmd_split_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
{
if (madvise(gopts->area_dst, gopts->nr_pages * gopts->page_size, MADV_NOHUGEPAGE))
err("madvise(MADV_NOHUGEPAGE) failure");
uffd_move_test_common(gopts, targs, read_pmd_pagesize(),
uffd_move_pmd_handle_fault);
}
static bool
uffdio_verify_results(const char *name, int ret, int error, long result)
{
/*
* Should always return -1 with errno=EAGAIN, with corresponding
* result field updated in ioctl() args to be -EAGAIN too
* (e.g. copy.copy field for UFFDIO_COPY).
*/
if (ret != -1) {
uffd_test_fail("%s should have returned -1", name);
return false;
}
if (error != EAGAIN) {
uffd_test_fail("%s should have errno==EAGAIN", name);
return false;
}
if (result != -EAGAIN) {
uffd_test_fail("%s should have been updated for -EAGAIN",
name);
return false;
}
return true;
}
/*
* This defines a function to test one ioctl. Note that here "field" can
* be 1 or anything not -EAGAIN. With that initial value set, we can
* verify later that it should be updated by kernel (when -EAGAIN
* returned), by checking whether it is also updated to -EAGAIN.
*/
#define DEFINE_MMAP_CHANGING_TEST(name, ioctl_name, field) \
static bool uffdio_mmap_changing_test_##name(int fd) \
{ \
int ret; \
struct uffdio_##name args = { \
.field = 1, \
}; \
ret = ioctl(fd, ioctl_name, &args); \
return uffdio_verify_results(#ioctl_name, ret, errno, args.field); \
}
DEFINE_MMAP_CHANGING_TEST(zeropage, UFFDIO_ZEROPAGE, zeropage)
DEFINE_MMAP_CHANGING_TEST(copy, UFFDIO_COPY, copy)
DEFINE_MMAP_CHANGING_TEST(move, UFFDIO_MOVE, move)
DEFINE_MMAP_CHANGING_TEST(poison, UFFDIO_POISON, updated)
DEFINE_MMAP_CHANGING_TEST(continue, UFFDIO_CONTINUE, mapped)
typedef enum {
/* We actually do not care about any state except UNINTERRUPTIBLE.. */
THR_STATE_UNKNOWN = 0,
THR_STATE_UNINTERRUPTIBLE,
} thread_state;
typedef struct {
uffd_global_test_opts_t *gopts;
volatile pid_t *pid;
} mmap_changing_thread_args;
static void sleep_short(void)
{
usleep(1000);
}
static thread_state thread_state_get(pid_t tid)
{
const char *header = "State:\t";
char tmp[256], *p, c;
FILE *fp;
snprintf(tmp, sizeof(tmp), "/proc/%d/status", tid);
fp = fopen(tmp, "r");
if (!fp)
return THR_STATE_UNKNOWN;
while (fgets(tmp, sizeof(tmp), fp)) {
p = strstr(tmp, header);
if (p) {
/* For example, "State:\tD (disk sleep)" */
c = *(p + strlen(header));
return c == 'D' ?
THR_STATE_UNINTERRUPTIBLE : THR_STATE_UNKNOWN;
}
}
return THR_STATE_UNKNOWN;
}
static void thread_state_until(pid_t tid, thread_state state)
{
thread_state s;
do {
s = thread_state_get(tid);
sleep_short();
} while (s != state);
}
static void *uffd_mmap_changing_thread(void *opaque)
{
mmap_changing_thread_args *args = opaque;
uffd_global_test_opts_t *gopts = args->gopts;
volatile pid_t *pid = args->pid;
int ret;
/* Unfortunately, it's only fetch-able from the thread itself.. */
assert(*pid == 0);
*pid = syscall(SYS_gettid);
/* Inject an event, this will hang solid until the event read */
ret = madvise(gopts->area_dst, gopts->page_size, MADV_REMOVE);
if (ret)
err("madvise(MADV_REMOVE) failed");
return NULL;
}
static void uffd_consume_message(uffd_global_test_opts_t *gopts)
{
struct uffd_msg msg = { 0 };
while (uffd_read_msg(gopts, &msg));
}
static void uffd_mmap_changing_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
{
/*
* This stores the real PID (which can be different from how tid is
* defined..) for the child thread, 0 means not initialized.
*/
pid_t pid = 0;
pthread_t tid;
int ret;
mmap_changing_thread_args args = { gopts, &pid };
if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
true, false, false))
err("uffd_register() failed");
/* Create a thread to generate the racy event */
ret = pthread_create(&tid, NULL, uffd_mmap_changing_thread, &args);
if (ret)
err("pthread_create() failed");
/*
* Wait until the thread setup the pid. Use volatile to make sure
* it reads from RAM not regs.
*/
while (!(volatile pid_t)pid)
sleep_short();
/* Wait until the thread hangs at REMOVE event */
thread_state_until(pid, THR_STATE_UNINTERRUPTIBLE);
if (!uffdio_mmap_changing_test_copy(gopts->uffd))
return;
if (!uffdio_mmap_changing_test_zeropage(gopts->uffd))
return;
if (!uffdio_mmap_changing_test_move(gopts->uffd))
return;
if (!uffdio_mmap_changing_test_poison(gopts->uffd))
return;
if (!uffdio_mmap_changing_test_continue(gopts->uffd))
return;
/*
* All succeeded above! Recycle everything. Start by reading the
* event so as to kick the thread roll again..
*/
uffd_consume_message(gopts);
ret = pthread_join(tid, NULL);
assert(ret == 0);
uffd_test_pass();
}
static int prevent_hugepages(uffd_global_test_opts_t *gopts, const char **errmsg)
{
/* This should be done before source area is populated */
if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_NOHUGEPAGE)) {
/* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */
if (errno != EINVAL) {
if (errmsg)
*errmsg = "madvise(MADV_NOHUGEPAGE) failed";
return -errno;
}
}
return 0;
}
static int request_hugepages(uffd_global_test_opts_t *gopts, const char **errmsg)
{
/* This should be done before source area is populated */
if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_HUGEPAGE)) {
if (errmsg) {
*errmsg = (errno == EINVAL) ?
"CONFIG_TRANSPARENT_HUGEPAGE is not set" :
"madvise(MADV_HUGEPAGE) failed";
}
return -errno;
}
return 0;
}
struct uffd_test_case_ops uffd_move_test_case_ops = {
.post_alloc = prevent_hugepages,
};
struct uffd_test_case_ops uffd_move_test_pmd_case_ops = {
.post_alloc = request_hugepages,
};
/*
* Test the returned uffdio_register.ioctls with different register modes.
* Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test.
*/
static void
do_register_ioctls_test(uffd_global_test_opts_t *gopts,
uffd_test_args_t *args,
bool miss,
bool wp,
bool minor)
{
uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE);
mem_type_t *mem_type = args->mem_type;
int ret;
ret = uffd_register_with_ioctls(gopts->uffd, gopts->area_dst, gopts->page_size,
miss, wp, minor, &ioctls);
/*
* Handle special cases of UFFDIO_REGISTER here where it should
* just fail with -EINVAL first..
*
* Case 1: register MINOR on anon
* Case 2: register with no mode selected
*/
if ((minor && (mem_type->mem_flag == MEM_ANON)) ||
(!miss && !wp && !minor)) {
if (ret != -EINVAL)
err("register (miss=%d, wp=%d, minor=%d) failed "
"with wrong errno=%d", miss, wp, minor, ret);
return;
}
/* UFFDIO_REGISTER should succeed, then check ioctls returned */
if (miss)
expected |= BIT_ULL(_UFFDIO_COPY);
if (wp)
expected |= BIT_ULL(_UFFDIO_WRITEPROTECT);
if (minor)
expected |= BIT_ULL(_UFFDIO_CONTINUE);
if ((ioctls & expected) != expected)
err("unexpected uffdio_register.ioctls "
"(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "
"returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);
if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size))
err("unregister");
}
static void uffd_register_ioctls_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
{
int miss, wp, minor;
for (miss = 0; miss <= 1; miss++)
for (wp = 0; wp <= 1; wp++)
for (minor = 0; minor <= 1; minor++)
do_register_ioctls_test(gopts, args, miss, wp, minor);
uffd_test_pass();
}
uffd_test_case_t uffd_tests[] = {
{
/* Test returned uffdio_register.ioctls. */
.name = "register-ioctls",
.uffd_fn = uffd_register_ioctls_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS |
UFFD_FEATURE_MISSING_SHMEM |
UFFD_FEATURE_PAGEFAULT_FLAG_WP |
UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
UFFD_FEATURE_MINOR_HUGETLBFS |
UFFD_FEATURE_MINOR_SHMEM,
},
{
.name = "zeropage",
.uffd_fn = uffd_zeropage_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = 0,
},
{
.name = "move",
.uffd_fn = uffd_move_test,
.mem_targets = MEM_ANON,
.uffd_feature_required = UFFD_FEATURE_MOVE,
.test_case_ops = &uffd_move_test_case_ops,
},
{
.name = "move-pmd",
.uffd_fn = uffd_move_pmd_test,
.mem_targets = MEM_ANON,
.uffd_feature_required = UFFD_FEATURE_MOVE,
.test_case_ops = &uffd_move_test_pmd_case_ops,
},
{
.name = "move-pmd-split",
.uffd_fn = uffd_move_pmd_split_test,
.mem_targets = MEM_ANON,
.uffd_feature_required = UFFD_FEATURE_MOVE,
.test_case_ops = &uffd_move_test_pmd_case_ops,
},
{
.name = "wp-fork",
.uffd_fn = uffd_wp_fork_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
},
{
.name = "wp-fork-with-event",
.uffd_fn = uffd_wp_fork_with_event_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
/* when set, child process should inherit uffd-wp bits */
UFFD_FEATURE_EVENT_FORK,
},
{
.name = "wp-fork-pin",
.uffd_fn = uffd_wp_fork_pin_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
},
{
.name = "wp-fork-pin-with-event",
.uffd_fn = uffd_wp_fork_pin_with_event_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
/* when set, child process should inherit uffd-wp bits */
UFFD_FEATURE_EVENT_FORK,
},
{
.name = "wp-unpopulated",
.uffd_fn = uffd_wp_unpopulated_test,
.mem_targets = MEM_ANON,
.uffd_feature_required =
UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED,
},
{
.name = "minor",
.uffd_fn = uffd_minor_test,
.mem_targets = MEM_SHMEM | MEM_HUGETLB,
.uffd_feature_required =
UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM,
},
{
.name = "minor-wp",
.uffd_fn = uffd_minor_wp_test,
.mem_targets = MEM_SHMEM | MEM_HUGETLB,
.uffd_feature_required =
UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM |
UFFD_FEATURE_PAGEFAULT_FLAG_WP |
/*
* HACK: here we leveraged WP_UNPOPULATED to detect whether
* minor mode supports wr-protect. There's no feature flag
* for it so this is the best we can test against.
*/
UFFD_FEATURE_WP_UNPOPULATED,
},
{
.name = "minor-collapse",
.uffd_fn = uffd_minor_collapse_test,
/* MADV_COLLAPSE only works with shmem */
.mem_targets = MEM_SHMEM,
/* We can't test MADV_COLLAPSE, so try our luck */
.uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM,
},
{
.name = "rwp-async",
.uffd_fn = uffd_rwp_async_test,
.mem_targets = MEM_ALL,
.uffd_feature_required =
UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
},
{
.name = "rwp-sync",
.uffd_fn = uffd_rwp_sync_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = UFFD_FEATURE_RWP,
},
{
.name = "rwp-pagemap",
.uffd_fn = uffd_rwp_pagemap_test,
.mem_targets = MEM_ALL,
.uffd_feature_required =
UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
},
{
.name = "rwp-mprotect",
.uffd_fn = uffd_rwp_mprotect_test,
.mem_targets = MEM_ALL,
.uffd_feature_required =
UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
},
{
.name = "rwp-gup",
.uffd_fn = uffd_rwp_gup_test,
.mem_targets = MEM_ALL,
.uffd_feature_required =
UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
},
{
.name = "rwp-async-toggle",
.uffd_fn = uffd_rwp_async_toggle_test,
.mem_targets = MEM_ALL,
.uffd_feature_required =
UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
},
{
.name = "rwp-close",
.uffd_fn = uffd_rwp_close_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = UFFD_FEATURE_RWP,
},
{
.name = "rwp-fork",
.uffd_fn = uffd_rwp_fork_test,
.mem_targets = MEM_ALL,
.uffd_feature_required =
UFFD_FEATURE_RWP | UFFD_FEATURE_EVENT_FORK,
},
{
.name = "rwp-fork-pin",
.uffd_fn = uffd_rwp_fork_pin_test,
.mem_targets = MEM_ANON,
.uffd_feature_required =
UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC |
UFFD_FEATURE_EVENT_FORK,
},
{
.name = "rwp-swap-cow",
.uffd_fn = uffd_rwp_swap_cow_test,
.mem_targets = MEM_ANON,
.uffd_feature_required = UFFD_FEATURE_RWP,
},
{
.name = "rwp-wp-exclusive",
.uffd_fn = uffd_rwp_wp_exclusive_test,
.mem_targets = MEM_ALL,
.uffd_feature_required =
UFFD_FEATURE_RWP |
UFFD_FEATURE_PAGEFAULT_FLAG_WP |
UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
},
{
.name = "sigbus",
.uffd_fn = uffd_sigbus_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = UFFD_FEATURE_SIGBUS |
UFFD_FEATURE_EVENT_FORK,
},
{
.name = "sigbus-wp",
.uffd_fn = uffd_sigbus_wp_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = UFFD_FEATURE_SIGBUS |
UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP |
UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
},
{
.name = "events",
.uffd_fn = uffd_events_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE,
},
{
.name = "events-wp",
.uffd_fn = uffd_events_wp_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE |
UFFD_FEATURE_PAGEFAULT_FLAG_WP |
UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
},
{
.name = "poison",
.uffd_fn = uffd_poison_test,
.mem_targets = MEM_ALL,
.uffd_feature_required = UFFD_FEATURE_POISON,
},
{
.name = "mmap-changing",
.uffd_fn = uffd_mmap_changing_test,
/*
* There's no point running this test over all mem types as
* they share the same code paths.
*
* Choose shmem for simplicity, because (1) shmem supports
* MINOR mode to cover UFFDIO_CONTINUE, and (2) shmem is
* almost always available (unlike hugetlb). Here we
* abused SHMEM for UFFDIO_MOVE, but the test we want to
* cover doesn't yet need the correct memory type..
*/
.mem_targets = MEM_SHMEM,
/*
* Any UFFD_FEATURE_EVENT_* should work to trigger the
* race logically, but choose the simplest (REMOVE).
*
* Meanwhile, since we'll cover quite a few new ioctl()s
* (CONTINUE, POISON, MOVE), skip this test for old kernels
* by choosing all of them.
*/
.uffd_feature_required = UFFD_FEATURE_EVENT_REMOVE |
UFFD_FEATURE_MOVE | UFFD_FEATURE_POISON |
UFFD_FEATURE_MINOR_SHMEM,
},
};
static void usage(const char *prog)
{
printf("usage: %s [-f TESTNAME]\n", prog);
puts("");
puts(" -f: test name to filter (e.g., event)");
puts(" -h: show the help msg");
puts(" -l: list tests only");
puts("");
exit(KSFT_FAIL);
}
static int uffd_count_tests(int n_tests, int n_mems, const char *test_filter)
{
uffd_test_case_t *test;
int i, j, count = 0;
if (!test_filter)
count += 2; /* test_uffd_api(false) + test_uffd_api(true) */
for (i = 0; i < n_tests; i++) {
test = &uffd_tests[i];
if (test_filter && !strstr(test->name, test_filter))
continue;
for (j = 0; j < n_mems; j++)
if (test->mem_targets & mem_types[j].mem_flag)
count++;
}
return count;
}
static unsigned long uffd_setup_hugetlb(void)
{
unsigned long nr_hugepages, hp_size;
hugetlb_save_settings();
hp_size = default_huge_page_size();
if (!hp_size)
return 0;
/* need twice UFFD_TEST_MEM_SIZE, one for src area and one for dst */
nr_hugepages = 2 * MAX(UFFD_TEST_MEM_SIZE, hp_size * 2) / hp_size;
hugetlb_set_nr_default_pages(nr_hugepages);
if (hugetlb_free_default_pages() < nr_hugepages)
return 0;
return hp_size;
}
int main(int argc, char *argv[])
{
int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t);
int n_mems = sizeof(mem_types) / sizeof(mem_type_t);
const char *test_filter = NULL;
unsigned long hugepage_size;
bool list_only = false;
uffd_test_case_t *test;
mem_type_t *mem_type;
uffd_test_args_t args;
const char *errmsg;
int i, j, opt;
while ((opt = getopt(argc, argv, "f:hl")) != -1) {
switch (opt) {
case 'f':
test_filter = optarg;
break;
case 'l':
list_only = true;
break;
case 'h':
default:
/* Unknown */
usage(argv[0]);
break;
}
}
if (list_only) {
for (i = 0; i < n_tests; i++) {
test = &uffd_tests[i];
if (test_filter && !strstr(test->name, test_filter))
continue;
printf("%s\n", test->name);
}
return KSFT_PASS;
}
hugepage_size = uffd_setup_hugetlb();
ksft_print_header();
ksft_set_plan(uffd_count_tests(n_tests, n_mems, test_filter));
if (!test_filter) {
test_uffd_api(false);
test_uffd_api(true);
}
for (i = 0; i < n_tests; i++) {
test = &uffd_tests[i];
if (test_filter && !strstr(test->name, test_filter))
continue;
for (j = 0; j < n_mems; j++) {
mem_type = &mem_types[j];
/* Initialize global test options */
uffd_global_test_opts_t gopts = { 0 };
gopts.map_shared = mem_type->shared;
uffd_test_ops = mem_type->mem_ops;
uffd_test_case_ops = test->test_case_ops;
if (!(test->mem_targets & mem_type->mem_flag))
continue;
uffd_test_start("%s on %s", test->name, mem_type->name);
if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB)) {
gopts.page_size = hugepage_size;
if (gopts.page_size == 0) {
uffd_test_skip("not enough HugeTLB pages");
continue;
}
} else {
gopts.page_size = psize();
}
/* Ensure we have at least 2 pages */
gopts.nr_pages = MAX(UFFD_TEST_MEM_SIZE, gopts.page_size * 2)
/ gopts.page_size;
gopts.nr_parallel = 1;
/* Initialize test arguments */
args.mem_type = mem_type;
if (!uffd_feature_supported(test)) {
uffd_test_skip("feature missing");
continue;
}
if (uffd_test_ctx_init(&gopts, test->uffd_feature_required, &errmsg)) {
uffd_test_skip(errmsg);
continue;
}
/*
* RWP tracks protection on ptes; a THP-backed shmem/anon
* range (e.g. shmem_enabled=always) would split on
* rwprotect and change behaviour under the test. Keep
* such ranges off THP. hugetlb is huge by definition and
* rejects MADV_NOHUGEPAGE, so skip it.
*/
if ((test->uffd_feature_required & UFFD_FEATURE_RWP) &&
!(mem_type->mem_flag & (MEM_HUGETLB | MEM_HUGETLB_PRIVATE))) {
unsigned long len = gopts.nr_pages * gopts.page_size;
/*
* EINVAL means CONFIG_TRANSPARENT_HUGEPAGE=n:
* nothing to opt out of.
*/
if (madvise(gopts.area_dst, len, MADV_NOHUGEPAGE) &&
errno != EINVAL)
err("madvise(MADV_NOHUGEPAGE)");
}
test->uffd_fn(&gopts, &args);
uffd_test_ctx_clear(&gopts);
}
}
ksft_finished();
}
#else /* __NR_userfaultfd */
#warning "missing __NR_userfaultfd definition"
int main(void)
{
ksft_print_header();
ksft_exit_skip("missing __NR_userfaultfd definition\n");
}
#endif /* __NR_userfaultfd */
