diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index da963e3a14cfdf..72fd9fe2c30702 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -2029,6 +2029,9 @@ config ARCH_SUPPORTS_MULTIKERNEL config ARCH_HAS_MK_HOST_PARK def_bool X86_64 +config ARCH_HAS_MK_FORCE_STOP + def_bool X86_64 + config ARCH_SUPPORTS_CRASH_DUMP def_bool X86_64 || (X86_32 && HIGHMEM) diff --git a/arch/x86/include/asm/multikernel.h b/arch/x86/include/asm/multikernel.h index 3e53e9cbb82c56..2317a63817f8ab 100644 --- a/arch/x86/include/asm/multikernel.h +++ b/arch/x86/include/asm/multikernel.h @@ -8,6 +8,13 @@ #define MK_SPAWN_F_SECONDARY BIT(0) /* Secondary CPU joining existing kernel */ #define MK_SPAWN_F_REPARK BIT(1) /* Move the CPU to another park slot */ +/* Shared C/assembly protocol. All participating kernels must use it. */ +#define MK_SLOT_IDLE 0 +#define MK_SLOT_PENDING 1 +#define MK_SLOT_BUSY 2 +#define MK_SLOT_ACK 3 +#define MK_SLOT_QUARANTINED 4 + #ifndef __ASSEMBLY__ #include @@ -126,7 +133,7 @@ struct mk_spawn_context { unsigned long ctrl_size; /* park page, page tables. Spawn must not reuse it */ u32 target_apic_id; /* Target CPU's APIC ID */ u32 flags; /* MK_SPAWN_F_* flags */ - u32 ready; /* Signal flag */ + u32 ready; /* MK_SLOT_* state, not a boolean */ u32 reserved; /* Padding for alignment */ /* Keep all existing context offsets unchanged. */ struct boot_params bp; /* Standard x86 boot params */ @@ -184,12 +191,12 @@ struct mk_spawn_context *mk_alloc_spawn_context(struct mk_instance *instance, struct boot_params *mk_spawn_context_boot_params(struct mk_spawn_context *ctx); /* Set up spawn context for kernel boot */ -void mk_set_spawn_context(struct mk_spawn_context *ctx, - unsigned long identity_cr3, - unsigned long kernel_entry, - unsigned long trampoline_virt, - unsigned long trampoline_phys, - unsigned long park_phys); +int mk_set_spawn_context(struct mk_spawn_context *ctx, + unsigned long identity_cr3, + unsigned long kernel_entry, + unsigned long trampoline_virt, + unsigned long trampoline_phys, + unsigned long park_phys); /* Trigger spawn on a pool CPU */ int mk_spawn_cpu(struct mk_instance *instance, int cpu, diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index b2fead7a136ff9..ebd159d47b3261 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -752,12 +752,14 @@ static void *bzImage64_load(struct kimage *image, char *kernel, goto out_free_params; /* For multikernel, setup custom e820 map */ +#ifdef CONFIG_MULTIKERNEL if (image->type == KEXEC_TYPE_MULTIKERNEL) { image->arch.mk_boot_params = bootparam_load_addr; ret = mk_e820_fill(image->mk_instance, params); if (ret) goto out_free_params; } +#endif /* Allocate loader specific data */ ldata = kzalloc_obj(struct bzimage64_data); diff --git a/arch/x86/multikernel/direct_boot.S b/arch/x86/multikernel/direct_boot.S index f326124a2a1e7e..8a38f7660b4176 100644 --- a/arch/x86/multikernel/direct_boot.S +++ b/arch/x86/multikernel/direct_boot.S @@ -322,7 +322,7 @@ SYM_CODE_START_LOCAL_NOALIGN(mk_park_identity) .Lpark_wait: movl MK_CTX_ready(%r12), %eax - cmpl $1, %eax + cmpl $MK_SLOT_PENDING, %eax je .Lpark_pending testq %r15, %r15 @@ -334,7 +334,7 @@ SYM_CODE_START_LOCAL_NOALIGN(mk_park_identity) xorl %edx, %edx monitor movl MK_CTX_ready(%r12), %eax - cmpl $1, %eax + cmpl $MK_SLOT_PENDING, %eax je .Lpark_pending xorl %eax, %eax xorl %ecx, %ecx @@ -346,17 +346,24 @@ SYM_CODE_START_LOCAL_NOALIGN(mk_park_identity) jmp .Lpark_wait .Lpark_pending: - /* Wakeup published; is it for us? */ + /* Advisory filter only; recheck after claiming the current publication. */ movl MK_CTX_target_apic_id(%r12), %eax cmpl %r13d, %eax jne .Lpark_relax /* - * Copy the publication into registers BEFORE claiming: once the - * claim clears ready, the publisher may reuse the slot for the - * next CPU. The claim (lock cmpxchg) makes exactly one of - * waker-revoke and us win; on loss the staged values are dropped. + * Claim before authoritative reads. A reader delayed across slot reuse + * can claim a newer publication, but then must re-read its target and + * every dispatch field. BUSY prevents writes while registers are staged. */ + movl $MK_SLOT_PENDING, %eax + movl $MK_SLOT_BUSY, %ebp + lock cmpxchgl %ebp, MK_CTX_ready(%r12) + jnz .Lpark_wait + movl MK_CTX_target_apic_id(%r12), %eax + cmpl %r13d, %eax + jne .Lpark_wrong_target + movl MK_CTX_flags(%r12), %r11d testl $MK_SPAWN_F_REPARK, %r11d jnz .Lpark_repark @@ -375,8 +382,8 @@ SYM_CODE_START_LOCAL_NOALIGN(mk_park_identity) addq $MK_CTX_bp, %rsi movq MK_CTX_kernel_entry(%r12), %rdx movq MK_CTX_trampoline_phys(%r12), %r8 - movl $1, %eax - xorl %ebp, %ebp + movl $MK_SLOT_BUSY, %eax + movl $MK_SLOT_ACK, %ebp lock cmpxchgl %ebp, MK_CTX_ready(%r12) jnz .Lpark_wait movq %r8, %rcx @@ -395,8 +402,8 @@ SYM_CODE_START_LOCAL_NOALIGN(mk_park_identity) movq MK_CTX_stack(%r12), %rcx movq MK_CTX_secondary_trampoline_phys(%r12), %r8 movq MK_CTX_spawn_cr3(%r12), %r9 - movl $1, %eax - xorl %ebp, %ebp + movl $MK_SLOT_BUSY, %eax + movl $MK_SLOT_ACK, %ebp lock cmpxchgl %ebp, MK_CTX_ready(%r12) jnz .Lpark_wait ANNOTATE_RETPOLINE_SAFE @@ -414,12 +421,15 @@ SYM_CODE_START_LOCAL_NOALIGN(mk_park_identity) * The repark_* dispatch fields are separate from the park_phys/ * park_cr3/self_phys anchor so that publishing a repark on a live * instance context never corrupts the context's own park identity. + * ACK below proves only staged dispatch. Before retiring this page or + * its page table the publisher must obtain a second claim from the + * destination loop, which cannot run until the jump below completes. */ movq MK_CTX_repark_park_cr3(%r12), %r14 movq MK_CTX_repark_park_phys(%r12), %rcx movq MK_CTX_repark_slot_phys(%r12), %r10 - movl $1, %eax - xorl %ebp, %ebp + movl $MK_SLOT_BUSY, %eax + movl $MK_SLOT_ACK, %ebp lock cmpxchgl %ebp, MK_CTX_ready(%r12) jnz .Lpark_wait movq %r10, %r12 @@ -430,6 +440,13 @@ SYM_CODE_START_LOCAL_NOALIGN(mk_park_identity) addq $mk_park_identity - mk_pool_park, %rcx ANNOTATE_RETPOLINE_SAFE jmp *%rcx + +.Lpark_wrong_target: + /* A newer request belongs to another CPU; never undo quarantine. */ + movl $MK_SLOT_BUSY, %eax + movl $MK_SLOT_PENDING, %ebp + lock cmpxchgl %ebp, MK_CTX_ready(%r12) + jmp .Lpark_wait SYM_CODE_END(mk_park_identity) /* diff --git a/arch/x86/multikernel/spawn.c b/arch/x86/multikernel/spawn.c index 8693e7924fbf3c..ec5af8e4a28010 100644 --- a/arch/x86/multikernel/spawn.c +++ b/arch/x86/multikernel/spawn.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include #include @@ -137,13 +139,51 @@ struct boot_params *mk_spawn_context_boot_params(struct mk_spawn_context *ctx) return &ctx->bp; } -void mk_set_spawn_context(struct mk_spawn_context *ctx, - unsigned long identity_cr3, - unsigned long kernel_entry, - unsigned long trampoline_virt, - unsigned long trampoline_phys, - unsigned long park_phys) +/* Cross-kernel writers cannot rely on this kernel's park_lock alone. */ +static void mk_slot_quarantine(struct mk_spawn_context *slot) { + struct mk_spawn_context *own = mk_own_context(); + + xchg(&slot->ready, MK_SLOT_QUARANTINED); + /* A failed wake can still reference our trampoline and pool page tables. */ + if (own) + xchg(&own->ready, MK_SLOT_QUARANTINED); + if (mk_pool && mk_pool->arch.slot) + xchg(&mk_pool->arch.slot->ready, MK_SLOT_QUARANTINED); +} + +static int mk_slot_begin(struct mk_spawn_context *slot) +{ + u32 state; + + if (!slot) + return -ENODEV; + state = cmpxchg(&slot->ready, MK_SLOT_IDLE, MK_SLOT_BUSY); + if (state == MK_SLOT_IDLE) + return 0; + if (state == MK_SLOT_QUARANTINED) { + mk_slot_quarantine(slot); + return -EIO; + } + return -EBUSY; +} + +static bool mk_slot_idle(struct mk_spawn_context *slot) +{ + return !slot || READ_ONCE(slot->ready) == MK_SLOT_IDLE; +} + +int mk_set_spawn_context(struct mk_spawn_context *ctx, + unsigned long identity_cr3, + unsigned long kernel_entry, + unsigned long trampoline_virt, + unsigned long trampoline_phys, + unsigned long park_phys) +{ + int ret = mk_slot_begin(ctx); + + if (ret) + return ret; ctx->identity_cr3 = identity_cr3; ctx->kernel_entry = kernel_entry; ctx->trampoline_virt = trampoline_virt; @@ -160,15 +200,16 @@ void mk_set_spawn_context(struct mk_spawn_context *ctx, ctx->gs_base = 0; ctx->stack = 0; ctx->flags = 0; - ctx->ready = 0; + return cmpxchg(&ctx->ready, MK_SLOT_BUSY, MK_SLOT_IDLE) == MK_SLOT_BUSY ? 0 : -EIO; } /* * Publish a wakeup in @slot for the CPU with @apic_id and wait for the - * parked CPU to claim it. The dispatch fields must be set before the - * call; parked CPUs stage them in registers before claiming, so once the - * claim is observed the slot may be reused. On timeout the publication - * is revoked; if the CPU claims concurrently, the wakeup proceeds. + * parked CPU to acknowledge its staged dispatch. The caller first reserves + * IDLE -> BUSY, then fills dispatch fields. Readers claim PENDING -> BUSY + * before authoritative target/data reads, and ACK only after staging all + * fields. Only the publisher consumes ACK -> IDLE. Timeout permanently + * quarantines the slot: no retry, rewrite or retirement until host reboot. */ static int mk_slot_wake(struct mk_spawn_context *slot, u32 apic_id, const char *what) @@ -178,21 +219,18 @@ static int mk_slot_wake(struct mk_spawn_context *slot, u32 apic_id, int ret; slot->target_apic_id = apic_id; - /* Ensure the publication is visible before setting ready */ - smp_wmb(); - WRITE_ONCE(slot->ready, 1); + if (cmpxchg(&slot->ready, MK_SLOT_BUSY, MK_SLOT_PENDING) != MK_SLOT_BUSY) + return -EIO; - ret = read_poll_timeout(READ_ONCE, ready, !ready, 100, USEC_PER_SEC, + ret = read_poll_timeout(READ_ONCE, ready, ready == MK_SLOT_ACK, 100, USEC_PER_SEC, false, slot->ready); - if (ret) { - if (cmpxchg(&slot->ready, 1, 0) == 1) - pr_err("mk_spawn: %s: CPU (APIC %u) did not claim wakeup on slot %pa - CPU is not parked there\n", - what, apic_id, &slot_phys); - else - ret = 0; - } - - return ret; + if (cmpxchg(&slot->ready, MK_SLOT_ACK, MK_SLOT_IDLE) == MK_SLOT_ACK) + return 0; + /* Bounded even if wrong-target readers repeatedly relinquish BUSY. */ + mk_slot_quarantine(slot); + pr_err("mk_spawn: %s: CPU (APIC %u) unconfirmed on slot %pa; quarantined until host reboot\n", + what, apic_id, &slot_phys); + return ret ?: -EIO; } /* @@ -200,7 +238,7 @@ static int mk_slot_wake(struct mk_spawn_context *slot, u32 apic_id, * at the instance's own context, so the spawn kernel can wake them by * writing memory inside its own partition. */ -/* The instance's set of CPUs parked on its own context, allocated on demand */ +/* CPUs that may use the instance context, including uncertain handoffs. */ static struct mk_cpu_set *mk_slot_set(struct mk_instance *instance) { if (!instance->cpus_on_slot) @@ -217,13 +255,39 @@ static int mk_publish_repark(struct mk_spawn_context *from, u32 apic_id, unsigned long park_phys, unsigned long park_cr3, unsigned long slot_phys, const char *what) { + struct mk_spawn_context *to = __va(slot_phys); + int ret; + lockdep_assert_held(&mk_pool->park_lock); + if (!mk_slot_idle(to)) + return -EBUSY; + ret = mk_slot_begin(from); + if (ret) + return ret; from->repark_park_phys = park_phys; from->repark_park_cr3 = park_cr3; from->repark_slot_phys = slot_phys; from->flags = MK_SPAWN_F_REPARK; - return mk_slot_wake(from, apic_id, what); + ret = mk_slot_wake(from, apic_id, what); + if (ret || from == to) + return ret; + + /* + * ACK confirms staged registers, not departure from the + * old park page/page table. Only the destination loop can claim this + * self-repark, after jumping off both. Keep tracking and old allocations + * until that proof arrives. All destinations are host-managed control + * allocations in the direct map; park_lock serializes both mailboxes. + */ + ret = mk_slot_begin(to); + if (ret) + return ret; + to->repark_park_phys = park_phys; + to->repark_park_cr3 = park_cr3; + to->repark_slot_phys = slot_phys; + to->flags = MK_SPAWN_F_REPARK; + return mk_slot_wake(to, apic_id, "confirm repark arrival"); } static int mk_repark_to_instance(struct mk_instance *instance, @@ -239,9 +303,8 @@ static int mk_repark_to_instance(struct mk_instance *instance, return -ENOMEM; /* - * Guarantee the tracking add after each successful wake cannot - * fail: a CPU that moved but was not recorded would be woken - * through the wrong mailbox forever after. + * Reserve tracking before publication: a timed-out wake may already + * have moved a CPU, so it must remain recorded even on failure. */ ret = mk_cpu_set_reserve(on_slot, mk_cpu_set_count(instance->cpus)); if (ret) @@ -259,13 +322,15 @@ static int mk_repark_to_instance(struct mk_instance *instance, if (mk_cpu_set_contains(on_slot, phys_cpu)) continue; + ret = mk_cpu_set_add(on_slot, phys_cpu); + if (ret) + return ret; + ret = mk_publish_repark(slot, (u32)phys_cpu, ctx->park_phys, ctx->park_cr3, virt_to_phys(ctx), "repark to instance"); if (ret) return ret; - - mk_cpu_set_add(on_slot, phys_cpu); } return 0; @@ -285,6 +350,8 @@ int mk_spawn_cpu(struct mk_instance *instance, int cpu, guard(mutex)(&mk_pool->park_lock); + if (!mk_slot_idle(slot) || !mk_slot_idle(ctx)) + return -EBUSY; /* * Move every secondary still parked on the host slot onto the * instance context, where this instance's kernel wakes them. A @@ -298,26 +365,26 @@ int mk_spawn_cpu(struct mk_instance *instance, int cpu, if (mk_cpu_set_contains(instance->cpus_on_slot, apic_id)) { /* The boot CPU parked on this context when the instance halted */ + ret = mk_slot_begin(ctx); + if (ret) + return ret; ctx->flags = 0; return mk_slot_wake(ctx, apic_id, "re-spawn boot cpu"); } + /* Record possible execution before touching a shared publication. */ + ret = mk_cpu_set_add(instance->cpus_on_slot, apic_id); + if (ret) + return ret; + ret = mk_slot_begin(slot); + if (ret) + return ret; slot->identity_cr3 = ctx->identity_cr3; slot->kernel_entry = ctx->kernel_entry; slot->trampoline_phys = ctx->trampoline_phys; slot->self_phys = virt_to_phys(ctx); slot->flags = 0; - ret = mk_slot_wake(slot, apic_id, "first spawn boot cpu"); - if (!ret) { - /* - * It is running the instance's kernel now, and will park on - * that kernel's context when the instance halts. The add - * cannot fail: mk_repark_to_instance() reserved room for - * every CPU of the instance. - */ - mk_cpu_set_add(instance->cpus_on_slot, apic_id); - } - return ret; + return mk_slot_wake(slot, apic_id, "first spawn boot cpu"); } /** @@ -334,9 +401,9 @@ int mk_spawn_cpu(struct mk_instance *instance, int cpu, * never unwound, even when the spawn itself fails: all of them are carved * from the instance's control block, which only ever grows, and a partial * spawn may already have reparked CPUs onto the new park page, so tearing - * it down would free memory a CPU is executing. A retry picks up exactly - * where this attempt stopped; everything is returned to the pool together - * when the instance's memory is torn down. + * it down would free memory a CPU is executing. Only pre-publication failures + * may be retried; a mailbox timeout quarantines the retained resources until + * host reboot. Ordinary release still requires confirmed destination arrival. */ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, int cpu) @@ -346,6 +413,11 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, void *park_va; int ret; + /* Never rewrite retained executable state, even with empty CPU tracking. */ + if (!mk_pool || !mk_slot_idle(mk_pool->arch.slot) || + !mk_slot_idle(instance->arch.spawn_ctx)) + return -EBUSY; + if (!instance->arch.ident_pgt) { struct mk_ident_pgtable *pgt; struct mk_memory_region *region; @@ -441,12 +513,14 @@ int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, if (ret) return ret; - mk_set_spawn_context(instance->arch.spawn_ctx, - mk_get_identity_cr3(instance->arch.ident_pgt), - image->arch.mk_kernel_entry, - (unsigned long)instance->arch.trampoline_va, - virt_to_phys(instance->arch.trampoline_va), - park_phys); + ret = mk_set_spawn_context(instance->arch.spawn_ctx, + mk_get_identity_cr3(instance->arch.ident_pgt), + image->arch.mk_kernel_entry, + (unsigned long)instance->arch.trampoline_va, + virt_to_phys(instance->arch.trampoline_va), + park_phys); + if (ret) + return ret; instance->arch.spawn_ctx->boot_lps = cpu_data(cpu).loops_per_jiffy; if (!instance->arch.spawn_ctx->boot_lps) instance->arch.spawn_ctx->boot_lps = loops_per_jiffy; @@ -519,22 +593,25 @@ int mk_repark_cpu_to_instance(struct mk_instance *instance, mk_phys_cpu_t phys_c if (!ctx || !slot || !ctx->park_phys || !on_slot) return -ENODEV; - /* Make the tracking add below infallible before moving the CPU */ + /* Reserve tracking before moving the CPU, including failed wakeups. */ ret = mk_cpu_set_reserve(on_slot, 1); if (ret) return ret; guard(mutex)(&mk_pool->park_lock); + if (!mk_slot_idle(ctx) || !mk_slot_idle(slot)) + return -EBUSY; if (mk_cpu_set_contains(on_slot, phys_cpu)) return 0; - ret = mk_publish_repark(slot, (u32)phys_cpu, ctx->park_phys, + ret = mk_cpu_set_add(on_slot, phys_cpu); + if (ret) + return ret; + + return mk_publish_repark(slot, (u32)phys_cpu, ctx->park_phys, ctx->park_cr3, instance->arch.spawn_ctx_phys, "repark hot-added cpu"); - if (!ret) - mk_cpu_set_add(on_slot, phys_cpu); - return ret; } /** @@ -551,8 +628,12 @@ int mk_repark_cpu_to_host(struct mk_instance *instance, mk_phys_cpu_t phys_cpu) struct mk_spawn_context *ctx = instance->arch.spawn_ctx; int ret; + if (!mk_pool) + return -ENODEV; guard(mutex)(&mk_pool->park_lock); + if (!mk_slot_idle(ctx) || !mk_slot_idle(mk_pool->arch.slot)) + return -EBUSY; /* Already on the host slot: nothing watches this context for it */ if (!mk_cpu_set_contains(instance->cpus_on_slot, phys_cpu)) return 0; @@ -585,37 +666,24 @@ int mk_arch_confirm_parked(struct mk_instance *instance, mk_phys_cpu_t phys_cpu) struct mk_spawn_context *ctx = instance->arch.spawn_ctx; int ret; - if (!ctx || !ctx->park_phys) + if (!mk_pool || !mk_pool->arch.slot) return -ENODEV; guard(mutex)(&mk_pool->park_lock); - ret = mk_publish_repark(ctx, (u32)phys_cpu, ctx->park_phys, - ctx->park_cr3, instance->arch.spawn_ctx_phys, - "confirm parked"); - if (!ret || !mk_pool || !mk_pool->arch.slot) - return ret; - - /* - * Not on the context we recorded. It may still be parked on the host - * slot, which our bookkeeping can miss: an instance rebuilt while a - * CPU was elsewhere starts with an empty set. Ask the host slot the - * same way. A claim there answers the question this function exists - * for - the CPU is parked, not running the image we are about to - * overwrite - so believe it and correct the record. - */ - if (!mk_publish_repark(mk_pool->arch.slot, (u32)phys_cpu, - mk_pool->arch.park_phys, mk_pool->arch.cr3, - mk_pool->arch.slot_phys, "confirm parked (host slot)")) { - pr_warn("mk_spawn: CPU %llu was parked on the host slot, not instance %d\n", - phys_cpu, instance->id); - mk_cpu_set_del(instance->cpus_on_slot, phys_cpu); - return 0; + if (!mk_slot_idle(ctx) || !mk_slot_idle(mk_pool->arch.slot)) + return -EBUSY; + /* Select once from ownership history; never probe another slot on timeout. */ + if (mk_cpu_set_contains(instance->cpus_on_slot, phys_cpu)) { + if (!ctx || !ctx->park_phys) + return -ENODEV; + return mk_publish_repark(ctx, (u32)phys_cpu, ctx->park_phys, + ctx->park_cr3, instance->arch.spawn_ctx_phys, + "confirm parked"); } - - pr_err("mk_spawn: CPU %llu is parked nowhere: not on instance %d context %pa, not on host slot %pa\n", - phys_cpu, instance->id, &instance->arch.spawn_ctx_phys, - &mk_pool->arch.slot_phys); + ret = mk_publish_repark(mk_pool->arch.slot, (u32)phys_cpu, + mk_pool->arch.park_phys, mk_pool->arch.cr3, + mk_pool->arch.slot_phys, "confirm parked (host slot)"); return ret; } @@ -638,8 +706,12 @@ int mk_repark_instance_to_host(struct mk_instance *instance) unsigned int i; int failed = 0; - if (mk_cpu_set_empty(on_slot) || !ctx || !mk_pool || !mk_pool->arch.slot) + if (!mk_slot_idle(ctx) || (mk_pool && !mk_slot_idle(mk_pool->arch.slot))) + return -EBUSY; + if (mk_cpu_set_empty(on_slot)) return 0; + if (!ctx || !mk_pool || !mk_pool->arch.slot) + return -ENODEV; guard(mutex)(&mk_pool->park_lock); @@ -910,16 +982,29 @@ static int mk_build_trampoline_pgtable(unsigned long trampoline_va, * Wakes a pool CPU to join this spawn kernel as a secondary CPU. */ /* - * Where to publish a wakeup for @apicid. A CPU the fence parked on the - * host's slot is woken from that slot, which this kernel adopted as - * its own after fencing (host_instance->pool_slot_phys); every other - * secondary is woken from this kernel's own context. The returned slot - * is memremapped when it differs from @own and must be released with + * Where to publish a wakeup for @apicid. A CPU donated to this kernel's + * local pool watches the local pool slot, even if this spawn originally + * received it on its parent's slot. Otherwise a CPU the fence parked on + * the host's slot is woken from that adopted slot; initial secondaries + * are woken from this kernel's own boot context. Do not change the boot + * context: it still supplies this kernel's dispatch and final halt target. + * The caller holds park_lock and cpus_lock while selecting and publishing. + * The returned slot is memremapped when it differs from @own and released with * mk_put_wake_slot(). */ static struct mk_spawn_context *mk_wake_slot(u32 apicid, struct mk_spawn_context *own) { + if (mk_pool) { + lockdep_assert_held(&mk_pool->park_lock); + lockdep_assert_held(&mk_pool->cpus_lock); + if (mk_cpu_set_contains(mk_pool->cpus, apicid)) { + if (!mk_pool->arch.slot_phys) + return NULL; + return memremap(mk_pool->arch.slot_phys, + sizeof(struct mk_spawn_context), MEMREMAP_WB); + } + } if (host_instance && host_instance->pool_slot_phys && mk_cpu_set_contains(host_instance->cpus_on_slot, apicid)) return memremap(host_instance->pool_slot_phys, @@ -937,6 +1022,7 @@ static void mk_put_wake_slot(struct mk_spawn_context *slot, int multikernel_wakeup_secondary_cpu_64(u32 apicid, unsigned long start_eip, unsigned int cpu) { + struct mk_pool *pool = mk_pool; struct mk_spawn_context *own = mk_own_context(); struct mk_spawn_context *slot; unsigned long trampoline_phys, trampoline_va; @@ -952,6 +1038,8 @@ int multikernel_wakeup_secondary_cpu_64(u32 apicid, unsigned long start_eip, pr_err("mk_spawn: trampoline not prepared\n"); return -ENODEV; } + if (!mk_slot_idle(own)) + return -EBUSY; /* mk_prepare_trampoline() put this kernel's own copy in the page */ trampoline_phys = own->trampoline_phys; trampoline_va = (unsigned long)__va(trampoline_phys); @@ -981,17 +1069,41 @@ int multikernel_wakeup_secondary_cpu_64(u32 apicid, unsigned long start_eip, if (ret) return ret; + /* Initial secondary boot precedes baseline creation (pool == NULL). */ + if (pool) { + mutex_lock(&pool->park_lock); + mutex_lock(&pool->cpus_lock); + if (!mk_slot_idle(pool->arch.slot)) { + ret = -EBUSY; + goto out_unlock; + } + } + if (pool && mk_cpu_set_contains(pool->cpus, apicid)) { + /* + * The park loop jumps to the physical trampoline before that + * trampoline switches CR3. A nested pool maps only its donated + * chunks; the parent-given boot trampoline can be outside them. + * Map its initial fetch in the OLD park table before publishing. + * The mapping and any allocated table pages live with this pool. + */ + if (!pool->arch.pgt) { + ret = -ENODEV; + goto out_unlock; + } + ret = mk_ident_map_range(pool->arch.pgt, trampoline_phys, + trampoline_phys + PAGE_SIZE); + if (ret) + goto out_unlock; + } slot = mk_wake_slot(apicid, own); - if (!slot) - return -ENODEV; + if (!slot) { + ret = -ENODEV; + goto out_unlock; + } - /* - * Revoke any unconsumed wakeup left over from a failed earlier - * bringup, so a parked CPU cannot wake mid-rewrite and read a mix - * of old and new fields. A CPU that already claimed the context - * still sees the old, consistent snapshot. - */ - cmpxchg(&slot->ready, 1, 0); + ret = mk_slot_begin(slot); + if (ret) + goto out_slot; /* * The dispatch is this kernel's (its trampoline, its secondary @@ -1016,11 +1128,15 @@ int multikernel_wakeup_secondary_cpu_64(u32 apicid, unsigned long start_eip, * no IPI is needed and none could be delivered anyway, since * parked CPUs keep interrupts disabled. */ - smp_wmb(); - WRITE_ONCE(slot->ready, 1); - + ret = mk_slot_wake(slot, apicid, "secondary cpu"); +out_slot: mk_put_wake_slot(slot, own); - return 0; +out_unlock: + if (pool) { + mutex_unlock(&pool->cpus_lock); + mutex_unlock(&pool->park_lock); + } + return ret; } struct mk_ident_pgtable { @@ -1048,7 +1164,8 @@ static void *mk_pool_ctrl_alloc(size_t size) static void mk_pool_ctrl_free(phys_addr_t phys, size_t size) { - mk_pool_unmap(phys, size); + if (WARN_ON_ONCE(mk_pool_unmap(phys, size))) + return; multikernel_free(phys, size); } @@ -1427,7 +1544,7 @@ static int __mk_pool_park_setup(void) lockdep_assert_held(&mk_pool->park_lock); if (mk_pool->arch.slot) - return 0; + return mk_slot_idle(mk_pool->arch.slot) ? 0 : -EBUSY; if (mk_pool_empty()) return -ENODEV; @@ -1503,9 +1620,10 @@ int mk_arch_pool_map(phys_addr_t start, size_t size) return 0; } -void mk_arch_pool_unmap(phys_addr_t start, size_t size) +int mk_arch_pool_unmap(phys_addr_t start, size_t size) { kernel_physical_mapping_remove(start, start + size); + return 0; } int mk_arch_pool_chunk_added(phys_addr_t start, size_t size) @@ -1514,6 +1632,8 @@ int mk_arch_pool_chunk_added(phys_addr_t start, size_t size) return -ENODEV; guard(mutex)(&mk_pool->park_lock); + if (!mk_slot_idle(mk_pool->arch.slot)) + return -EBUSY; /* * A pool emptied of memory has no park area, so the chunk that @@ -1674,6 +1794,8 @@ int mk_pool_park_teardown(void) if (!mk_pool->arch.slot) return 0; + if (!mk_slot_idle(mk_pool->arch.slot) || !mk_slot_idle(mk_own_context())) + return -EBUSY; if (!mk_pool_cpus_returned()) return -EBUSY; @@ -1852,9 +1974,40 @@ EXPORT_SYMBOL_GPL(multikernel_restore_ap); * machine including the host. Park every CPU in the pool wait loop * instead, returning them to the multikernel pool. */ +static bool mk_halt_quiesced; + +static void mk_halt_shutdown_prepare(void *unused) +{ + /* Ordinary kexec also runs syscore shutdown, then reopens CPU hotplug. */ + if (panic_in_progress() || kexec_in_progress) + return; + if (system_state != SYSTEM_HALT && system_state != SYSTEM_POWER_OFF && + system_state != SYSTEM_RESTART) + return; + /* + * migrate_to_reboot_cpu() has already drained native CPU hotplug. + * Exclude local controllers and received resource mutations until park. + * Never wait for a contended controller from this terminal path. + */ + if (!mutex_trylock(&mk_overlay_mutex)) + return; + /* The machine-stop path acquires this permit before publishing HALTED. */ + smp_store_release(&mk_halt_quiesced, true); +} + +static const struct syscore_ops mk_halt_syscore_ops = { + .shutdown = mk_halt_shutdown_prepare, +}; + +static struct syscore mk_halt_syscore = { + .ops = &mk_halt_syscore_ops, +}; + static void __noreturn mk_machine_halt(void) { - mk_halt_to_pool(); + /* Pair with syscore's permit publication while it retains the mutex. */ + mk_halt_to_pool(smp_load_acquire(&mk_halt_quiesced) && + !panic_in_progress() && !kexec_in_progress); } static void __noreturn mk_machine_restart(char *cmd) @@ -1884,11 +2037,12 @@ static int __init mk_spawn_reboot_init(void) machine_ops.restart = mk_machine_restart; machine_ops.emergency_restart = mk_machine_halt; smp_ops.stop_this_cpu = mk_stop_this_cpu; + register_syscore(&mk_halt_syscore); /* * A panicked spawn kernel spinning in the panic loop holds its * CPUs hostage. "Reboot on panic" is mk_machine_halt here, which - * notifies the host and parks, so make it the default; the panic + * parks without authorizing host reuse, so make it the default. Panic * output has already been flushed to the console by then. */ if (!panic_timeout) diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index 875abdc9942e13..1f7a359c752da3 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -45,11 +45,20 @@ static void change_cpu_under_node(struct cpu *cpu, cpu->node_id = to_nid; } -static int cpu_subsys_online(struct device *dev) +/* Caller serializes the CPU device with device_hotplug_lock and device_lock. */ +void cpu_device_update_node(struct device *dev, int from_nid) { struct cpu *cpu = container_of(dev, struct cpu, dev); + int to_nid = cpu_to_node(dev->id); + + if (from_nid != to_nid) + change_cpu_under_node(cpu, from_nid, to_nid); +} + +static int cpu_subsys_online(struct device *dev) +{ int cpuid = dev->id; - int from_nid, to_nid; + int from_nid; int ret; int retries = 0; @@ -79,9 +88,7 @@ static int cpu_subsys_online(struct device *dev) * When hot adding memory to memoryless node and enabling a cpu * on the node, node number of the cpu may internally change. */ - to_nid = cpu_to_node(cpuid); - if (from_nid != to_nid) - change_cpu_under_node(cpu, from_nid, to_nid); + cpu_device_update_node(dev, from_nid); return ret; } diff --git a/drivers/tty/mktty.c b/drivers/tty/mktty.c index f544bf342a09d0..c932445b2b3b4c 100644 --- a/drivers/tty/mktty.c +++ b/drivers/tty/mktty.c @@ -20,12 +20,15 @@ */ #include +#include +#include #include #include #include #include #include #include +#include #include #define MKTTY_IPI_TYPE 0x4D4B5459U /* "MKTY" */ @@ -54,20 +57,22 @@ struct mktty_host_conn { int instance_id; /* -1 = not connected */ struct list_head list; wait_queue_head_t wait; - spinlock_t rx_lock; + struct work_struct wake_work; + raw_spinlock_t rx_lock; char *rx_buf; int rx_head; int rx_tail; }; static LIST_HEAD(mktty_host_conns); -static DEFINE_SPINLOCK(mktty_host_conns_lock); +static DEFINE_RAW_SPINLOCK(mktty_host_conns_lock); static struct mk_ipi_handler *mktty_host_handler; static struct mktty_host_conn *mktty_find_conn(int instance_id) { struct mktty_host_conn *conn; + lockdep_assert_held(&mktty_host_conns_lock); list_for_each_entry(conn, &mktty_host_conns, list) { if (conn->instance_id == instance_id) return conn; @@ -75,9 +80,25 @@ static struct mktty_host_conn *mktty_find_conn(int instance_id) return NULL; } +static void mktty_host_wake_work(struct work_struct *work) +{ + struct mktty_host_conn *conn = container_of(work, struct mktty_host_conn, + wake_work); + unsigned long flags; + bool readable; + + /* Pair with RX publication even when queue_work coalesces an enqueue. */ + raw_spin_lock_irqsave(&conn->rx_lock, flags); + readable = conn->rx_head != conn->rx_tail; + raw_spin_unlock_irqrestore(&conn->rx_lock, flags); + if (readable) + wake_up_interruptible(&conn->wait); +} + static int mktty_host_open(struct inode *inode, struct file *filp) { struct mktty_host_conn *conn; + unsigned long flags; conn = kzalloc(sizeof(*conn), GFP_KERNEL); if (!conn) @@ -91,13 +112,14 @@ static int mktty_host_open(struct inode *inode, struct file *filp) conn->instance_id = -1; init_waitqueue_head(&conn->wait); - spin_lock_init(&conn->rx_lock); + INIT_WORK(&conn->wake_work, mktty_host_wake_work); + raw_spin_lock_init(&conn->rx_lock); conn->rx_head = 0; conn->rx_tail = 0; - spin_lock(&mktty_host_conns_lock); + raw_spin_lock_irqsave(&mktty_host_conns_lock, flags); list_add(&conn->list, &mktty_host_conns); - spin_unlock(&mktty_host_conns_lock); + raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); filp->private_data = conn; return 0; @@ -106,11 +128,14 @@ static int mktty_host_open(struct inode *inode, struct file *filp) static int mktty_host_release(struct inode *inode, struct file *filp) { struct mktty_host_conn *conn = filp->private_data; + unsigned long flags; - spin_lock(&mktty_host_conns_lock); + raw_spin_lock_irqsave(&mktty_host_conns_lock, flags); list_del(&conn->list); - spin_unlock(&mktty_host_conns_lock); + raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); + /* Unlink excludes new enqueues; do not wait while holding either raw lock. */ + flush_work(&conn->wake_work); kfree(conn->rx_buf); kfree(conn); return 0; @@ -124,26 +149,26 @@ static ssize_t mktty_host_read(struct file *filp, char __user *buf, unsigned long flags; int ret; - if (conn->instance_id < 0) + if (READ_ONCE(conn->instance_id) < 0) return -ENOTCONN; while (copied == 0) { - spin_lock_irqsave(&conn->rx_lock, flags); + raw_spin_lock_irqsave(&conn->rx_lock, flags); while (copied < count && conn->rx_head != conn->rx_tail) { char c = conn->rx_buf[conn->rx_tail]; conn->rx_tail = (conn->rx_tail + 1) % MKTTY_RX_BUF_SIZE; - spin_unlock_irqrestore(&conn->rx_lock, flags); + raw_spin_unlock_irqrestore(&conn->rx_lock, flags); if (put_user(c, buf + copied)) { return copied > 0 ? copied : -EFAULT; } copied++; - spin_lock_irqsave(&conn->rx_lock, flags); + raw_spin_lock_irqsave(&conn->rx_lock, flags); } - spin_unlock_irqrestore(&conn->rx_lock, flags); + raw_spin_unlock_irqrestore(&conn->rx_lock, flags); if (copied > 0) break; @@ -169,10 +194,12 @@ static ssize_t mktty_host_write(struct file *filp, const char __user *buf, char numbuf[16]; size_t sent = 0; size_t chunk; + unsigned long flags; + int instance_id = READ_ONCE(conn->instance_id); int ret; /* First write selects instance */ - if (conn->instance_id < 0) { + if (instance_id < 0) { size_t len = min(count, sizeof(numbuf) - 1); int id; @@ -191,7 +218,14 @@ static ssize_t mktty_host_write(struct file *filp, const char __user *buf, return -ENODEV; mk_instance_put(instance); - conn->instance_id = id; + /* Publish once under the callback's lookup/lifetime lock. */ + raw_spin_lock_irqsave(&mktty_host_conns_lock, flags); + if (conn->instance_id >= 0) { + raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); + return -EISCONN; + } + WRITE_ONCE(conn->instance_id, id); + raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); pr_info("mktty: fd connected to instance %d\n", id); return count; } @@ -214,7 +248,7 @@ static ssize_t mktty_host_write(struct file *filp, const char __user *buf, msg->len = chunk; msg->reserved = 0; - ret = multikernel_send_ipi_data(conn->instance_id, msg, + ret = multikernel_send_ipi_data(instance_id, msg, sizeof(*msg) - MKTTY_MAX_DATA + chunk, MKTTY_IPI_TYPE); if (ret < 0) { @@ -235,15 +269,15 @@ static __poll_t mktty_host_poll(struct file *filp, poll_table *wait) __poll_t mask = 0; unsigned long flags; - if (conn->instance_id < 0) + if (READ_ONCE(conn->instance_id) < 0) return EPOLLOUT | EPOLLWRNORM; /* can write instance ID */ poll_wait(filp, &conn->wait, wait); - spin_lock_irqsave(&conn->rx_lock, flags); + raw_spin_lock_irqsave(&conn->rx_lock, flags); if (conn->rx_head != conn->rx_tail) mask |= EPOLLIN | EPOLLRDNORM; - spin_unlock_irqrestore(&conn->rx_lock, flags); + raw_spin_unlock_irqrestore(&conn->rx_lock, flags); mask |= EPOLLOUT | EPOLLWRNORM; return mask; @@ -275,13 +309,14 @@ static void mktty_host_ipi_handler(struct mk_ipi_data *data, void *ctx) return; if (msg->type != MKTTY_MSG_OUTPUT) return; - if (msg->len > MKTTY_MAX_DATA) + if (msg->len > MKTTY_MAX_DATA || + msg->len > data->data_size - (sizeof(*msg) - MKTTY_MAX_DATA)) return; - spin_lock_irqsave(&mktty_host_conns_lock, flags); + raw_spin_lock_irqsave(&mktty_host_conns_lock, flags); conn = mktty_find_conn(msg->console_id); if (conn) { - spin_lock(&conn->rx_lock); + raw_spin_lock(&conn->rx_lock); for (i = 0; i < msg->len; i++) { space = (conn->rx_tail - conn->rx_head - 1 + MKTTY_RX_BUF_SIZE) % MKTTY_RX_BUF_SIZE; if (space == 0) @@ -289,10 +324,11 @@ static void mktty_host_ipi_handler(struct mk_ipi_data *data, void *ctx) conn->rx_buf[conn->rx_head] = msg->data[i]; conn->rx_head = (conn->rx_head + 1) % MKTTY_RX_BUF_SIZE; } - spin_unlock(&conn->rx_lock); - wake_up_interruptible(&conn->wait); + raw_spin_unlock(&conn->rx_lock); + /* Queue before dropping the lookup/lifetime lock, never after it. */ + schedule_work(&conn->wake_work); } - spin_unlock_irqrestore(&mktty_host_conns_lock, flags); + raw_spin_unlock_irqrestore(&mktty_host_conns_lock, flags); } static int mktty_host_init(void) @@ -307,8 +343,10 @@ static int mktty_host_init(void) mktty_host_handler = multikernel_register_handler(mktty_host_ipi_handler, NULL, MKTTY_IPI_TYPE); - if (!mktty_host_handler) - pr_warn("mktty: failed to register IPI handler\n"); + if (!mktty_host_handler) { + misc_deregister(&mktty_host_miscdev); + return -ENOMEM; + } pr_info("mktty: host ready - open /dev/mktty, write instance ID, then read/write\n"); return 0; @@ -329,23 +367,103 @@ static void mktty_host_cleanup(void) struct mktty_spawn_state { struct tty_port port; - spinlock_t lock; + raw_spinlock_t lock; + bool rx_enabled; + bool stopping; + u64 rx_dropped; int instance_id; }; static struct tty_driver *mktty_spawn_driver; static struct mktty_spawn_state mktty_spawn; +static void mktty_spawn_receive_work(struct work_struct *work); +static DECLARE_WORK(mktty_spawn_rx_work, mktty_spawn_receive_work); +static DEFINE_KFIFO(mktty_spawn_rx_fifo, u8, MKTTY_RX_BUF_SIZE); static struct mk_ipi_handler *mktty_spawn_handler; static struct console mktty_spawn_console; static bool mktty_console_registered; static int mktty_spawn_activate(struct tty_port *port, struct tty_struct *tty) { - return 0; + unsigned long flags; + int ret = 0; + + /* Port mutex serializes activation with the preceding shutdown/drain. */ + raw_spin_lock_irqsave(&mktty_spawn.lock, flags); + if (mktty_spawn.stopping) { + ret = -ENODEV; + } else { + kfifo_reset(&mktty_spawn_rx_fifo); + mktty_spawn.rx_enabled = true; + } + raw_spin_unlock_irqrestore(&mktty_spawn.lock, flags); + return ret; +} + +static void mktty_spawn_report_drops(void) +{ + unsigned long flags; + u64 dropped; + + raw_spin_lock_irqsave(&mktty_spawn.lock, flags); + dropped = mktty_spawn.rx_dropped; + mktty_spawn.rx_dropped = 0; + raw_spin_unlock_irqrestore(&mktty_spawn.lock, flags); + if (dropped) + pr_warn_ratelimited("mktty: dropped %llu input bytes\n", + (unsigned long long)dropped); +} + +static void mktty_spawn_receive_work(struct work_struct *work) +{ + struct tty_struct *tty = tty_port_tty_get(&mktty_spawn.port); + unsigned int budget = MKTTY_RX_BUF_SIZE, count, inserted; + unsigned long flags; + u8 buf[512]; + + while (budget) { + raw_spin_lock_irqsave(&mktty_spawn.lock, flags); + count = mktty_spawn.rx_enabled ? + kfifo_out(&mktty_spawn_rx_fifo, buf, + min_t(unsigned int, sizeof(buf), budget)) : 0; + raw_spin_unlock_irqrestore(&mktty_spawn.lock, flags); + if (!count) + break; + inserted = tty ? tty_insert_flip_string(&mktty_spawn.port, buf, count) : 0; + if (inserted) + tty_flip_buffer_push(&mktty_spawn.port); + if (inserted < count) { + raw_spin_lock_irqsave(&mktty_spawn.lock, flags); + mktty_spawn.rx_dropped += count - inserted; + raw_spin_unlock_irqrestore(&mktty_spawn.lock, flags); + } + budget -= count; + } + if (tty) + tty_kref_put(tty); + + /* Serialize requeue with shutdown's admission close before cancellation. */ + raw_spin_lock_irqsave(&mktty_spawn.lock, flags); + if (mktty_spawn.rx_enabled && !kfifo_is_empty(&mktty_spawn_rx_fifo)) + schedule_work(&mktty_spawn_rx_work); + raw_spin_unlock_irqrestore(&mktty_spawn.lock, flags); + mktty_spawn_report_drops(); } static void mktty_spawn_shutdown(struct tty_port *port) { + unsigned long flags; + + raw_spin_lock_irqsave(&mktty_spawn.lock, flags); + mktty_spawn.rx_enabled = false; + raw_spin_unlock_irqrestore(&mktty_spawn.lock, flags); + /* The worker never takes the port mutex held by tty_port_shutdown(). */ + cancel_work_sync(&mktty_spawn_rx_work); + raw_spin_lock_irqsave(&mktty_spawn.lock, flags); + mktty_spawn.rx_dropped += kfifo_len(&mktty_spawn_rx_fifo); + kfifo_reset(&mktty_spawn_rx_fifo); + raw_spin_unlock_irqrestore(&mktty_spawn.lock, flags); + mktty_spawn_report_drops(); } static const struct tty_port_operations mktty_spawn_port_ops = { @@ -368,8 +486,11 @@ static void mktty_spawn_close(struct tty_struct *tty, struct file *filp) static void mktty_spawn_hangup(struct tty_struct *tty) { - if (tty->index == 0) + if (tty->index == 0) { tty_port_hangup(&mktty_spawn.port); + /* Core's first ldisc flush precedes driver hangup; RX is drained now. */ + tty_ldisc_flush(tty); + } } static ssize_t mktty_spawn_write(struct tty_struct *tty, const u8 *buf, @@ -424,25 +545,29 @@ static const struct tty_operations mktty_spawn_ops = { static void mktty_spawn_ipi_handler(struct mk_ipi_data *data, void *ctx) { struct mktty_message *msg = (struct mktty_message *)data->buffer; - struct tty_struct *tty; + unsigned long flags; + unsigned int copied; if (data->data_size < sizeof(*msg) - MKTTY_MAX_DATA) return; if (msg->type != MKTTY_MSG_INPUT) return; - if (msg->len > MKTTY_MAX_DATA) + if (msg->len > MKTTY_MAX_DATA || + msg->len > data->data_size - (sizeof(*msg) - MKTTY_MAX_DATA)) return; - tty = tty_port_tty_get(&mktty_spawn.port); - if (tty) { - tty_insert_flip_string(&mktty_spawn.port, msg->data, msg->len); - tty_flip_buffer_push(&mktty_spawn.port); - tty_kref_put(tty); + /* The dispatcher releases the ring slot immediately after we return. */ + raw_spin_lock_irqsave(&mktty_spawn.lock, flags); + if (mktty_spawn.rx_enabled) { + copied = kfifo_in(&mktty_spawn_rx_fifo, msg->data, msg->len); + mktty_spawn.rx_dropped += msg->len - copied; + schedule_work(&mktty_spawn_rx_work); } + raw_spin_unlock_irqrestore(&mktty_spawn.lock, flags); } static struct mktty_message mktty_console_msg; -static DEFINE_SPINLOCK(mktty_console_lock); +static DEFINE_RAW_SPINLOCK(mktty_console_lock); static void mktty_spawn_console_write(struct console *con, const char *s, unsigned int count) @@ -450,7 +575,7 @@ static void mktty_spawn_console_write(struct console *con, const char *s, unsigned long flags; size_t chunk; - spin_lock_irqsave(&mktty_console_lock, flags); + raw_spin_lock_irqsave(&mktty_console_lock, flags); while (count > 0) { chunk = min_t(size_t, count, MKTTY_MAX_DATA); mktty_console_msg.type = MKTTY_MSG_OUTPUT; @@ -465,7 +590,7 @@ static void mktty_spawn_console_write(struct console *con, const char *s, s += chunk; count -= chunk; } - spin_unlock_irqrestore(&mktty_console_lock, flags); + raw_spin_unlock_irqrestore(&mktty_console_lock, flags); } static struct tty_driver *mktty_console_device(struct console *c, int *index) @@ -515,7 +640,7 @@ static int mktty_spawn_init(void) return -ENODEV; mktty_spawn.instance_id = mk_self->id; - spin_lock_init(&mktty_spawn.lock); + raw_spin_lock_init(&mktty_spawn.lock); pr_info("mktty: initializing spawn (instance %d)\n", mktty_spawn.instance_id); @@ -546,8 +671,13 @@ static int mktty_spawn_init(void) mktty_spawn_handler = multikernel_register_handler(mktty_spawn_ipi_handler, NULL, MKTTY_IPI_TYPE); - if (!mktty_spawn_handler) - pr_warn("mktty: failed to register IPI handler\n"); + if (!mktty_spawn_handler) { + mktty_spawn_driver = NULL; + tty_unregister_driver(driver); + tty_driver_kref_put(driver); + tty_port_destroy(&mktty_spawn.port); + return -ENOMEM; + } if (!mktty_console_registered) { register_console(&mktty_spawn_console); @@ -560,8 +690,15 @@ static int mktty_spawn_init(void) static void mktty_spawn_cleanup(void) { + unsigned long flags; + + raw_spin_lock_irqsave(&mktty_spawn.lock, flags); + mktty_spawn.stopping = true; + mktty_spawn.rx_enabled = false; + raw_spin_unlock_irqrestore(&mktty_spawn.lock, flags); if (mktty_spawn_handler) multikernel_unregister_handler(mktty_spawn_handler); + mktty_spawn_shutdown(&mktty_spawn.port); if (mktty_console_registered) { unregister_console(&mktty_spawn_console); diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 9b6b0d87fdb056..96dd402c7303b6 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -38,6 +38,10 @@ extern void trap_init(void); extern int register_cpu(struct cpu *cpu, int num); extern struct device *get_cpu_device(unsigned cpu); extern bool cpu_is_hotpluggable(unsigned cpu); +#ifdef CONFIG_HOTPLUG_CPU +int reclaim_cpu(unsigned int cpu, bool *attempted); +void cpu_device_update_node(struct device *dev, int from_nid); +#endif extern bool arch_match_cpu_phys_id(int cpu, u64 phys_id); extern bool arch_find_n_match_cpu_physical_id(struct device_node *cpun, int cpu, unsigned int *thread); diff --git a/include/linux/multikernel.h b/include/linux/multikernel.h index 76e51d19b91381..9da1ce79075ebb 100644 --- a/include/linux/multikernel.h +++ b/include/linux/multikernel.h @@ -16,6 +16,8 @@ #include #include #include +#include +#include #ifdef CONFIG_MULTIKERNEL #include @@ -140,7 +142,7 @@ struct mk_ipi_handler { mk_ipi_callback_t callback; void *context; unsigned int ipi_type; /* IPI type this handler is registered for */ - struct mk_ipi_handler *next; + struct mk_ipi_handler __rcu *next; }; /** @@ -156,6 +158,11 @@ struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback, /** * multikernel_unregister_handler - Unregister a multikernel IPI callback * @handler: Handler pointer returned from multikernel_register_handler + * + * Sleeps until entered callbacks have returned. Call from process context, + * never from an IPI callback, and do not hold locks needed by callbacks. + * After return, the caller may release the callback context. The caller must + * serialize unregister calls for a given handler. */ void multikernel_unregister_handler(struct mk_ipi_handler *handler); @@ -175,8 +182,8 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns void generic_multikernel_interrupt(void); -/* Drain the message ring where the doorbell interrupt cannot be taken */ -void mk_ipi_poll(void); +/* Drain one inbox snapshot; -EBUSY for an unpublished slot or callback reentry. */ +int mk_ipi_poll(void); bool mk_ipi_in_drain(void); struct mk_instance; @@ -184,7 +191,6 @@ int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, size_t data_size, unsigned long type); /* Discard everything queued in this kernel's ring (instance re-spawn) */ -void mk_ipi_ring_drop_pending(void); /* * Multikernel Messaging System @@ -350,7 +356,7 @@ struct mk_shutdown_payload { /** * Message handler callback type */ -typedef void (*mk_msg_handler_t)(u32 msg_type, u32 subtype, +typedef void (*mk_msg_handler_t)(u32 msg_type, u32 subtype, u64 msg_id, void *payload, u32 payload_len, void *ctx); /* Opaque type for pending message tracking */ @@ -372,6 +378,8 @@ struct mk_pending_msg; */ int mk_send_message(int instance_id, u32 msg_type, u32 subtype, void *payload, u32 payload_len); +int mk_send_message_id(int instance_id, u32 msg_type, u32 subtype, u64 msg_id, + void *payload, u32 payload_len); /* * The same for a caller that holds a reference to @instance. Looking an @@ -413,9 +421,16 @@ int mk_register_msg_handler(u32 msg_type, mk_msg_handler_t handler, void *ctx); */ int mk_unregister_msg_handler(u32 msg_type, mk_msg_handler_t handler); -/* Pending message tracking for request-response pattern */ +/* + * Pending requests: add returns ERR_PTR on failure. The requester owns the + * pointer until exactly one wait or cancel; neither may run concurrently on + * the same pointer. IDs are unique only within this kernel's lifetime. + */ struct mk_pending_msg *mk_msg_pending_add(u32 msg_type, u32 operation, u64 resource_id); -void mk_msg_pending_complete(u32 msg_type, u32 operation, u64 resource_id, int result); +u64 mk_msg_pending_id(const struct mk_pending_msg *pending); +void mk_msg_pending_cancel(struct mk_pending_msg *pending); +void mk_msg_pending_complete(u32 msg_type, u32 operation, u64 resource_id, + u64 msg_id, int result); int mk_msg_pending_wait(struct mk_pending_msg *pending, unsigned long timeout_ms); /** @@ -436,7 +451,8 @@ static inline int mk_send_irq_forward(int instance_id, u32 irq_number, &payload, sizeof(payload)); } -/* CPU resource management - these wait for operation to complete */ +/* Controller resource senders below require mk_overlay_mutex across the wait. */ +/* CPU resource management */ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags); int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id); @@ -478,6 +494,13 @@ struct mk_pool_chunk { */ struct mk_pool { struct mk_cpu_set *cpus; /* Parked CPUs free to assign */ + /* + * Stabilizes cpus storage and membership against CPUHP wake readers. + * Writers are serialized by mk_overlay_mutex, but CPUHP does not take + * it. Nests inside park_lock and mk_instance_mutex; never hold across + * CPU hotplug, park/repark calls or ACK waits (wake takes it again). + */ + struct mutex cpus_lock; struct list_head chunks; /* Donated memory (struct mk_pool_chunk) */ struct gen_pool *node_pools[MAX_NUMNODES]; struct mutex mem_lock; /* Protects chunks and node_pools */ @@ -500,10 +523,11 @@ extern struct mk_pool *mk_pool; int mk_pool_mem_grow(size_t size, int node, phys_addr_t *out_base); int mk_pool_mem_shrink(phys_addr_t start, size_t size); +/* Caller owns the range. Map failure adds no mappings; unmap failure retains them. */ int mk_pool_map(phys_addr_t start, size_t size); -void mk_pool_unmap(phys_addr_t start, size_t size); +int mk_pool_unmap(phys_addr_t start, size_t size); int mk_arch_pool_map(phys_addr_t start, size_t size); -void mk_arch_pool_unmap(phys_addr_t start, size_t size); +int mk_arch_pool_unmap(phys_addr_t start, size_t size); extern phys_addr_t multikernel_alloc(size_t size, int node); extern void multikernel_free(phys_addr_t addr, size_t size); struct resource *mk_pool_chunk_resource(phys_addr_t addr); @@ -709,6 +733,9 @@ struct mk_instance { int id; /* Kernel-assigned instance ID */ char *name; /* User-provided instance name */ enum mk_instance_state state; /* Current state */ + /* Protected by mk_overlay_mutex; a failed remote rollback pins grants. */ + bool resource_uncertain; + raw_spinlock_t ipi_lock; /* Ring writers versus reset/publication */ /* Resource management - list of reserved memory regions */ struct list_head memory_regions; /* List of struct mk_memory_region */ @@ -775,9 +802,10 @@ struct mk_instance { size_t ctrl_used; /* - * CPUs parked inside this instance's memory (its arch park area) - * rather than in the host pool. They must be confirmed parked before - * the image is rewritten and they pin the pool until they leave. + * CPUs that may use this instance's memory (its kernel or arch park + * area), including handoffs whose wake acknowledgment timed out. + * They must be confirmed parked before the image is rewritten and + * they pin the pool until they leave. * Membership is per CPU because it changes one CPU at a time: * hotplug moves CPUs in and out while the instance runs, and a * stopped instance can gain CPUs that still sit in the host pool. @@ -806,6 +834,9 @@ struct mk_instance { /* Reference counting */ struct kref refcount; /* Reference count for cleanup */ + struct work_struct release_work; /* Last reference may be put from an IPI */ + struct work_struct halted_work; /* Holds a reference while settling a notice */ + atomic_t halted_pending; /* Excludes restart until the notice retires */ }; /** @@ -922,14 +953,28 @@ struct mk_instance *mk_instance_find_by_name(const char *name); */ struct mk_instance *mk_instance_get(struct mk_instance *instance); int mk_instance_alloc_boot_pages(struct mk_instance *instance); +#ifdef CONFIG_MULTIKERNEL +void mk_instance_ipi_publish(struct mk_instance *instance); +void mk_instance_ipi_deactivate(struct mk_instance *instance); +void mk_instance_ipi_reset(struct mk_instance *instance); +void mk_instance_ipi_retarget(struct mk_instance *instance); +#else +static inline void mk_instance_ipi_publish(struct mk_instance *instance) { } +static inline void mk_instance_ipi_deactivate(struct mk_instance *instance) { } +static inline void mk_instance_ipi_reset(struct mk_instance *instance) { } +static inline void mk_instance_ipi_retarget(struct mk_instance *instance) { } +#endif /** * mk_halt_to_pool() - Halt this spawn kernel, returning its CPUs to the pool + * @notify: Caller has excluded new resource operations through final parking * - * Notifies the host and parks every CPU in the pool wait loop, keeping - * the instance re-spawnable. Never returns. + * Parks every CPU in the pool wait loop. Only a qualified caller may notify + * the host that the instance can be considered for reuse after stop proof. + * Unqualified exits must pass false and leave the host retaining resources. + * Never returns. */ -void __noreturn mk_halt_to_pool(void); +void __noreturn mk_halt_to_pool(bool notify); /** * mk_instance_reserve_resources() - Reserve CPU and memory resources for instance @@ -950,8 +995,9 @@ int mk_instance_reserve_resources(struct mk_instance *instance, * * Returns all reserved memory regions back to the multikernel pool * and removes them from the resource hierarchy. + * Return: 0 on success, or an error with memory and metadata retained. */ -void mk_instance_free_memory(struct mk_instance *instance); +int mk_instance_free_memory(struct mk_instance *instance); int mk_instance_transfer_cpus(struct mk_instance *instance, @@ -1003,6 +1049,13 @@ struct pci_bus; struct pci_dev; #ifdef CONFIG_MULTIKERNEL +/* + * Serializes controller resource changes with MK image load, exec and cleanup. + * Process context only. Lock order: kexec_lock -> mk_overlay_mutex -> + * mk_instance_mutex. Do not wait for remote work under mk_instance_mutex. + */ +extern struct mutex mk_overlay_mutex; + bool multikernel_allow_emergency_restart(void); int multikernel_halt_by_id(int mk_id); int multikernel_force_halt_by_id(int mk_id); diff --git a/kernel/cpu.c b/kernel/cpu.c index 08b4ae6ec59399..ffc698311f9090 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -1536,19 +1536,52 @@ EXPORT_SYMBOL_GPL(remove_cpu); * * Like remove_cpu(), but the takedown ends in CPUHP_DEPARTED instead * of CPUHP_OFFLINE: the CPU is handed to another kernel and is not - * expected back. It stays departed until a later add_cpu() brings it - * up; a bringup that fails rolls it back to departed. + * expected back. On success it is also absent, so native hotplug cannot + * bring it back before the owning kernel explicitly restores presence. + * A bringup that fails rolls it back to departed. * * Return: %0 on success or a negative errno code */ int depart_cpu(unsigned int cpu) { + struct cpuhp_cpu_state *st; int ret; lock_device_hotplug(); - ret = cpu_down(cpu, CPUHP_DEPARTED); - if (!ret) - cpuhp_offline_cpu_device(cpu); + cpu_maps_update_begin(); + + ret = -EINVAL; + if (!cpu || cpu >= nr_cpu_ids || !cpu_possible(cpu) || + !cpu_present(cpu) || !cpu_online(cpu)) + goto out; + if (!get_cpu_device(cpu)) { + ret = -ENODEV; + goto out; + } + st = per_cpu_ptr(&cpuhp_state, cpu); + if (st->state != CPUHP_ONLINE) + goto out; + + /* + * Keep native CPU-map serialization across both teardown and presence + * withdrawal. The device-hotplug lock alone does not exclude in-kernel + * CPU bringup, including SMT changes. _cpu_down() takes the CPU write + * lock itself; do not hold it across the teardown callbacks. + */ + ret = cpu_down_maps_locked(cpu, CPUHP_DEPARTED); + if (ret) + goto out; + if (WARN_ON_ONCE(st->state != CPUHP_DEPARTED || cpu_online(cpu))) { + ret = -EIO; + goto out; + } + + cpus_write_lock(); + set_cpu_present(cpu, false); + cpus_write_unlock(); + cpuhp_offline_cpu_device(cpu); +out: + cpu_maps_update_done(); unlock_device_hotplug(); return ret; @@ -1789,6 +1822,95 @@ int add_cpu(unsigned int cpu) } EXPORT_SYMBOL_GPL(add_cpu); +#ifdef CONFIG_HOTPLUG_CPU +static void cpuhp_online_cpu_device(unsigned int cpu); + +/** + * reclaim_cpu - Bring a CPU into this kernel with serialized presence changes + * @cpu: Logical CPU already enumerated and registered in this kernel + * @attempted: Set if native bringup was entered, including failed attempts + * + * The caller must reserve ownership before calling. A failed attempt does not + * prove physical quiescence: retain that ownership even if CPUHP rolled back. + * Presence is restored on failure only when the CPU is offline; an incomplete + * rollback must not make an online CPU absent. Ordinary add_cpu() is unchanged. + * + * Return: %0 on success or a negative errno code. No automatic retry. + */ +int reclaim_cpu(unsigned int cpu, bool *attempted) +{ + struct cpuhp_cpu_state *st; + struct device *dev; + bool was_present; + int from_nid, ret; + + *attempted = false; + if (cpu >= nr_cpu_ids || !cpu_possible(cpu)) + return -EINVAL; + + lock_device_hotplug(); + dev = get_cpu_device(cpu); + if (!dev) { + ret = -ENODEV; + goto out_hotplug; + } + device_lock(dev); + from_nid = cpu_to_node(cpu); + if (from_nid == NUMA_NO_NODE) { + ret = -ENODEV; + goto out_device; + } + ret = try_online_node(from_nid); + if (ret) + goto out_node; + + cpu_maps_update_begin(); + st = per_cpu_ptr(&cpuhp_state, cpu); + was_present = cpu_present(cpu); + ret = -EINVAL; + if (cpu_online(cpu)) { + if (was_present && st->state == CPUHP_ONLINE) + ret = 0; + goto out_maps; + } + if (st->state != CPUHP_OFFLINE && st->state != CPUHP_DEPARTED) + goto out_maps; + ret = -EBUSY; + if (cpu_hotplug_disabled) + goto out_maps; + ret = -EPERM; + if (!cpu_bootable(cpu)) + goto out_maps; + + if (!was_present) { + cpus_write_lock(); + set_cpu_present(cpu, true); + cpus_write_unlock(); + } + *attempted = true; + ret = _cpu_up(cpu, 0, CPUHP_ONLINE); + if (!ret && WARN_ON_ONCE(!cpu_online(cpu) || st->state != CPUHP_ONLINE)) + ret = -EIO; + if (ret && !was_present && !cpu_online(cpu)) { + cpus_write_lock(); + set_cpu_present(cpu, false); + cpus_write_unlock(); + } +out_maps: + if (!ret && dev->offline) + cpuhp_online_cpu_device(cpu); + cpu_maps_update_done(); +out_node: + /* As in cpu_subsys_online(), update NUMA links outside the maps lock. */ + cpu_device_update_node(dev, from_nid); +out_device: + device_unlock(dev); +out_hotplug: + unlock_device_hotplug(); + return ret; +} +#endif /* CONFIG_HOTPLUG_CPU */ + /** * bringup_hibernate_cpu - Bring up the CPU that we hibernated on * @sleep_cpu: The cpu we hibernated on and should be brought up. @@ -2106,6 +2228,7 @@ static struct cpuhp_step cpuhp_hp_states[] = { .name = "departed", .startup.single = NULL, .teardown.single = NULL, + .cant_stop = true, }, #ifdef CONFIG_SMP [CPUHP_CREATE_THREADS]= { @@ -2692,8 +2815,7 @@ void __cpuhp_remove_state(enum cpuhp_state state, bool invoke) } EXPORT_SYMBOL(__cpuhp_remove_state); -#ifdef CONFIG_HOTPLUG_SMT - +#if defined(CONFIG_HOTPLUG_SMT) || defined(CONFIG_HOTPLUG_CPU) static void cpuhp_online_cpu_device(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); @@ -2702,7 +2824,9 @@ static void cpuhp_online_cpu_device(unsigned int cpu) /* Tell user space about the state change */ kobject_uevent(&dev->kobj, KOBJ_ONLINE); } +#endif +#ifdef CONFIG_HOTPLUG_SMT int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval) { int cpu, ret = 0; diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index b7a6fc6eb6ee0d..b0b44c73930ce5 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -596,11 +596,40 @@ static void kimage_free_cma(struct kimage *image) * its load-time buffers and its reference on the instance. Also unwinds a * load that failed before the image was published. */ +int kimage_can_free_multikernel(struct kimage *image) +{ +#ifdef CONFIG_MULTIKERNEL + struct mk_instance *instance = image->mk_instance; + + lockdep_assert_held(&mk_overlay_mutex); + /* An unpublished loader candidate has never released a CPU into its Image. */ + if (!instance || instance->kimage != image) + return 0; + if (instance->state == MK_STATE_ACTIVE) + return -EBUSY; + return mk_instance_confirm_parked(instance); +#else + return -EOPNOTSUPP; +#endif +} + void kimage_free_multikernel(struct kimage *image) { struct mk_instance *instance = image->mk_instance; unsigned long i; +#ifdef CONFIG_MULTIKERNEL + lockdep_assert_held(&mk_overlay_mutex); +#endif + /* Published Images reach here only after kimage_free() validated stop. */ + if (!instance) + return; + if (instance->kimage == image) { + /* Retire PCI access before returning memory it could still reach. */ + mk_instance_set_state(instance, MK_STATE_READY); + mk_instance_ipi_deactivate(instance); + } + for (i = 0; i < image->nr_segments; i++) mk_kimage_free(image, phys_to_virt(image->segment[i].mem), image->segment[i].memsz); @@ -610,21 +639,26 @@ void kimage_free_multikernel(struct kimage *image) if (instance->kimage == image) { /* No kernel left to read the ring */ - instance->ipi_data = NULL; instance->kimage = NULL; - mk_instance_set_state(instance, MK_STATE_READY); } mk_instance_put(instance); image->mk_instance = NULL; } -void kimage_free(struct kimage *image) +int kimage_free(struct kimage *image) { kimage_entry_t *ptr, entry; kimage_entry_t ind = 0; + int ret; if (!image) - return; + return 0; + /* All free paths, including future bulk callers, must fail before unlink. */ + if (image->type == KEXEC_TYPE_MULTIKERNEL) { + ret = kimage_can_free_multikernel(image); + if (ret) + return ret; + } /* Remove from linked list and update compatibility pointers */ kimage_remove_from_list(image); @@ -681,6 +715,7 @@ void kimage_free(struct kimage *image) } kfree(image); + return 0; } static kimage_entry_t *kimage_dst_used(struct kimage *image, @@ -1645,6 +1680,9 @@ struct kimage *kimage_find_by_id(int mk_id) struct mk_instance *instance; struct kimage *image = NULL; + /* The caller keeps the image alive through its complete operation. */ + lockdep_assert_held(&mk_overlay_mutex); + if (mk_id <= 0) return NULL; @@ -1668,6 +1706,8 @@ int multikernel_kexec_by_id(int mk_id) if (!kexec_trylock()) return -EBUSY; + mutex_lock(&mk_overlay_mutex); + mk_image = kimage_find_by_id(mk_id); if (!mk_image) { pr_err("No multikernel image found with ID %d\n", mk_id); @@ -1676,6 +1716,12 @@ int multikernel_kexec_by_id(int mk_id) } instance = mk_image->mk_instance; + /* Execution must not bypass halt settlement, even if CPUs look parked. */ + if (instance->state == MK_STATE_ACTIVE) { + rc = -EBUSY; + goto unlock; + } + if (!mk_cpu_set_empty(instance->cpus)) { mk_phys_cpu_t phys_cpu = mk_cpu_set_first(instance->cpus); @@ -1730,14 +1776,16 @@ int multikernel_kexec_by_id(int mk_id) } rc = mk_manifest_finalize(mk_image); - if (rc) - pr_warn("Manifest finalization failed: %d\n", rc); - else - pr_info("Manifest finalized for multikernel instance\n"); + if (rc) { + pr_err("Manifest finalization failed: %d; refusing start\n", rc); + goto unlock; + } + pr_info("Manifest finalized for multikernel instance\n"); rc = mk_instance_start(instance, mk_image, cpu); unlock: + mutex_unlock(&mk_overlay_mutex); kexec_unlock(); return rc; } diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index c8172a8a3e8641..218db2d36d56f2 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -454,6 +454,7 @@ kimage_file_alloc_init(struct kimage **rimage, int kernel_fd, out_free_image: if (image->type == KEXEC_TYPE_MULTIKERNEL) kimage_free_multikernel(image); + kimage_file_free_cmdline(image); kfree(image); return ret; } @@ -466,7 +467,7 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, KEXEC_TYPE_CRASH : KEXEC_TYPE_DEFAULT; struct kimage **dest_image, *image; struct mk_instance *instance = NULL; - int ret = 0, i; + int ret = 0, i, free_ret; /* We only trust the superuser with rebooting the system. */ if (!kexec_load_permitted(image_type)) @@ -485,6 +486,11 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, if (!kexec_trylock()) return -EBUSY; +#ifdef CONFIG_MULTIKERNEL + if (flags & KEXEC_MULTIKERNEL) + mutex_lock(&mk_overlay_mutex); +#endif + if (flags & KEXEC_MULTIKERNEL) { int mk_id = KEXEC_GET_MK_ID(flags); @@ -505,7 +511,6 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, } pr_info("Unloading kernel from multikernel instance %d\n", mk_id); - kimage_remove_from_list(mk_image); image = mk_image; ret = 0; goto out; @@ -517,7 +522,8 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, goto out; } - if (instance->kimage) { + if (instance == mk_self || instance->state != MK_STATE_READY || + instance->kimage) { pr_err("Multikernel instance %d already has an associated kimage\n", mk_id); mk_instance_put(instance); ret = -EBUSY; @@ -614,7 +620,7 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, if (image && image->type == KEXEC_TYPE_MULTIKERNEL) { /* Publish only a complete image to exec, unload and IPI */ image->mk_instance->kimage = image; - image->mk_instance->ipi_data = phys_to_virt(image->mk_instance->ipi_phys); + mk_instance_ipi_publish(image->mk_instance); mk_instance_set_state(image->mk_instance, MK_STATE_LOADED); kimage_add_to_list(image); image = NULL; /* Don't free the new image */ @@ -651,8 +657,15 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, arch_kexec_protect_crashkres(); #endif + /* Failed-load and unload cleanup still use the instance pool. */ + free_ret = kimage_free(image); + if (!ret) + ret = free_ret; +#ifdef CONFIG_MULTIKERNEL + if (flags & KEXEC_MULTIKERNEL) + mutex_unlock(&mk_overlay_mutex); +#endif kexec_unlock(); - kimage_free(image); return ret; } diff --git a/kernel/kexec_internal.h b/kernel/kexec_internal.h index 79b31d9bdb9f3f..eb23fb0382f347 100644 --- a/kernel/kexec_internal.h +++ b/kernel/kexec_internal.h @@ -9,8 +9,9 @@ struct kexec_segment; struct kimage *do_kimage_alloc_init(void); int sanity_check_segment_list(struct kimage *image); void kimage_free_page_list(struct list_head *list); -void kimage_free(struct kimage *image); +int kimage_free(struct kimage *image); void kimage_free_multikernel(struct kimage *image); +int kimage_can_free_multikernel(struct kimage *image); int kimage_load_segment(struct kimage *image, int idx); void kimage_terminate(struct kimage *image); int kimage_is_destination_range(struct kimage *image, diff --git a/kernel/multikernel/Kconfig b/kernel/multikernel/Kconfig index 79658fbf5b4118..3653d0a1380cd4 100644 --- a/kernel/multikernel/Kconfig +++ b/kernel/multikernel/Kconfig @@ -10,13 +10,19 @@ config ARCH_SUPPORTS_MULTIKERNEL config ARCH_HAS_MK_HOST_PARK bool +# The architecture can forcibly stop a remote instance's CPUs (x86 NMI). +config ARCH_HAS_MK_FORCE_STOP + bool + config MULTIKERNEL bool "Multikernel support" depends on ARCH_SUPPORTS_MULTIKERNEL + depends on HOTPLUG_CPU depends on KEXEC_CORE depends on MEMORY_HOTPLUG depends on MEMORY_HOTREMOVE depends on CONTIG_ALLOC + depends on OF && OF_EARLY_FLATTREE select LIBFDT select OF_DYNAMIC if OF help diff --git a/kernel/multikernel/baseline.c b/kernel/multikernel/baseline.c index 7e7d73f76ff8ed..9ccbfb1a6e1887 100644 --- a/kernel/multikernel/baseline.c +++ b/kernel/multikernel/baseline.c @@ -148,15 +148,18 @@ static void mk_baseline_free_pci_list(struct list_head *pci_list) } } -static void mk_baseline_clear_resources(struct mk_instance *instance) +static int mk_baseline_clear_resources(struct mk_instance *instance) { struct mk_pci_device *pci_dev, *pci_tmp; struct mk_platform_device *plat_dev, *plat_tmp; + int ret; if (!instance) - return; + return 0; - mk_instance_free_memory(instance); + ret = mk_instance_free_memory(instance); + if (ret) + return ret; list_for_each_entry_safe(pci_dev, pci_tmp, &instance->pci_devices, list) { list_del(&pci_dev->list); kfree(pci_dev); @@ -170,6 +173,7 @@ static void mk_baseline_clear_resources(struct mk_instance *instance) } instance->platform_device_count = 0; instance->platform_devices_valid = false; + return 0; } static int mk_baseline_validate_cpus(const struct mk_cpu_set *requested) @@ -471,7 +475,9 @@ int mk_baseline_validate_and_initialize(const void *fdt, size_t fdt_size) return -ENOMEM; } - mk_baseline_clear_resources(mk_self); + ret = mk_baseline_clear_resources(mk_self); + if (ret) + goto out; ret = mk_baseline_parse_cpus(fdt, resources_node, requested); if (ret) { @@ -520,6 +526,7 @@ int mk_baseline_validate_and_initialize(const void *fdt, size_t fdt_size) INIT_LIST_HEAD(&pool->chunks); mutex_init(&pool->mem_lock); mutex_init(&pool->park_lock); + mutex_init(&pool->cpus_lock); mk_pool = pool; } diff --git a/kernel/multikernel/core.c b/kernel/multikernel/core.c index 7fffac2c1c25aa..20c8201a55ee48 100644 --- a/kernel/multikernel/core.c +++ b/kernel/multikernel/core.c @@ -14,17 +14,48 @@ #include #include #include +#include #include "internal.h" -static void mk_instance_return_all_cpus(struct mk_instance *instance) +int mk_instance_idr_add(struct mk_instance *instance, int start, int end) { - if (!instance || mk_cpu_set_empty(instance->cpus)) - return; + int id; - if (instance == mk_self || instance->id == 0) - return; + lockdep_assert_held(&mk_instance_mutex); + /* Reserve an empty slot; RCU readers must see the initialized ID. */ + id = idr_alloc(&mk_instance_idr, NULL, start, end, GFP_KERNEL); + if (id >= 0) { + instance->id = id; + idr_replace(&mk_instance_idr, instance, id); + } + return id; +} - mk_instance_return_cpus(instance, instance->cpus); +void mk_instance_idr_remove(int id) +{ + lockdep_assert_held(&mk_instance_mutex); + idr_remove(&mk_instance_idr, id); +} + +int mk_instance_prepare_destroy(struct mk_instance *instance) +{ + int ret; + + lockdep_assert_held(&mk_overlay_mutex); + ret = mk_instance_confirm_parked(instance); + if (ret) + return ret; + ret = mk_arch_release_instance(instance); + if (ret) + return ret; + + if (mk_cpu_set_empty(instance->cpus)) + return 0; + /* Bookkeeping only: no firmware/park waits under the registry mutex. */ + mutex_lock(&mk_instance_mutex); + ret = mk_instance_return_cpus(instance, instance->cpus); + mutex_unlock(&mk_instance_mutex); + return ret; } static void mk_instance_return_pci_devices(struct mk_instance *instance) @@ -211,17 +242,46 @@ static void mk_instance_free_boot_pages(struct mk_instance *instance) instance->manifest_phys = 0; } -static void mk_instance_release(struct kref *kref) +int mk_instance_retire_resources(struct mk_instance *instance) { - struct mk_instance *instance = container_of(kref, struct mk_instance, refcount); + lockdep_assert_held(&mk_overlay_mutex); + mk_instance_return_pci_devices(instance); + mk_instance_return_platform_devices(instance); + return mk_instance_free_memory(instance); +} + +static void mk_instance_release_work(struct work_struct *work) +{ + struct mk_instance *instance = container_of(work, struct mk_instance, release_work); + int ret; + + /* Drain lookups that saw this record but did not acquire a reference. */ + synchronize_rcu(); + mutex_lock(&mk_overlay_mutex); + if (instance->state != MK_STATE_READY || instance->kimage) { + pr_err("Instance %d final release refused: not reusable; retaining allocations\n", + instance->id); + mutex_unlock(&mk_overlay_mutex); + return; + } pr_info("Releasing multikernel instance %d (%s), returning resources to root\n", instance->id, instance->name); - mk_instance_return_all_cpus(instance); - mk_instance_return_pci_devices(instance); - mk_instance_return_platform_devices(instance); - mk_instance_free_memory(instance); + ret = mk_instance_prepare_destroy(instance); + if (ret) { + pr_err("Instance %d final release refused: %d; retaining allocations\n", + instance->id, ret); + mutex_unlock(&mk_overlay_mutex); + return; + } + ret = mk_instance_retire_resources(instance); + if (ret) { + pr_err("Instance %d final retirement failed: %d; retaining allocations\n", + instance->id, ret); + mutex_unlock(&mk_overlay_mutex); + return; + } mk_instance_free_boot_pages(instance); mk_instance_track_dump(instance, instance->state, MK_STATE_READY); @@ -229,6 +289,16 @@ static void mk_instance_release(struct kref *kref) mk_cpu_set_free(instance->cpus); kfree(instance->name); kfree(instance); + mutex_unlock(&mk_overlay_mutex); +} + +static void mk_instance_release(struct kref *kref) +{ + struct mk_instance *instance = container_of(kref, struct mk_instance, refcount); + + /* A ring sender can drop the last reference from hard-IPI context. */ + INIT_WORK(&instance->release_work, mk_instance_release_work); + schedule_work(&instance->release_work); } /** @@ -247,6 +317,8 @@ void mk_instance_put(struct mk_instance *instance) kref_put(&instance->refcount, mk_instance_release); } +static void mk_halted_work_fn(struct work_struct *work); + /** * mk_instance_alloc - Construct an unpublished instance record * @id: Instance ID to record (a later publish may reassign it) @@ -278,11 +350,14 @@ struct mk_instance *mk_instance_alloc(int id, const char *name) instance->state = MK_STATE_READY; instance->ipi_target = MK_PHYS_CPU_INVALID; + raw_spin_lock_init(&instance->ipi_lock); INIT_LIST_HEAD(&instance->memory_regions); INIT_LIST_HEAD(&instance->list); INIT_LIST_HEAD(&instance->pci_devices); INIT_LIST_HEAD(&instance->platform_devices); kref_init(&instance->refcount); + INIT_WORK(&instance->halted_work, mk_halted_work_fn); + atomic_set(&instance->halted_pending, 0); return instance; @@ -307,9 +382,11 @@ int mk_instance_publish(struct mk_instance *instance) { int ret; + /* Self inboxes must also be usable without walking their CPU set in IRQ. */ + if (instance->ipi_target == MK_PHYS_CPU_INVALID) + instance->ipi_target = mk_cpu_set_first(instance->cpus); mutex_lock(&mk_instance_mutex); - ret = idr_alloc(&mk_instance_idr, instance, instance->id, - instance->id + 1, GFP_KERNEL); + ret = mk_instance_idr_add(instance, instance->id, instance->id + 1); if (ret < 0) { mutex_unlock(&mk_instance_mutex); return ret; @@ -361,12 +438,12 @@ void mk_instance_set_state(struct mk_instance *instance, if (old_state == state) return; + /* FAILED retains devices; callers settle execution before READY/LOADED. */ + if ((old_state == MK_STATE_ACTIVE || old_state == MK_STATE_FAILED) && + (state == MK_STATE_READY || state == MK_STATE_LOADED)) + mk_pciback_unlend(instance); instance->state = state; mk_instance_track_dump(instance, old_state, state); - - /* Whatever stopped it, its functions are no longer its to reach */ - if (old_state == MK_STATE_ACTIVE) - mk_pciback_unlend(instance); pr_debug("Instance %d (%s) state: %s -> %s\n", instance->id, instance->name, mk_state_to_string(old_state), @@ -399,11 +476,11 @@ struct mk_instance *mk_instance_find(int mk_id) { struct mk_instance *instance; - mutex_lock(&mk_instance_mutex); + rcu_read_lock(); instance = idr_find(&mk_instance_idr, mk_id); - if (instance) - mk_instance_get(instance); - mutex_unlock(&mk_instance_mutex); + if (instance && !kref_get_unless_zero(&instance->refcount)) + instance = NULL; + rcu_read_unlock(); return instance; } @@ -415,9 +492,9 @@ struct mk_instance *mk_instance_find(int mk_id) * @cpu: Logical CPU it boots on * * Gives the instance what it reaches from its first instruction on, its - * rings and the config space of its PCI functions, then starts it. What - * it was given is taken back when it leaves MK_STATE_ACTIVE, see - * mk_instance_set_state(), or here if it never gets there. + * rings and the config space of its PCI functions, then starts it. A start + * error can leave CPUs executing, so FAILED retains its lent functions until + * the caller settles execution and publishes READY or LOADED. * * Returns 0 once the instance runs, or a negative error code. */ @@ -426,6 +503,11 @@ int mk_instance_start(struct mk_instance *instance, struct kimage *image, { int ret; + lockdep_assert_held(&mk_overlay_mutex); + /* A delayed halt notice must retire before this record runs a new Image. */ + if (instance->resource_uncertain || atomic_read(&instance->halted_pending)) + return -EBUSY; + /* * Start the instance with an empty ring. It outlives the kernel * that was using it, so a new instance would otherwise inherit that @@ -434,16 +516,13 @@ int mk_instance_start(struct mk_instance *instance, struct kimage *image, * filling this one". Anything left in there was addressed to a * kernel that is gone. */ - if (instance->ipi_data) - memset(instance->ipi_data, 0, sizeof(*instance->ipi_data)); + mk_instance_ipi_reset(instance); /* - * Same for the other direction: whatever the halted instance left - * queued for us is addressed from a kernel that no longer exists, - * and a slot it claimed but never published stalls our ring for - * good. + * Do not reset the host receive ring: it is shared with live peers + * and its IRQ consumer. Stopped instance CPUs do not quiesce either. + * An abandoned unpublished slot needs a separate ownership protocol. */ - mk_ipi_ring_drop_pending(); ret = mk_pciback_lend(instance); if (ret) { @@ -452,13 +531,14 @@ int mk_instance_start(struct mk_instance *instance, struct kimage *image, return ret; } + /* A failed start may already have released CPUs into this Image. */ + mk_instance_set_state(instance, MK_STATE_ACTIVE); ret = mk_arch_spawn_instance(image, instance, cpu); if (ret) { - mk_pciback_unlend(instance); + mk_instance_set_state(instance, MK_STATE_FAILED); return ret; } - mk_instance_set_state(instance, MK_STATE_ACTIVE); pr_info("Multikernel instance %d is now active\n", instance->id); return 0; } @@ -475,7 +555,8 @@ bool multikernel_allow_emergency_restart(void) continue; if (instance->state == MK_STATE_ACTIVE || - instance->state == MK_STATE_LOADED) { + instance->state == MK_STATE_LOADED || + instance->state == MK_STATE_FAILED) { pr_info("Found active spawn instance %d (%s) in state %d\n", instance->id, instance->name, instance->state); has_active_spawn = true; @@ -506,9 +587,11 @@ bool multikernel_allow_emergency_restart(void) * shutdown path, which lives in the instance's own kernel image. Loading * a new image over that memory while a CPU is still executing it kills * the machine, so callers that are about to rewrite the image wait here - * first. + * first. Then dispatch the already-queued inbox prefix before old grants can + * be retired or an instance ID reused. An abandoned slot must retain resources. * - * Returns 0 when every CPU is parked, -EBUSY if any did not get there. + * Returns 0 when every CPU is parked and resource ownership is settled, + * -EBUSY if CPU or resource retirement remains uncertain. */ int mk_instance_confirm_parked(struct mk_instance *instance) { @@ -516,8 +599,29 @@ int mk_instance_confirm_parked(struct mk_instance *instance) unsigned int i; int ret, failed = 0; - /* Empty until the instance first ran, so nothing of it is executing */ - mk_cpu_set_for_each(i, phys_cpu, instance->cpus_on_slot) { + /* + * CPU stop does not settle remote resource rollback or PCI/DMA state. + * shortcut: quarantine lasts until controller reboot; recovery needs a + * protocol that proves every admitted resource operation has retired. + */ + if (instance->resource_uncertain) + return -EBUSY; + + /* Ownership is authoritative; tracking may be incomplete after a failure. */ + mk_cpu_set_for_each(i, phys_cpu, instance->cpus) { + ret = mk_arch_confirm_parked(instance, phys_cpu); + if (ret) { + pr_err("Instance %d (%s): CPU %llu is not parked: %d\n", + instance->id, instance->name, phys_cpu, ret); + failed++; + } + } + + /* Also retain/probe orphan tracking; confirmation may delete its entry. */ + for (i = mk_cpu_set_count(instance->cpus_on_slot); i-- > 0; ) { + phys_cpu = instance->cpus_on_slot->ids[i]; + if (mk_cpu_set_contains(instance->cpus, phys_cpu)) + continue; ret = mk_arch_confirm_parked(instance, phys_cpu); if (ret) { pr_err("Instance %d (%s): CPU %llu is not parked: %d\n", @@ -526,7 +630,7 @@ int mk_instance_confirm_parked(struct mk_instance *instance) } } - return failed ? -EBUSY : 0; + return failed ? -EBUSY : mk_ipi_poll(); } /** @@ -553,6 +657,7 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, return -EINVAL; } + guard(mutex)(&mk_pool->cpus_lock); requested_count = mk_cpu_set_count(cpus); if (requested_count == 0) { pr_info("No CPUs requested for instance %d (%s)\n", @@ -589,6 +694,7 @@ int mk_instance_transfer_cpus(struct mk_instance *instance, mk_cpu_set_del(mk_pool->cpus, phys_cpu); mk_cpu_set_add(instance->cpus, phys_cpu); } + mk_instance_ipi_retarget(instance); mk_cpu_set_format(buf, sizeof(buf), instance->cpus); pr_info("Transferred %u CPUs from pool to instance %d (%s): %s\n", @@ -621,6 +727,7 @@ int mk_instance_return_cpus(struct mk_instance *instance, return -EINVAL; } + guard(mutex)(&mk_pool->cpus_lock); requested_count = mk_cpu_set_count(cpus); if (requested_count == 0) { pr_info("No CPUs requested to return from instance %d (%s)\n", @@ -659,6 +766,7 @@ int mk_instance_return_cpus(struct mk_instance *instance, mk_cpu_set_add(mk_pool->cpus, phys_cpu); mk_cpu_set_del(instance->cpus, phys_cpu); } + mk_instance_ipi_retarget(instance); pr_info("Returned %u CPUs from instance %d (%s) to the pool: %s\n", requested_count, instance->id, instance->name, buf); @@ -669,8 +777,8 @@ int mk_instance_return_cpus(struct mk_instance *instance, /** * mk_pool_cpus_returned() - Is every pool CPU back in this kernel? * - * True when the pool holds no free CPU and no instance owns one, so - * nothing can be sitting in a park loop. Pool memory that parked CPUs + * True when the pool holds no free CPU, no child instance owns one, and + * locally owned CPUs are online. Pool memory that parked CPUs * execute from may only be freed once this holds. The answer is only as * good as the invariant that a CPU is recorded in the pool set or in an * instance's set before it parks, which is why the move paths reserve @@ -679,11 +787,29 @@ int mk_instance_return_cpus(struct mk_instance *instance, bool mk_pool_cpus_returned(void) { struct mk_instance *instance; + mk_phys_cpu_t phys_cpu; + unsigned int i; bool returned = true; + lockdep_assert_held(&mk_overlay_mutex); if (mk_pool && !mk_cpu_set_empty(mk_pool->cpus)) return false; + /* + * A failed CPUHP handback remains locally owned, even after mailbox ACK. + * Ownership alone cannot prove it left the pool code and page tables. + * Native offlining after an online observation stays in this kernel; + * donation to the pool is excluded by the resource controller lock. + */ + if (!mk_self || !mk_self->cpus) + return false; + mk_cpu_set_for_each(i, phys_cpu, mk_self->cpus) { + int cpu = arch_cpu_from_physical_id(phys_cpu); + + if (cpu < 0 || (unsigned int)cpu >= nr_cpu_ids || !cpu_online(cpu)) + return false; + } + mutex_lock(&mk_instance_mutex); list_for_each_entry(instance, &mk_instance_list, list) { if (instance == mk_self) @@ -924,7 +1050,7 @@ static int mk_instance_reserve_platform_devices(struct mk_instance *instance, * @devfn: PCI device and function (combined) * * Transfers a single PCI device from self instance to the specified instance. - * Used for dynamic PCI device hotplug to non-running instances. + * Reserve ownership before lending config access or publishing a hotplug request. * * Returns: 0 on success, negative error code on failure */ @@ -1080,6 +1206,27 @@ bool mk_root_has_pci_device(u16 domain, u8 bus, u8 devfn) return false; } +/* The free-pool path must not duplicate another instance's retained grant. */ +bool mk_pci_device_assigned(u16 domain, u8 bus, u8 devfn) +{ + struct mk_instance *instance; + struct mk_pci_device *device; + + lockdep_assert_held(&mk_overlay_mutex); + guard(mutex)(&mk_instance_mutex); + list_for_each_entry(instance, &mk_instance_list, list) { + if (instance == mk_self || !instance->pci_devices_valid) + continue; + list_for_each_entry(device, &instance->pci_devices, list) { + if (device->domain == domain && device->bus == bus && + device->slot == PCI_SLOT(devfn) && + device->func == PCI_FUNC(devfn)) + return true; + } + } + return false; +} + /** * mk_root_add_pci_device - List a PCI device as free in the root pool * @domain: PCI domain @@ -1265,8 +1412,7 @@ static int mk_instance_transfer_memory(struct mk_instance *instance, u64 size, return 0; cleanup: - mk_instance_free_memory(instance); - return ret; + return mk_instance_free_memory(instance) ?: ret; } static int mk_instance_reserve_memory(struct mk_instance *instance, @@ -1286,14 +1432,40 @@ static int mk_instance_reserve_memory(struct mk_instance *instance, * Note: The memory is returned to the global multikernel pool by * multikernel_destroy_instance_pool(), which makes it available for * future instance allocations (including mk_self). + * Return: 0 on success, or an error with memory and metadata retained. */ -void mk_instance_free_memory(struct mk_instance *instance) +int mk_instance_free_memory(struct mk_instance *instance) { struct mk_memory_region *region, *tmp; u64 total_freed = 0; + int ret; if (!instance) - return; + return 0; + /* Never detach metadata or forget the pool when execution is uncertain. */ + if (instance->resource_uncertain || + instance->state == MK_STATE_ACTIVE || instance->state == MK_STATE_FAILED || + instance->kimage) + goto busy; + + /* First host baseline has no allocation to retire; its CPUs are running. */ + if (instance == mk_self && !instance->instance_pool && !instance->ctrl_va && + list_empty(&instance->memory_regions) && + mk_cpu_set_empty(instance->cpus_on_slot)) + return 0; + + if (mk_arch_release_instance(instance)) + goto busy; + + /* Unmap can fail; keep the entire allocation and its metadata on error. */ + if (instance->ctrl_va) { + ret = mk_pool_unmap(instance->ctrl_phys, MK_CTRL_BLOCK_SIZE); + if (ret) { + pr_err("Instance %d control unmap failed: %d; retaining memory\n", + instance->id, ret); + return ret; + } + } list_for_each_entry_safe(region, tmp, &instance->memory_regions, list) { u64 region_size = resource_size(®ion->res); @@ -1327,24 +1499,7 @@ void mk_instance_free_memory(struct mk_instance *instance) * instance's memory back to the host pool before the block * goes away. */ - if (mk_arch_release_instance(instance)) { - /* - * A CPU never claimed its repark: it is still parked - * on this instance's context, executing from the park - * page inside this pool. Handing the memory back - * would let the next instance overwrite code a CPU is - * running, so leak the whole pool instead. - */ - pr_err("Instance %d (%s): leaking its %zu byte pool, a lost CPU still parks in it\n", - instance->id, instance->name, - instance->pool_size); - instance->instance_pool = NULL; - instance->pool_size = 0; - return; - } - if (instance->ctrl_va) { - mk_pool_unmap(instance->ctrl_phys, MK_CTRL_BLOCK_SIZE); mk_instance_mem_free(instance, instance->ctrl_va, MK_CTRL_BLOCK_SIZE); instance->ctrl_va = NULL; @@ -1359,6 +1514,12 @@ void mk_instance_free_memory(struct mk_instance *instance) pr_debug("Freed all memory regions and pool for instance %d (%s)\n", instance->id, instance->name); + return 0; + +busy: + pr_err("Instance %d memory release refused; retaining pool and metadata\n", + instance->id); + return -EBUSY; } /** @@ -1381,8 +1542,10 @@ int mk_instance_reserve_resources(struct mk_instance *instance, return -EINVAL; } - /* Free any existing memory regions first */ - mk_instance_free_memory(instance); + /* Do not overwrite ownership when the old allocation cannot retire. */ + ret = mk_instance_free_memory(instance); + if (ret) + return ret; /* Reserve memory regions */ ret = mk_instance_reserve_memory(instance, config); @@ -1586,10 +1749,14 @@ void mk_kimage_free(struct kimage *image, void *virt_addr, size_t size) struct mk_shutdown_work { struct work_struct work; + u64 msg_id; u32 flags; int sender_instance_id; }; +/* Shutdown parks forever; one slot avoids slab allocation under mk_drain_lock. */ +static struct mk_shutdown_work mk_shutdown; +static atomic_t mk_shutdown_requested = ATOMIC_INIT(0); /* * Notify @target_id that this kernel is going down, while messaging @@ -1598,7 +1765,8 @@ struct mk_shutdown_work { * from a voluntary halt the parent never asked for (HALTED); both mean * "my CPUs are about to park on my context". */ -static void __noreturn mk_notify_down_and_park(int target_id, u32 subtype) +static void __noreturn mk_notify_down_and_park(int target_id, u32 subtype, + u64 msg_id, bool notify) { struct mk_resource_ack ack; @@ -1606,7 +1774,10 @@ static void __noreturn mk_notify_down_and_park(int target_id, u32 subtype) ack.result = 0; ack.resource_id = mk_self->id; - mk_send_message(target_id, MK_MSG_SYSTEM, subtype, &ack, sizeof(ack)); + /* An unqualified exit still parks, but cannot authorize host reuse. */ + if (notify) + mk_send_message_id(target_id, MK_MSG_SYSTEM, subtype, msg_id, + &ack, sizeof(ack)); pr_info("Multikernel instance %d shutting down\n", mk_self->id); @@ -1620,85 +1791,119 @@ static void __noreturn mk_notify_down_and_park(int target_id, u32 subtype) /** * mk_halt_to_pool - Halt this spawn kernel, returning its CPUs to the pool + * @notify: Caller has excluded new resource operations through final parking * - * Called from the spawn kernel's machine halt path. A voluntary exit - * cannot stay contained in this kernel: the parent owns the instance's - * lifecycle, and without notice it would consider the instance running - * forever. Send the parent (instance 0) a HALTED event, then park every - * CPU in the pool wait loop. + * Called from the spawn kernel's machine halt path. Only a qualified normal + * shutdown may send the parent (instance 0) a HALTED event. Panic, unqualified + * emergency and contended exits still park every CPU, but leave the parent retaining + * the instance because a parked snapshot alone cannot prove quiescence. */ -void __noreturn mk_halt_to_pool(void) +void __noreturn mk_halt_to_pool(bool notify) { - mk_notify_down_and_park(0, MK_SYS_HALTED); + mk_notify_down_and_park(0, MK_SYS_HALTED, 0, notify); } static void mk_shutdown_work_fn(struct work_struct *work) { struct mk_shutdown_work *sw = container_of(work, struct mk_shutdown_work, work); int sender_instance_id = sw->sender_instance_id; - - kfree(sw); - mk_notify_down_and_park(sender_instance_id, MK_SYS_SHUTDOWN_ACK); + u64 msg_id = sw->msg_id; + + /* This worker may sleep; the voluntary machine-halt path may not. */ + mk_hotplug_quiesce(); + /* Drain native CPU hotplug and prevent new starts before the ACK. */ + cpu_hotplug_disable(); + /* Finish local controllers, then exclude them until this kernel stops. */ + mutex_lock(&mk_overlay_mutex); + mk_notify_down_and_park(sender_instance_id, MK_SYS_SHUTDOWN_ACK, msg_id, true); } /* - * Mark a halted instance re-spawnable. No wakeups are published here: - * the instance's CPUs may still be on their way to the park loop, and - * poking its context while its (old or next) kernel also publishes on - * it corrupts the single-producer mailbox. The kexec path confirms the - * CPUs are parked before it rewrites the image. + * A notice is not stop proof. Keep the old state and allocations until every + * owned/tracked CPU is confirmed parked. The resource lock excludes exec, + * ownership changes and teardown throughout confirmation and publication. */ -static void mk_instance_settle_halted(struct mk_instance *instance) +static int mk_instance_settle_halted(struct mk_instance *instance) { - pr_info("Instance %d (%s) halted, CPUs parking in pool\n", + int ret; + + lockdep_assert_held(&mk_overlay_mutex); + ret = mk_instance_confirm_parked(instance); + if (ret) + return ret; + pr_info("Instance %d (%s) halted, all CPUs confirmed parked\n", instance->id, instance->name); mk_instance_set_state(instance, MK_STATE_LOADED); + return 0; } -struct mk_halted_work { - struct work_struct work; - int instance_id; -}; - static void mk_halted_work_fn(struct work_struct *work) { - struct mk_halted_work *aw = - container_of(work, struct mk_halted_work, work); - struct mk_instance *instance; + struct mk_instance *instance = + container_of(work, struct mk_instance, halted_work); + int ret; - instance = mk_instance_find(aw->instance_id); - if (instance) { - mk_instance_settle_halted(instance); - mk_instance_put(instance); - } else { - pr_warn("Shutdown ACK from unknown instance %d\n", - aw->instance_id); + guard(mutex)(&mk_overlay_mutex); + + if (instance->state == MK_STATE_ACTIVE || instance->state == MK_STATE_FAILED) { + ret = mk_instance_settle_halted(instance); + if (ret) + pr_warn("Instance %d halt remains unconfirmed: %d\n", + instance->id, ret); } - kfree(aw); + atomic_set(&instance->halted_pending, 0); + mk_instance_put(instance); } -static void mk_system_msg_handler(u32 msg_type, u32 subtype, +static void mk_queue_halted(int id) +{ + struct mk_instance *instance = mk_instance_find(id); + + if (!instance) + return; + /* Pin identity and coalesce notices without allocating under mk_drain_lock. */ + if (atomic_cmpxchg(&instance->halted_pending, 0, 1)) { + mk_instance_put(instance); + return; + } + if (!schedule_work(&instance->halted_work)) { + atomic_set(&instance->halted_pending, 0); + mk_instance_put(instance); + } +} + +static void mk_system_msg_handler(u32 msg_type, u32 subtype, u64 msg_id, void *payload, u32 payload_len, void *ctx) { + union { + struct mk_shutdown_payload shutdown; + struct mk_resource_ack ack; + } decoded; + if (msg_type != MK_MSG_SYSTEM) return; + /* Preserve the wire layout while providing aligned typed accesses. */ + if (payload_len) { + memcpy(&decoded, payload, min_t(size_t, payload_len, sizeof(decoded))); + payload = &decoded; + } + switch (subtype) { case MK_SYS_SHUTDOWN: { struct mk_shutdown_payload *req = payload; - struct mk_shutdown_work *sw; + struct mk_shutdown_work *sw = &mk_shutdown; if (payload_len < sizeof(*req)) return; + if (atomic_cmpxchg(&mk_shutdown_requested, 0, 1)) + return; pr_info("Shutdown requested by instance %d\n", req->sender_instance_id); - sw = kmalloc(sizeof(*sw), GFP_ATOMIC); - if (!sw) - return; - INIT_WORK(&sw->work, mk_shutdown_work_fn); + sw->msg_id = msg_id; sw->flags = req->flags; sw->sender_instance_id = req->sender_instance_id; schedule_work(&sw->work); @@ -1707,7 +1912,7 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, case MK_SYS_SHUTDOWN_ACK: { struct mk_resource_ack *ack = payload; - if (payload_len < sizeof(*ack)) + if (payload_len < sizeof(*ack) || ack->operation != MK_SYS_SHUTDOWN) return; /* * Reply to a shutdown this kernel requested: wake the @@ -1715,30 +1920,18 @@ static void mk_system_msg_handler(u32 msg_type, u32 subtype, * and settles its state itself. */ mk_msg_pending_complete(MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, - ack->resource_id, ack->result); + ack->resource_id, msg_id, ack->result); break; } case MK_SYS_HALTED: { struct mk_resource_ack *ack = payload; - struct mk_halted_work *aw; - if (payload_len < sizeof(*ack)) + if (payload_len < sizeof(*ack) || ack->operation != MK_SYS_SHUTDOWN || + ack->result || ack->resource_id > INT_MAX) return; - /* - * The instance halted itself; nobody is waiting on it, so - * settle its state from here. Deferred to a workqueue - * because instance lookup takes a mutex and the CPUs still - * need time to reach the park loop, while this runs in IPI - * context. - */ - aw = kmalloc(sizeof(*aw), GFP_ATOMIC); - if (!aw) - break; - - INIT_WORK(&aw->work, mk_halted_work_fn); - aw->instance_id = ack->resource_id; - schedule_work(&aw->work); + /* Lookup is IRQ-safe; stop confirmation still needs process context. */ + mk_queue_halted(ack->resource_id); break; } default: @@ -1764,11 +1957,14 @@ int multikernel_halt_by_id(int mk_id) struct mk_pending_msg *pending; int ret; + guard(mutex)(&mk_overlay_mutex); + instance = mk_instance_find(mk_id); if (!instance) return -ENOENT; - if (instance->state != MK_STATE_ACTIVE) { + if (instance == mk_self || (instance->state != MK_STATE_ACTIVE && + instance->state != MK_STATE_FAILED)) { mk_instance_put(instance); return -EINVAL; } @@ -1777,28 +1973,22 @@ int multikernel_halt_by_id(int mk_id) payload.sender_instance_id = mk_self->id; pending = mk_msg_pending_add(MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, mk_id); - if (!pending) { + if (IS_ERR(pending)) { mk_instance_put(instance); - return -ENOMEM; + return PTR_ERR(pending); } - ret = mk_send_message(mk_id, MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, - &payload, sizeof(payload)); + ret = mk_send_message_id(mk_id, MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, + mk_msg_pending_id(pending), &payload, sizeof(payload)); if (ret < 0) { - mk_msg_pending_wait(pending, 0); + mk_msg_pending_cancel(pending); mk_instance_put(instance); return ret; } ret = mk_msg_pending_wait(pending, 30000); - if (ret == 0) { - if (mk_instance_confirm_parked(instance)) - pr_warn("Multikernel instance %d halted with CPUs unaccounted for\n", - mk_id); - - mk_instance_set_state(instance, MK_STATE_LOADED); - pr_info("Multikernel instance %d halted (graceful)\n", mk_id); - } + if (!ret) + ret = mk_instance_settle_halted(instance); mk_instance_put(instance); return ret; @@ -1943,6 +2133,11 @@ int multikernel_force_halt_by_id(int mk_id) int cpu_count = 0; int ret; + if (!IS_ENABLED(CONFIG_ARCH_HAS_MK_FORCE_STOP)) + return -EOPNOTSUPP; + + guard(mutex)(&mk_overlay_mutex); + instance = mk_instance_find(mk_id); if (!instance) return -ENOENT; @@ -1954,13 +2149,13 @@ int multikernel_force_halt_by_id(int mk_id) } /* - * LOADED is allowed for the retry case: a previous halt already - * settled the state, but a CPU that missed its NMI is still - * running the old image and kexec refuses to reload it. Without - * a rerun the instance is stuck for good. The parent is ACTIVE. + * FAILED permits recovery after an uncertain partial start. LOADED + * permits an explicit retry, but is never stop proof: settlement + * must still confirm every assigned CPU before publishing it again. */ if (instance->state != MK_STATE_ACTIVE && - instance->state != MK_STATE_LOADED) { + instance->state != MK_STATE_LOADED && + instance->state != MK_STATE_FAILED) { pr_err("Instance %d not running (state=%d), nothing to force halt\n", mk_id, instance->state); mk_instance_put(instance); @@ -1985,8 +2180,10 @@ int multikernel_force_halt_by_id(int mk_id) pr_info("Force halting multikernel instance %d via NMI\n", mk_id); ret = mk_arm_force_halt(instance); - if (ret) - pr_err("Failed to arm force-halt marker: %d (sending NMI anyway)\n", ret); + if (ret) { + pr_err("Failed to arm force-halt marker: %d\n", ret); + goto out_targets; + } mk_cpu_set_for_each(i, phys_cpu, targets) { mk_force_stop_cpu(phys_cpu); @@ -2010,9 +2207,10 @@ int multikernel_force_halt_by_id(int mk_id) if (!ret) mk_pcifront_host_fenced(); } else { - mk_instance_settle_halted(instance); + ret = mk_instance_settle_halted(instance); } +out_targets: mk_cpu_set_free(targets); mk_instance_put(instance); return ret; @@ -2028,17 +2226,18 @@ static int __init multikernel_init(void) return ret; } - ret = mk_register_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler, NULL); + /* Resource admission must be ready before a shutdown can be received. */ + ret = mk_hotplug_init(); if (ret < 0) { - pr_err("Failed to register system message handler: %d\n", ret); + pr_err("Failed to initialize multikernel hotplug: %d\n", ret); mk_messaging_cleanup(); return ret; } - ret = mk_hotplug_init(); + ret = mk_register_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler, NULL); if (ret < 0) { - pr_err("Failed to initialize multikernel hotplug: %d\n", ret); - mk_unregister_msg_handler(MK_MSG_SYSTEM, mk_system_msg_handler); + pr_err("Failed to register system message handler: %d\n", ret); + mk_hotplug_cleanup(); mk_messaging_cleanup(); return ret; } diff --git a/kernel/multikernel/dts.c b/kernel/multikernel/dts.c index c34b2254c9a65d..d312d265b76fb5 100644 --- a/kernel/multikernel/dts.c +++ b/kernel/multikernel/dts.c @@ -890,11 +890,9 @@ static int mk_dt_emit_pool_members(void *fdt) return -ENOMEM; /* - * The root device tree is only generated from kernfs reads, which - * hold no instance lock; instance creation generates a device tree - * for the new instance, never for the root. The pool set itself is - * mutated under no lock, so a concurrent move can still make this - * snapshot stale. + * The caller holds mk_overlay_mutex to stabilize pool and instance + * CPU membership. Take the nested registry lock for list traversal; + * the root kernfs reader does not already hold it. */ lockdep_assert_not_held(&mk_instance_mutex); diff --git a/kernel/multikernel/hotplug.c b/kernel/multikernel/hotplug.c index bd0e03983fd822..6d3ee86e6eb0a0 100644 --- a/kernel/multikernel/hotplug.c +++ b/kernel/multikernel/hotplug.c @@ -23,6 +23,9 @@ #include #include #include +#include +#include +#include #include "internal.h" static const char mk_mem_resource_name[] = "System RAM (multikernel)"; @@ -66,17 +69,147 @@ static DEFINE_MUTEX(mk_hotplug_mutex); static LIST_HEAD(mk_hotplug_ops); /* - * CPU Hotplug Operations + * A remove ACK must not overtake an earlier queued add. CPU requests share an + * ordered queue; memory and device workers retain their existing scheduling. + * The raw lock protects resource admission and CPU queue detachment; + * the receive path runs in hard-IPI context, including on PREEMPT_RT. */ -struct mk_cpu_hotplug_work { +static DEFINE_RAW_SPINLOCK(mk_cpu_hotplug_lock); +static struct workqueue_struct *mk_cpu_hotplug_wq; +static bool mk_hotplug_accepting; +static unsigned int mk_hotplug_inflight; +static DECLARE_WAIT_QUEUE_HEAD(mk_hotplug_wait); + +struct mk_hotplug_work { struct work_struct work; - mk_phys_cpu_t cpu_id; - u32 numa_node; - u32 flags; - int sender_instance_id; /* For sending ACK back */ - u32 operation; /* MK_RES_CPU_ADD or MK_RES_CPU_REMOVE */ + bool busy; + u32 operation; + u64 msg_id; + union { + struct mk_cpu_resource_payload cpu; + struct mk_mem_resource_payload mem; + struct mk_device_resource_payload device; + } payload; }; +/* A slot keeps its callback and queue for its entire lifetime. */ +static struct mk_hotplug_work mk_cpu_requests[MK_IPI_RING_SIZE]; +static struct mk_hotplug_work mk_other_requests[MK_IPI_RING_SIZE]; + +static int mk_queue_hotplug_request(u32 operation, u64 msg_id, + const void *payload, u32 payload_len) +{ + struct mk_hotplug_work *requests, *request; + unsigned long flags; + size_t size; + bool cpu = false; + int i, ret = -ENOMEM; + + switch (operation) { + case MK_RES_CPU_ADD: + case MK_RES_CPU_REMOVE: + size = sizeof(request->payload.cpu); + cpu = true; + break; + case MK_RES_MEM_ADD: + case MK_RES_MEM_REMOVE: + size = sizeof(request->payload.mem); + break; + case MK_RES_DEVICE_ADD: + case MK_RES_DEVICE_REMOVE: + size = sizeof(request->payload.device); + break; + default: + return -EINVAL; + } + if (payload_len < size) + return -EINVAL; + + requests = cpu ? mk_cpu_requests : mk_other_requests; + raw_spin_lock_irqsave(&mk_cpu_hotplug_lock, flags); + if (!mk_hotplug_accepting) { + ret = -ESHUTDOWN; + goto out; + } + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + request = &requests[i]; + if (request->busy) + continue; + request->busy = true; + request->operation = operation; + request->msg_id = msg_id; + memcpy(&request->payload, payload, size); + mk_hotplug_inflight++; + if (cpu) + queue_work(mk_cpu_hotplug_wq, &request->work); + else + schedule_work(&request->work); + ret = 0; + break; + } +out: + raw_spin_unlock_irqrestore(&mk_cpu_hotplug_lock, flags); + return ret; +} + +static void mk_hotplug_work_done(struct mk_hotplug_work *request) +{ + unsigned long flags; + bool idle; + + /* Payload use ends here; requeue keeps the same work function and queue. */ + raw_spin_lock_irqsave(&mk_cpu_hotplug_lock, flags); + request->busy = false; + idle = !--mk_hotplug_inflight; + raw_spin_unlock_irqrestore(&mk_cpu_hotplug_lock, flags); + if (idle) + wake_up_all(&mk_hotplug_wait); +} + +static bool mk_hotplug_idle(void) +{ + unsigned long flags; + bool idle; + + raw_spin_lock_irqsave(&mk_cpu_hotplug_lock, flags); + idle = !mk_hotplug_inflight; + raw_spin_unlock_irqrestore(&mk_cpu_hotplug_lock, flags); + return idle; +} + +/* + * Process context only, outside all controller/tracking locks and resource + * workers. Close incoming admission before waiting for accepted workers to + * finish mutation, ACK transmission and request-slot retirement. A stuck + * worker must prevent a successful shutdown ACK, not permit unsafe reuse. + * This does not fence local controllers, drivers or other ring writers. + */ +void mk_hotplug_quiesce(void) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&mk_cpu_hotplug_lock, flags); + mk_hotplug_accepting = false; + raw_spin_unlock_irqrestore(&mk_cpu_hotplug_lock, flags); + wait_event(mk_hotplug_wait, mk_hotplug_idle()); +} + +static struct workqueue_struct *mk_cpu_hotplug_detach(void) +{ + struct workqueue_struct *wq; + unsigned long flags; + + raw_spin_lock_irqsave(&mk_cpu_hotplug_lock, flags); + wq = mk_cpu_hotplug_wq; + mk_cpu_hotplug_wq = NULL; + raw_spin_unlock_irqrestore(&mk_cpu_hotplug_lock, flags); + return wq; +} + +/* + * CPU Hotplug Operations + */ + /* * Ownership tracking for this kernel's own hotplug: mk_self->cpus * is the set of CPUs this kernel owns, in the host and in spawn kernels @@ -84,16 +217,10 @@ struct mk_cpu_hotplug_work { * it belongs to the mk_send_cpu_* initiator paths of the kernel that * manages the pool. */ -static void mk_account_cpu_online(mk_phys_cpu_t cpu_id) -{ - if (mk_self->cpus && mk_cpu_set_add(mk_self->cpus, cpu_id)) - pr_warn("Multikernel hotplug: Failed to track CPU %llu\n", - cpu_id); -} - static void mk_account_cpu_offline(mk_phys_cpu_t cpu_id) { mk_cpu_set_del(mk_self->cpus, cpu_id); + mk_instance_ipi_retarget(mk_self); } /** @@ -117,10 +244,12 @@ static int mk_cpu_to_logical(mk_phys_cpu_t cpu_id) static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) { + bool owned, attempted = false; int logical_cpu; int ret; struct mk_hotplug_op *op; + lockdep_assert_held(&mk_overlay_mutex); pr_info("Multikernel hotplug: Adding CPU %llu (numa=%d, flags=0x%x)\n", cpu_id, (int)numa_node, flags); @@ -130,7 +259,7 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) * Not present: this pool CPU has never run here (or was * removed earlier). It was enumerated at boot from the host * tree's cpus list, so it has a logical slot in the possible - * map; mark it present again before onlining. + * map. reclaim_cpu() restores presence under native serialization. */ logical_cpu = arch_cpu_from_physical_id(cpu_id); if (logical_cpu < 0) { @@ -138,35 +267,37 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) cpu_id); return -ENODEV; } - set_cpu_present(logical_cpu, true); } - if (cpu_online(logical_cpu)) { - pr_warn("Multikernel hotplug: CPU %d (phys %llu) already online\n", - logical_cpu, cpu_id); - mk_account_cpu_online(cpu_id); - return 0; - } + if (logical_cpu >= nr_cpu_ids || !cpu_possible(logical_cpu)) + return -ENODEV; + if (!mk_self->cpus) + return -EINVAL; + owned = mk_cpu_set_contains(mk_self->cpus, cpu_id); + ret = mk_cpu_set_add(mk_self->cpus, cpu_id); + if (ret) + return ret; + /* Register absent enumerated CPUs before exposing them to native bringup. */ + lock_device_hotplug(); if (!get_cpu_device(logical_cpu)) { struct cpu *c = &per_cpu(cpu_devices, logical_cpu); + c->hotpluggable = true; - ret = register_cpu(c, logical_cpu); - if (ret) { - pr_err("Multikernel hotplug: Failed to register CPU %d: %d\n", - logical_cpu, ret); - return ret; - } + ret = cpu_present(logical_cpu) ? -ENODEV : register_cpu(c, logical_cpu); } + unlock_device_hotplug(); + if (ret) + goto failed; - ret = add_cpu(logical_cpu); + ret = reclaim_cpu(logical_cpu, &attempted); if (ret < 0) { pr_err("Multikernel hotplug: Failed to add CPU %d (phys %llu): %d\n", logical_cpu, cpu_id, ret); - return ret; + goto failed; } - mk_account_cpu_online(cpu_id); + mk_instance_ipi_retarget(mk_self); /* Track the operation for potential rollback */ op = kzalloc(sizeof(*op), GFP_KERNEL); @@ -184,6 +315,20 @@ static int mk_do_cpu_add(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) logical_cpu, cpu_id); return 0; +failed: + if (!attempted) { + if (!owned) + mk_cpu_set_del(mk_self->cpus, cpu_id); + } else if (mk_pool) { + /* + * Keep pool membership until after the wake callback: it selects + * the local park slot. Once bringup was attempted, even a rolled + * back CPU must not become assignable without independent proof. + */ + guard(mutex)(&mk_pool->cpus_lock); + mk_cpu_set_del(mk_pool->cpus, cpu_id); + } + return ret; } static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) @@ -198,19 +343,6 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) return -ENODEV; } - if (!cpu_online(logical_cpu)) { - pr_warn("Multikernel hotplug: CPU %d (phys %llu) already offline\n", - logical_cpu, cpu_id); - mk_account_cpu_offline(cpu_id); - return 0; - } - - /* Don't allow removing CPU 0 (boot processor) */ - if (logical_cpu == 0) { - pr_err("Multikernel hotplug: Cannot remove boot CPU\n"); - return -EINVAL; - } - ret = depart_cpu(logical_cpu); if (ret < 0) { pr_err("Multikernel hotplug: Failed to remove CPU %d (phys %llu): %d\n", @@ -220,12 +352,6 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) mk_account_cpu_offline(cpu_id); - /* - * Clear CPU from present mask to prevent host kernel from trying - * to interact with it after spawn kernel takes over. - */ - set_cpu_present(logical_cpu, false); - /* Track the operation for potential rollback */ op = kzalloc(sizeof(*op), GFP_KERNEL); if (op) { @@ -246,122 +372,95 @@ static int mk_do_cpu_remove(mk_phys_cpu_t cpu_id) static void mk_cpu_add_work_fn(struct work_struct *work) { - struct mk_cpu_hotplug_work *hp_work = container_of(work, struct mk_cpu_hotplug_work, work); + struct mk_hotplug_work *hp_work = container_of(work, struct mk_hotplug_work, work); + struct mk_cpu_resource_payload *payload = &hp_work->payload.cpu; struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_cpu_add(hp_work->cpu_id, hp_work->numa_node, hp_work->flags); + /* A controller may hold this lock while waiting for a remote ACK. */ + ret = -EBUSY; + if (mutex_trylock(&mk_overlay_mutex)) { + ret = mk_do_cpu_add(payload->cpu_id, payload->numa_node, payload->flags); + mutex_unlock(&mk_overlay_mutex); + } /* Send ACK back to sender */ ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = hp_work->cpu_id; + ack.resource_id = payload->cpu_id; - ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, - &ack, sizeof(ack)); + ack_ret = mk_send_message_id(payload->sender_instance_id, MK_MSG_RESOURCE, + MK_RES_ACK, hp_work->msg_id, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for CPU %llu: %d\n", - hp_work->cpu_id, ack_ret); + payload->cpu_id, ack_ret); } - kfree(hp_work); + mk_hotplug_work_done(hp_work); } static void mk_cpu_remove_work_fn(struct work_struct *work) { - struct mk_cpu_hotplug_work *hp_work = container_of(work, struct mk_cpu_hotplug_work, work); + struct mk_hotplug_work *hp_work = container_of(work, struct mk_hotplug_work, work); + struct mk_cpu_resource_payload *payload = &hp_work->payload.cpu; struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_cpu_remove(hp_work->cpu_id); + ret = -EBUSY; + if (mutex_trylock(&mk_overlay_mutex)) { + ret = mk_do_cpu_remove(payload->cpu_id); + mutex_unlock(&mk_overlay_mutex); + } /* Send ACK back to sender */ ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = hp_work->cpu_id; + ack.resource_id = payload->cpu_id; - ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, - &ack, sizeof(ack)); + ack_ret = mk_send_message_id(payload->sender_instance_id, MK_MSG_RESOURCE, + MK_RES_ACK, hp_work->msg_id, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for CPU %llu: %d\n", - hp_work->cpu_id, ack_ret); + payload->cpu_id, ack_ret); } - kfree(hp_work); + mk_hotplug_work_done(hp_work); } /** * mk_handle_cpu_add - Handle CPU addition request * @payload: CPU resource payload * @payload_len: Payload length + * @msg_id: Request ID to echo in the reply * * Brings a CPU online in the receiving kernel. This is called on the spawn * kernel side when the host kernel transfers a CPU to it. * * Returns 0 on success, negative error code on failure */ -static int mk_handle_cpu_add(struct mk_cpu_resource_payload *payload, u32 payload_len) +static int mk_handle_cpu_add(struct mk_cpu_resource_payload *payload, u32 payload_len, + u64 msg_id) { - struct mk_cpu_hotplug_work *hp_work; - - if (payload_len < sizeof(*payload)) { - pr_err("Multikernel hotplug: Invalid CPU add payload size: %u\n", payload_len); - return -EINVAL; - } - - /* - * We're in IRQ context (IPI handler), so we can't call add_cpu() directly. - */ - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) - return -ENOMEM; - - INIT_WORK(&hp_work->work, mk_cpu_add_work_fn); - hp_work->cpu_id = payload->cpu_id; - hp_work->numa_node = payload->numa_node; - hp_work->flags = payload->flags; - hp_work->sender_instance_id = payload->sender_instance_id; - hp_work->operation = MK_RES_CPU_ADD; - schedule_work(&hp_work->work); - - return 0; + return mk_queue_hotplug_request(MK_RES_CPU_ADD, msg_id, + payload, payload_len); } /** * mk_handle_cpu_remove - Handle CPU removal request * @payload: CPU resource payload * @payload_len: Payload length + * @msg_id: Request ID to echo in the reply * * Takes a CPU offline in the current kernel. This is called on the host * kernel side before transferring a CPU to a spawn kernel. * * Returns 0 on success, negative error code on failure */ -int mk_handle_cpu_remove(struct mk_cpu_resource_payload *payload, u32 payload_len) +int mk_handle_cpu_remove(struct mk_cpu_resource_payload *payload, u32 payload_len, + u64 msg_id) { - struct mk_cpu_hotplug_work *hp_work; - - if (payload_len < sizeof(*payload)) { - pr_err("Multikernel hotplug: Invalid CPU remove payload size: %u\n", payload_len); - return -EINVAL; - } - - /* - * We're in IRQ context (IPI handler), so we can't call remove_cpu() directly. - */ - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) - return -ENOMEM; - - INIT_WORK(&hp_work->work, mk_cpu_remove_work_fn); - hp_work->cpu_id = payload->cpu_id; - hp_work->numa_node = payload->numa_node; - hp_work->flags = payload->flags; - hp_work->sender_instance_id = payload->sender_instance_id; - hp_work->operation = MK_RES_CPU_REMOVE; - schedule_work(&hp_work->work); - - return 0; + return mk_queue_hotplug_request(MK_RES_CPU_REMOVE, msg_id, + payload, payload_len); } /* @@ -573,113 +672,73 @@ static int mk_do_mem_remove(u64 start_pfn, u64 nr_pages) return 0; } -struct mk_mem_hotplug_work { - struct work_struct work; - u64 start_pfn; - u64 nr_pages; - u32 numa_node; - u32 mem_type; - int sender_instance_id; - u32 operation; -}; - static void mk_mem_add_work_fn(struct work_struct *work) { - struct mk_mem_hotplug_work *hp_work = container_of(work, struct mk_mem_hotplug_work, work); + struct mk_hotplug_work *hp_work = container_of(work, struct mk_hotplug_work, work); + struct mk_mem_resource_payload *payload = &hp_work->payload.mem; struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_mem_add(hp_work->start_pfn, hp_work->nr_pages, - hp_work->numa_node, hp_work->mem_type); + ret = -EBUSY; + if (mutex_trylock(&mk_overlay_mutex)) { + ret = mk_do_mem_add(payload->start_pfn, payload->nr_pages, + payload->numa_node, payload->mem_type); + mutex_unlock(&mk_overlay_mutex); + } ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = hp_work->start_pfn; + ack.resource_id = payload->start_pfn; - ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, - &ack, sizeof(ack)); + ack_ret = mk_send_message_id(payload->sender_instance_id, MK_MSG_RESOURCE, + MK_RES_ACK, hp_work->msg_id, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for mem add at 0x%llx: %d\n", - (u64)hp_work->start_pfn, ack_ret); + (u64)payload->start_pfn, ack_ret); } - kfree(hp_work); + mk_hotplug_work_done(hp_work); } static void mk_mem_remove_work_fn(struct work_struct *work) { - struct mk_mem_hotplug_work *hp_work = container_of(work, struct mk_mem_hotplug_work, work); + struct mk_hotplug_work *hp_work = container_of(work, struct mk_hotplug_work, work); + struct mk_mem_resource_payload *payload = &hp_work->payload.mem; struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_mem_remove(hp_work->start_pfn, hp_work->nr_pages); + ret = -EBUSY; + if (mutex_trylock(&mk_overlay_mutex)) { + ret = mk_do_mem_remove(payload->start_pfn, payload->nr_pages); + mutex_unlock(&mk_overlay_mutex); + } ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = hp_work->start_pfn; + ack.resource_id = payload->start_pfn; - ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, - &ack, sizeof(ack)); + ack_ret = mk_send_message_id(payload->sender_instance_id, MK_MSG_RESOURCE, + MK_RES_ACK, hp_work->msg_id, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for mem remove at 0x%llx: %d\n", - (u64)hp_work->start_pfn, ack_ret); + (u64)payload->start_pfn, ack_ret); } - kfree(hp_work); + mk_hotplug_work_done(hp_work); } -static int mk_handle_mem_add(struct mk_mem_resource_payload *payload, u32 payload_len) +static int mk_handle_mem_add(struct mk_mem_resource_payload *payload, u32 payload_len, + u64 msg_id) { - struct mk_mem_hotplug_work *hp_work; - - if (payload_len < sizeof(*payload)) { - pr_err("Multikernel hotplug: Invalid memory add payload size: %u\n", payload_len); - return -EINVAL; - } - - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) { - pr_err("Multikernel hotplug: Failed to allocate work for mem add at 0x%llx\n", - (u64)payload->start_pfn); - return -ENOMEM; - } - - INIT_WORK(&hp_work->work, mk_mem_add_work_fn); - hp_work->start_pfn = payload->start_pfn; - hp_work->nr_pages = payload->nr_pages; - hp_work->numa_node = payload->numa_node; - hp_work->mem_type = payload->mem_type; - hp_work->sender_instance_id = payload->sender_instance_id; - hp_work->operation = MK_RES_MEM_ADD; - schedule_work(&hp_work->work); - - return 0; + return mk_queue_hotplug_request(MK_RES_MEM_ADD, msg_id, + payload, payload_len); } -static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, u32 payload_len) +static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, u32 payload_len, + u64 msg_id) { - struct mk_mem_hotplug_work *hp_work; - - if (payload_len < sizeof(*payload)) { - pr_err("Multikernel hotplug: Invalid memory remove payload size: %u\n", payload_len); - return -EINVAL; - } - - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) { - pr_err("Multikernel hotplug: Failed to allocate work for mem remove at 0x%llx\n", - (u64)payload->start_pfn); - return -ENOMEM; - } - - INIT_WORK(&hp_work->work, mk_mem_remove_work_fn); - hp_work->start_pfn = payload->start_pfn; - hp_work->nr_pages = payload->nr_pages; - hp_work->sender_instance_id = payload->sender_instance_id; - hp_work->operation = MK_RES_MEM_REMOVE; - schedule_work(&hp_work->work); - - return 0; + return mk_queue_hotplug_request(MK_RES_MEM_REMOVE, msg_id, + payload, payload_len); } /* @@ -689,6 +748,7 @@ static int mk_handle_mem_remove(struct mk_mem_resource_payload *payload, u32 pay static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, const char *driver_override, u32 flags) { +#ifdef CONFIG_PCI struct pci_dev *pdev; struct pci_bus *pci_bus; struct device_driver *drv; @@ -800,10 +860,14 @@ static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); return 0; +#else + return -EOPNOTSUPP; +#endif } static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) { +#ifdef CONFIG_PCI struct pci_dev *pdev; struct pci_bus *pci_bus; struct mk_hotplug_op *op; @@ -850,136 +914,114 @@ static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); return 0; +#else + return -EOPNOTSUPP; +#endif } -struct mk_device_hotplug_work { - struct work_struct work; - u16 domain; - u8 bus; - u8 devfn; - u32 flags; - char driver_override[64]; - int sender_instance_id; - u32 operation; - u16 vendor; - u16 device; - struct mk_pci_bus_desc bus_desc; -}; - static void mk_device_add_work_fn(struct work_struct *work) { - struct mk_device_hotplug_work *hp_work = container_of(work, struct mk_device_hotplug_work, work); + struct mk_hotplug_work *hp_work = container_of(work, struct mk_hotplug_work, work); + struct mk_device_resource_payload *payload = &hp_work->payload.device; struct mk_resource_ack ack; int ret, ack_ret; - ret = 0; - if (mk_manifest_phys()) - ret = mk_pcifront_add_function(hp_work->domain, hp_work->bus, - hp_work->devfn, hp_work->vendor, - hp_work->device, &hp_work->bus_desc); - if (!ret) - ret = mk_do_device_add(hp_work->domain, hp_work->bus, - hp_work->devfn, hp_work->driver_override, - hp_work->flags); + payload->driver_override[sizeof(payload->driver_override) - 1] = '\0'; + ret = -EBUSY; + if (mutex_trylock(&mk_overlay_mutex)) { + ret = 0; + if (mk_manifest_phys()) + ret = mk_pcifront_add_function(payload->domain, payload->bus, + payload->devfn, payload->vendor, + payload->device, &payload->bus_desc); + if (!ret) + ret = mk_do_device_add(payload->domain, payload->bus, + payload->devfn, payload->driver_override, + payload->flags); + mutex_unlock(&mk_overlay_mutex); + } ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = (hp_work->domain << 16) | (hp_work->bus << 8) | hp_work->devfn; + ack.resource_id = ((u32)payload->domain << 16) | (payload->bus << 8) | payload->devfn; - ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, - &ack, sizeof(ack)); + ack_ret = mk_send_message_id(payload->sender_instance_id, MK_MSG_RESOURCE, + MK_RES_ACK, hp_work->msg_id, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for device %04x:%02x:%02x.%x: %d\n", - hp_work->domain, hp_work->bus, - PCI_SLOT(hp_work->devfn), PCI_FUNC(hp_work->devfn), ack_ret); + payload->domain, payload->bus, + PCI_SLOT(payload->devfn), PCI_FUNC(payload->devfn), ack_ret); } - kfree(hp_work); + mk_hotplug_work_done(hp_work); } static void mk_device_remove_work_fn(struct work_struct *work) { - struct mk_device_hotplug_work *hp_work = container_of(work, struct mk_device_hotplug_work, work); + struct mk_hotplug_work *hp_work = container_of(work, struct mk_hotplug_work, work); + struct mk_device_resource_payload *payload = &hp_work->payload.device; struct mk_resource_ack ack; int ret, ack_ret; - ret = mk_do_device_remove(hp_work->domain, hp_work->bus, hp_work->devfn); + ret = -EBUSY; + if (mutex_trylock(&mk_overlay_mutex)) { + ret = mk_do_device_remove(payload->domain, payload->bus, payload->devfn); + mutex_unlock(&mk_overlay_mutex); + } ack.operation = hp_work->operation; ack.result = ret; - ack.resource_id = (hp_work->domain << 16) | (hp_work->bus << 8) | hp_work->devfn; + ack.resource_id = ((u32)payload->domain << 16) | (payload->bus << 8) | payload->devfn; - ack_ret = mk_send_message(hp_work->sender_instance_id, MK_MSG_RESOURCE, MK_RES_ACK, - &ack, sizeof(ack)); + ack_ret = mk_send_message_id(payload->sender_instance_id, MK_MSG_RESOURCE, + MK_RES_ACK, hp_work->msg_id, &ack, sizeof(ack)); if (ack_ret < 0) { pr_warn("Multikernel hotplug: Failed to send ACK for device %04x:%02x:%02x.%x: %d\n", - hp_work->domain, hp_work->bus, - PCI_SLOT(hp_work->devfn), PCI_FUNC(hp_work->devfn), ack_ret); + payload->domain, payload->bus, + PCI_SLOT(payload->devfn), PCI_FUNC(payload->devfn), ack_ret); } - kfree(hp_work); + mk_hotplug_work_done(hp_work); } -static int mk_handle_device_add(struct mk_device_resource_payload *payload, u32 payload_len) +static int mk_handle_device_add(struct mk_device_resource_payload *payload, u32 payload_len, + u64 msg_id) { - struct mk_device_hotplug_work *hp_work; - - if (payload_len < sizeof(*payload)) { - pr_err("Multikernel hotplug: Invalid device add payload size: %u\n", payload_len); - return -EINVAL; - } - - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) { - pr_err("Multikernel hotplug: Failed to allocate work structure for device %04x:%02x:%02x.%x\n", - payload->domain, payload->bus, - PCI_SLOT(payload->devfn), PCI_FUNC(payload->devfn)); - return -ENOMEM; - } - - INIT_WORK(&hp_work->work, mk_device_add_work_fn); - hp_work->domain = payload->domain; - hp_work->bus = payload->bus; - hp_work->devfn = payload->devfn; - hp_work->flags = payload->flags; - strscpy(hp_work->driver_override, payload->driver_override, sizeof(hp_work->driver_override)); - hp_work->sender_instance_id = payload->sender_instance_id; - hp_work->operation = MK_RES_DEVICE_ADD; - hp_work->vendor = payload->vendor; - hp_work->device = payload->device; - hp_work->bus_desc = payload->bus_desc; - schedule_work(&hp_work->work); - - return 0; + return mk_queue_hotplug_request(MK_RES_DEVICE_ADD, msg_id, + payload, payload_len); } -static int mk_handle_device_remove(struct mk_device_resource_payload *payload, u32 payload_len) +static int mk_handle_device_remove(struct mk_device_resource_payload *payload, u32 payload_len, + u64 msg_id) { - struct mk_device_hotplug_work *hp_work; + return mk_queue_hotplug_request(MK_RES_DEVICE_REMOVE, msg_id, + payload, payload_len); +} - if (payload_len < sizeof(*payload)) { - pr_err("Multikernel hotplug: Invalid device remove payload size: %u\n", payload_len); - return -EINVAL; - } +static void mk_hotplug_work_fn(struct work_struct *work) +{ + struct mk_hotplug_work *request = container_of(work, struct mk_hotplug_work, work); - hp_work = kmalloc(sizeof(*hp_work), GFP_ATOMIC); - if (!hp_work) { - pr_err("Multikernel hotplug: Failed to allocate work structure for device %04x:%02x:%02x.%x\n", - payload->domain, payload->bus, - PCI_SLOT(payload->devfn), PCI_FUNC(payload->devfn)); - return -ENOMEM; + switch (request->operation) { + case MK_RES_CPU_ADD: + mk_cpu_add_work_fn(work); + break; + case MK_RES_CPU_REMOVE: + mk_cpu_remove_work_fn(work); + break; + case MK_RES_MEM_ADD: + mk_mem_add_work_fn(work); + break; + case MK_RES_MEM_REMOVE: + mk_mem_remove_work_fn(work); + break; + case MK_RES_DEVICE_ADD: + mk_device_add_work_fn(work); + break; + case MK_RES_DEVICE_REMOVE: + mk_device_remove_work_fn(work); + break; } - - INIT_WORK(&hp_work->work, mk_device_remove_work_fn); - hp_work->domain = payload->domain; - hp_work->bus = payload->bus; - hp_work->devfn = payload->devfn; - hp_work->flags = payload->flags; - hp_work->sender_instance_id = payload->sender_instance_id; - hp_work->operation = MK_RES_DEVICE_REMOVE; - schedule_work(&hp_work->work); - - return 0; } /* @@ -990,6 +1032,7 @@ static int mk_handle_device_remove(struct mk_device_resource_payload *payload, u * mk_resource_msg_handler - Handle resource management messages * @msg_type: Message type (should be MK_MSG_RESOURCE) * @subtype: Message subtype (CPU_ADD, CPU_REMOVE, MEM_ADD, MEM_REMOVE, DEVICE_ADD, DEVICE_REMOVE) + * @msg_id: Request ID carried through deferred work and replies * @payload: Payload data * @payload_len: Payload length * @ctx: Context (unused) @@ -998,9 +1041,15 @@ static int mk_handle_device_remove(struct mk_device_resource_payload *payload, u * messaging system. It dispatches to specific handlers based on the * message subtype. */ -static void mk_resource_msg_handler(u32 msg_type, u32 subtype, +static void mk_resource_msg_handler(u32 msg_type, u32 subtype, u64 msg_id, void *payload, u32 payload_len, void *ctx) { + union { + struct mk_cpu_resource_payload cpu; + struct mk_mem_resource_payload mem; + struct mk_device_resource_payload device; + struct mk_resource_ack ack; + } decoded; int ret = 0; if (msg_type != MK_MSG_RESOURCE) { @@ -1008,29 +1057,35 @@ static void mk_resource_msg_handler(u32 msg_type, u32 subtype, return; } + /* Wire payloads need not be aligned for their typed fields. */ + if (payload_len) { + memcpy(&decoded, payload, min_t(size_t, payload_len, sizeof(decoded))); + payload = &decoded; + } + switch (subtype) { case MK_RES_CPU_ADD: - ret = mk_handle_cpu_add((struct mk_cpu_resource_payload *)payload, payload_len); + ret = mk_handle_cpu_add(payload, payload_len, msg_id); break; case MK_RES_CPU_REMOVE: - ret = mk_handle_cpu_remove((struct mk_cpu_resource_payload *)payload, payload_len); + ret = mk_handle_cpu_remove(payload, payload_len, msg_id); break; case MK_RES_MEM_ADD: - ret = mk_handle_mem_add((struct mk_mem_resource_payload *)payload, payload_len); + ret = mk_handle_mem_add(payload, payload_len, msg_id); break; case MK_RES_MEM_REMOVE: - ret = mk_handle_mem_remove((struct mk_mem_resource_payload *)payload, payload_len); + ret = mk_handle_mem_remove(payload, payload_len, msg_id); break; case MK_RES_DEVICE_ADD: - ret = mk_handle_device_add((struct mk_device_resource_payload *)payload, payload_len); + ret = mk_handle_device_add(payload, payload_len, msg_id); break; case MK_RES_DEVICE_REMOVE: - ret = mk_handle_device_remove((struct mk_device_resource_payload *)payload, payload_len); + ret = mk_handle_device_remove(payload, payload_len, msg_id); break; case MK_RES_ACK: @@ -1040,7 +1095,7 @@ static void mk_resource_msg_handler(u32 msg_type, u32 subtype, pr_info("Multikernel hotplug: ACK op=0x%x resource=%llu result=%d\n", ack->operation, ack->resource_id, (int)ack->result); mk_msg_pending_complete(MK_MSG_RESOURCE, ack->operation, - ack->resource_id, ack->result); + ack->resource_id, msg_id, ack->result); } else { pr_warn("Multikernel hotplug: Invalid ACK payload size: %u\n", payload_len); } @@ -1072,14 +1127,32 @@ static void mk_resource_msg_handler(u32 msg_type, u32 subtype, */ int __init mk_hotplug_init(void) { - int ret; + struct workqueue_struct *wq; + unsigned long flags; + int i, ret; pr_info("Initializing multikernel hotplug subsystem\n"); + wq = alloc_ordered_workqueue("mk_cpu_hotplug", WQ_MEM_RECLAIM); + if (!wq) + return -ENOMEM; + + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + INIT_WORK(&mk_cpu_requests[i].work, mk_hotplug_work_fn); + INIT_WORK(&mk_other_requests[i].work, mk_hotplug_work_fn); + } + + raw_spin_lock_irqsave(&mk_cpu_hotplug_lock, flags); + mk_cpu_hotplug_wq = wq; + mk_hotplug_accepting = true; + raw_spin_unlock_irqrestore(&mk_cpu_hotplug_lock, flags); + /* Register handler for resource management messages */ ret = mk_register_msg_handler(MK_MSG_RESOURCE, mk_resource_msg_handler, NULL); if (ret < 0) { pr_err("Failed to register multikernel hotplug message handler: %d\n", ret); + mk_hotplug_quiesce(); + destroy_workqueue(mk_cpu_hotplug_detach()); return ret; } @@ -1094,13 +1167,21 @@ int __init mk_hotplug_init(void) */ void mk_hotplug_cleanup(void) { + struct workqueue_struct *wq; struct mk_hotplug_op *op, *tmp; pr_info("Cleaning up multikernel hotplug subsystem\n"); + /* Close all resource admission and finish workers before freeing tracking. */ + mk_hotplug_quiesce(); + wq = mk_cpu_hotplug_detach(); + /* Unregister message handler */ mk_unregister_msg_handler(MK_MSG_RESOURCE, mk_resource_msg_handler); + if (wq) + destroy_workqueue(wq); + /* Free operation tracking list */ mutex_lock(&mk_hotplug_mutex); list_for_each_entry_safe(op, tmp, &mk_hotplug_ops, list) { @@ -1141,7 +1222,10 @@ int mk_pool_cpu_add(mk_phys_cpu_t cpu_id) * below must not be able to fail: grow it while the CPU is still * ours to keep. */ - ret = mk_cpu_set_reserve(mk_pool->cpus, 1); + { + guard(mutex)(&mk_pool->cpus_lock); + ret = mk_cpu_set_reserve(mk_pool->cpus, 1); + } if (ret) return ret; @@ -1150,7 +1234,10 @@ int mk_pool_cpu_add(mk_phys_cpu_t cpu_id) return ret; /* The CPU parked in the pool park area and is assignable again */ - mk_cpu_set_add(mk_pool->cpus, cpu_id); + { + guard(mutex)(&mk_pool->cpus_lock); + mk_cpu_set_add(mk_pool->cpus, cpu_id); + } return 0; } @@ -1178,7 +1265,10 @@ int mk_pool_cpu_remove(mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) return ret; /* From here on it is an ordinary CPU of this kernel */ - mk_cpu_set_del(mk_pool->cpus, cpu_id); + { + guard(mutex)(&mk_pool->cpus_lock); + mk_cpu_set_del(mk_pool->cpus, cpu_id); + } return 0; } @@ -1196,6 +1286,9 @@ int mk_pool_device_add(u16 domain, u8 bus, u8 devfn, const char *alias) { int ret; + if (mk_pci_device_assigned(domain, bus, devfn)) + return -EBUSY; + ret = mk_do_device_remove(domain, bus, devfn); if (ret) return ret; @@ -1271,6 +1364,8 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) struct mk_instance *target_instance; int ret; + lockdep_assert_held(&mk_overlay_mutex); + /* For self-removal, execute directly (we're in process context) */ if (instance_id == mk_self->id) { if (mk_pool) @@ -1283,6 +1378,11 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) return -ENODEV; /* For non-running instances, return CPU to root using existing API */ + if (target_instance->resource_uncertain || + target_instance->state == MK_STATE_FAILED) { + ret = -EBUSY; + goto out; + } if (target_instance->state != MK_STATE_ACTIVE) { struct mk_cpu_set cpus = { .nr = 1, .cap = 1, .ids = &cpu_id }; @@ -1311,20 +1411,23 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) goto out; } - ret = mk_cpu_set_reserve(mk_pool->cpus, 1); + { + guard(mutex)(&mk_pool->cpus_lock); + ret = mk_cpu_set_reserve(mk_pool->cpus, 1); + } if (ret) goto out; pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, cpu_id); - if (!pending) { - ret = -ENOMEM; + if (IS_ERR(pending)) { + ret = PTR_ERR(pending); goto out; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, - &payload, sizeof(payload)); + ret = mk_send_message_id(instance_id, MK_MSG_RESOURCE, MK_RES_CPU_REMOVE, + mk_msg_pending_id(pending), &payload, sizeof(payload)); if (ret < 0) { - mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ + mk_msg_pending_cancel(pending); goto out; } @@ -1348,7 +1451,11 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) } mk_cpu_set_del(target_instance->cpus, cpu_id); - mk_cpu_set_add(mk_pool->cpus, cpu_id); + { + guard(mutex)(&mk_pool->cpus_lock); + mk_cpu_set_add(mk_pool->cpus, cpu_id); + } + mk_instance_ipi_retarget(target_instance); ret = 0; out: @@ -1372,6 +1479,7 @@ int mk_send_cpu_remove(int instance_id, mk_phys_cpu_t cpu_id) */ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 flags) { + struct mk_cpu_set cpus = { .nr = 1, .cap = 1, .ids = &cpu_id }; struct mk_cpu_resource_payload payload = { .cpu_id = cpu_id, .numa_node = numa_node, @@ -1382,6 +1490,8 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl struct mk_instance *target_instance; int ret; + lockdep_assert_held(&mk_overlay_mutex); + /* For self-addition, execute directly (we're in process context) */ if (instance_id == mk_self->id) { if (mk_pool) @@ -1397,9 +1507,12 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl } /* For non-running instances, transfer CPU from root using existing API */ + if (target_instance->resource_uncertain || + target_instance->state == MK_STATE_FAILED) { + ret = -EBUSY; + goto out; + } if (target_instance->state != MK_STATE_ACTIVE) { - struct mk_cpu_set cpus = { .nr = 1, .cap = 1, .ids = &cpu_id }; - ret = mk_instance_transfer_cpus(target_instance, &cpus); goto out; } @@ -1408,13 +1521,31 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl * Only a CPU from the assignable pool is parked on the host slot; * publishing a wakeup for any other CPU can only time out. */ - if (!mk_cpu_set_contains(mk_pool->cpus, cpu_id)) { + if (!mk_pool || !mk_cpu_set_contains(mk_pool->cpus, cpu_id)) { pr_err("Multikernel hotplug: CPU %llu is not in this kernel's pool\n", cpu_id); ret = -EBUSY; goto out; } + pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_ADD, cpu_id); + if (IS_ERR(pending)) { + ret = PTR_ERR(pending); + goto out; + } + + /* + * Reserve ownership before any handoff. Once a wakeup is published, + * an error or timeout cannot prove the CPU did not consume it. Keep + * the CPU assigned to the target, even on failure, until a separate + * quiesced removal can safely return it to the pool. + */ + ret = mk_instance_transfer_cpus(target_instance, &cpus); + if (ret) { + mk_msg_pending_cancel(pending); + goto out; + } + /* * The CPU is parked on the host slot, where the spawn kernel's * secondary wakeup cannot reach it. Point it at the instance's @@ -1424,42 +1555,19 @@ int mk_send_cpu_add(int instance_id, mk_phys_cpu_t cpu_id, u32 numa_node, u32 fl if (ret < 0) { pr_err("Multikernel hotplug: Failed to repark CPU %llu to instance %d: %d\n", cpu_id, instance_id, ret); + mk_msg_pending_cancel(pending); goto out; } - pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_ADD, cpu_id); - if (!pending) { - mk_repark_cpu_to_host(target_instance, cpu_id); - ret = -ENOMEM; - goto out; - } - - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_CPU_ADD, - &payload, sizeof(payload)); + ret = mk_send_message_id(instance_id, MK_MSG_RESOURCE, MK_RES_CPU_ADD, + mk_msg_pending_id(pending), &payload, sizeof(payload)); if (ret < 0) { - mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ - mk_repark_cpu_to_host(target_instance, cpu_id); + mk_msg_pending_cancel(pending); goto out; } + /* A timed-out waiter does not cancel the remote CPU-start work. */ ret = mk_msg_pending_wait(pending, 10000); - if (ret < 0) { - /* - * Best effort: if the instance never picked the CPU up, it - * is still parked on the instance context and comes home; - * if the instance onlined it despite the error, nothing is - * watching the context and this times out harmlessly. - */ - mk_repark_cpu_to_host(target_instance, cpu_id); - goto out; - } - - if (mk_cpu_set_add(target_instance->cpus, cpu_id)) - pr_warn("Multikernel hotplug: Failed to track CPU %llu in instance %d\n", - cpu_id, instance_id); - mk_cpu_set_del(mk_pool->cpus, cpu_id); - - ret = 0; out: mk_instance_put(target_instance); return ret; @@ -1493,7 +1601,9 @@ int mk_send_mem_add(int instance_id, u64 start_pfn, u64 nr_pages, }; struct mk_pending_msg *pending; struct mk_instance *target_instance; - int ret; + int ret, cleanup_ret; + + lockdep_assert_held(&mk_overlay_mutex); /* For self-addition, execute directly (we're in process context) */ if (instance_id == mk_self->id) @@ -1503,32 +1613,42 @@ int mk_send_mem_add(int instance_id, u64 start_pfn, u64 nr_pages, if (!target_instance) return -ENODEV; - /* Before the running kernel takes the memory on, not after */ + if (target_instance->resource_uncertain || + target_instance->state == MK_STATE_FAILED) { + ret = -EBUSY; + goto out; + } + /* Reserve the exact range before the running kernel can take it. */ ret = mk_instance_add_memory_region(target_instance, PFN_PHYS(start_pfn), PFN_PHYS(nr_pages)); if (ret || target_instance->state != MK_STATE_ACTIVE) goto out; pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_MEM_ADD, start_pfn); - if (!pending) { - ret = -ENOMEM; + if (IS_ERR(pending)) { + ret = PTR_ERR(pending); goto remove; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_MEM_ADD, - &payload, sizeof(payload)); + ret = mk_send_message_id(instance_id, MK_MSG_RESOURCE, MK_RES_MEM_ADD, + mk_msg_pending_id(pending), &payload, sizeof(payload)); if (ret < 0) { - mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ + mk_msg_pending_cancel(pending); goto remove; } ret = mk_msg_pending_wait(pending, 10000); - /* Without an answer the kernel may still add it, so it stays its own */ - if (!ret || ret == -ETIMEDOUT) - goto out; + /* Even a negative ACK may follow a failed, partial rollback. */ + if (ret) + target_instance->resource_uncertain = true; + goto out; remove: - mk_instance_remove_memory_region(target_instance, PFN_PHYS(start_pfn), - PFN_PHYS(nr_pages)); + cleanup_ret = mk_instance_remove_memory_region(target_instance, PFN_PHYS(start_pfn), + PFN_PHYS(nr_pages)); + if (cleanup_ret) { + target_instance->resource_uncertain = true; + ret = cleanup_ret; + } out: mk_instance_put(target_instance); return ret; @@ -1560,6 +1680,8 @@ int mk_send_mem_remove(int instance_id, u64 start_pfn, u64 nr_pages) struct mk_instance *target_instance; int ret; + lockdep_assert_held(&mk_overlay_mutex); + /* For self-removal, execute directly (we're in process context) */ if (instance_id == mk_self->id) return mk_do_mem_remove(start_pfn, nr_pages); @@ -1569,6 +1691,11 @@ int mk_send_mem_remove(int instance_id, u64 start_pfn, u64 nr_pages) return -ENODEV; /* For non-running instances, just remove the memory region from the instance */ + if (target_instance->resource_uncertain || + target_instance->state == MK_STATE_FAILED) { + ret = -EBUSY; + goto out; + } if (target_instance->state != MK_STATE_ACTIVE) { phys_addr_t phys_addr; size_t size; @@ -1586,21 +1713,24 @@ int mk_send_mem_remove(int instance_id, u64 start_pfn, u64 nr_pages) goto out; pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_MEM_REMOVE, start_pfn); - if (!pending) { - ret = -ENOMEM; + if (IS_ERR(pending)) { + ret = PTR_ERR(pending); goto abort; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_MEM_REMOVE, - &payload, sizeof(payload)); + ret = mk_send_message_id(instance_id, MK_MSG_RESOURCE, MK_RES_MEM_REMOVE, + mk_msg_pending_id(pending), &payload, sizeof(payload)); if (ret < 0) { - mk_msg_pending_wait(pending, 0); /* Immediate cleanup */ + mk_msg_pending_cancel(pending); goto abort; } ret = mk_msg_pending_wait(pending, 10000); - if (ret < 0) - goto abort; + if (ret) { + /* A late removal must not race a new allocation from this range. */ + target_instance->resource_uncertain = true; + goto out; + } mk_instance_finish_memory_removal(target_instance, PFN_PHYS(start_pfn), PFN_PHYS(nr_pages)); @@ -1642,15 +1772,17 @@ int mk_send_device_add(int instance_id, u16 domain, u8 bus, u8 devfn, struct mk_pending_msg *pending; struct mk_instance *target_instance; struct pci_dev *pdev; - int ret; + int ret, cleanup_ret; u32 resource_id; + lockdep_assert_held(&mk_overlay_mutex); + if (driver_override) strscpy(payload.driver_override, driver_override, sizeof(payload.driver_override)); else payload.driver_override[0] = '\0'; - resource_id = (domain << 16) | (bus << 8) | devfn; + resource_id = ((u32)domain << 16) | (bus << 8) | devfn; if (instance_id == mk_self->id) { if (mk_pool) @@ -1668,15 +1800,20 @@ int mk_send_device_add(int instance_id, u16 domain, u8 bus, u8 devfn, if (!target_instance) return -ENODEV; - if (target_instance->state != MK_STATE_ACTIVE) { - ret = mk_instance_add_pci_device(target_instance, domain, bus, devfn); + if (target_instance->resource_uncertain || + target_instance->state == MK_STATE_FAILED) { + ret = -EBUSY; goto out; } + /* Reserve ownership before config access or a probe can be admitted. */ + ret = mk_instance_add_pci_device(target_instance, domain, bus, devfn); + if (ret || target_instance->state != MK_STATE_ACTIVE) + goto out; /* The instance probes the function as soon as it hears of it */ ret = mk_pciback_lend_one(target_instance, domain, bus, devfn); if (ret) - goto out; + goto remove; /* It may not have the function in its tree, nor even its bus */ pdev = pci_get_domain_bus_and_slot(domain, bus, devfn); @@ -1690,35 +1827,37 @@ int mk_send_device_add(int instance_id, u16 domain, u8 bus, u8 devfn, pci_dev_put(pdev); pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_DEVICE_ADD, resource_id); - if (!pending) { - ret = -ENOMEM; + if (IS_ERR(pending)) { + ret = PTR_ERR(pending); goto unlend; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_DEVICE_ADD, - &payload, sizeof(payload)); + ret = mk_send_message_id(instance_id, MK_MSG_RESOURCE, MK_RES_DEVICE_ADD, + mk_msg_pending_id(pending), &payload, sizeof(payload)); if (ret < 0) { - mk_msg_pending_wait(pending, 0); + mk_msg_pending_cancel(pending); goto unlend; } ret = mk_msg_pending_wait(pending, 10000); - if (ret < 0) - goto unlend; - - ret = mk_instance_add_pci_device(target_instance, domain, bus, devfn); - if (ret < 0) { - pr_warn("Device added to target but failed to update tracking: %d\n", ret); + if (ret) { + target_instance->resource_uncertain = true; + goto out; } pr_info("Multikernel hotplug: Device %04x:%02x:%02x.%x successfully added to instance %d\n", domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), instance_id); - mk_instance_put(target_instance); - return 0; + goto out; unlend: mk_pciback_unlend_one(target_instance, domain, bus, devfn); +remove: + cleanup_ret = mk_instance_remove_pci_device(target_instance, domain, bus, devfn); + if (cleanup_ret) { + target_instance->resource_uncertain = true; + ret = cleanup_ret; + } out: mk_instance_put(target_instance); return ret; @@ -1752,8 +1891,10 @@ int mk_send_device_remove(int instance_id, u16 domain, u8 bus, u8 devfn) int ret; u32 resource_id; + lockdep_assert_held(&mk_overlay_mutex); + payload.driver_override[0] = '\0'; - resource_id = (domain << 16) | (bus << 8) | devfn; + resource_id = ((u32)domain << 16) | (bus << 8) | devfn; if (instance_id == mk_self->id) { if (mk_pool) @@ -1765,33 +1906,42 @@ int mk_send_device_remove(int instance_id, u16 domain, u8 bus, u8 devfn) if (!target_instance) return -ENODEV; + if (target_instance->resource_uncertain || + target_instance->state == MK_STATE_FAILED) { + ret = -EBUSY; + goto out; + } if (target_instance->state != MK_STATE_ACTIVE) { ret = mk_instance_remove_pci_device(target_instance, domain, bus, devfn); goto out; } pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_DEVICE_REMOVE, resource_id); - if (!pending) { - ret = -ENOMEM; + if (IS_ERR(pending)) { + ret = PTR_ERR(pending); goto out; } - ret = mk_send_message(instance_id, MK_MSG_RESOURCE, MK_RES_DEVICE_REMOVE, - &payload, sizeof(payload)); + ret = mk_send_message_id(instance_id, MK_MSG_RESOURCE, MK_RES_DEVICE_REMOVE, + mk_msg_pending_id(pending), &payload, sizeof(payload)); if (ret < 0) { - mk_msg_pending_wait(pending, 0); + mk_msg_pending_cancel(pending); goto out; } ret = mk_msg_pending_wait(pending, 10000); - if (ret < 0) + if (ret) { + target_instance->resource_uncertain = true; goto out; + } mk_pciback_unlend_one(target_instance, domain, bus, devfn); ret = mk_instance_remove_pci_device(target_instance, domain, bus, devfn); if (ret < 0) { pr_warn("Device removed from target but failed to update tracking: %d\n", ret); + target_instance->resource_uncertain = true; + goto out; } pr_info("Multikernel hotplug: Device %04x:%02x:%02x.%x successfully removed from instance %d\n", diff --git a/kernel/multikernel/internal.h b/kernel/multikernel/internal.h index 797cb2da2a1910..23f5395d0cf52b 100644 --- a/kernel/multikernel/internal.h +++ b/kernel/multikernel/internal.h @@ -14,6 +14,10 @@ extern struct list_head mk_instance_list; /* core.c */ struct mk_instance *mk_instance_alloc(int id, const char *name); int mk_instance_publish(struct mk_instance *instance); +int mk_instance_idr_add(struct mk_instance *instance, int start, int end); +void mk_instance_idr_remove(int id); +int mk_instance_prepare_destroy(struct mk_instance *instance); +int mk_instance_retire_resources(struct mk_instance *instance); void mk_instance_free(struct mk_instance *instance); /* kernfs.c */ @@ -37,18 +41,20 @@ int mk_dt_generate_instance_dtb(struct mk_instance *instance, /* overlay.c */ extern struct kernfs_node *mk_overlay_root_kn; -extern struct mutex mk_overlay_mutex; int mk_overlay_init(void); -void mk_overlay_exit(void); int mk_overlay_rmdir(struct kernfs_node *kn); /* ipi.c */ int mk_arm_force_halt(struct mk_instance *instance); +int mk_send_message_ipi(int instance_id, struct mk_instance *instance, + const struct mk_message *message, const void *payload); /* hotplug.c */ int mk_hotplug_init(void); +void mk_hotplug_quiesce(void); void mk_hotplug_cleanup(void); -int mk_handle_cpu_remove(struct mk_cpu_resource_payload *payload, u32 payload_len); +int mk_handle_cpu_remove(struct mk_cpu_resource_payload *payload, u32 payload_len, + u64 msg_id); /* * Move primitives between this kernel and the pool it manages. Valid @@ -62,6 +68,7 @@ int mk_pool_device_remove(u16 domain, u8 bus, u8 devfn, /* Root pool bookkeeping for PCI devices, see core.c */ bool mk_root_has_pci_device(u16 domain, u8 bus, u8 devfn); +bool mk_pci_device_assigned(u16 domain, u8 bus, u8 devfn); int mk_root_add_pci_device(u16 domain, u8 bus, u8 devfn, const char *alias); int mk_root_del_pci_device(u16 domain, u8 bus, u8 devfn); diff --git a/kernel/multikernel/ipi.c b/kernel/multikernel/ipi.c index 4a6a44f94c88dc..4c4535febfaaa1 100644 --- a/kernel/multikernel/ipi.c +++ b/kernel/multikernel/ipi.c @@ -12,16 +12,17 @@ #include #include #include +#include #include "internal.h" /* Callback management */ -static struct mk_ipi_handler *mk_handlers; +static struct mk_ipi_handler __rcu *mk_handlers; static raw_spinlock_t mk_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_handlers_lock); static DEFINE_RAW_SPINLOCK(mk_drain_lock); static DEFINE_PER_CPU(bool, mk_draining); -static void mk_ipi_drain_ring(void); +static int mk_ipi_drain_ring(void); /* * Ring indices live in memory another kernel instance can write, so every @@ -33,33 +34,6 @@ static inline unsigned int mk_ring_idx(unsigned int i) return i & (MK_IPI_RING_SIZE - 1); } -/** - * mk_ipi_ring_drop_pending - Discard everything queued in this kernel's ring - * - * Called when an instance is re-spawned. A halting instance parks its CPUs - * wherever they were, including between claiming a ring slot and publishing - * it, and the drain stops at such a slot forever. Anything still queued was - * sent by a kernel that is gone, so drop it all rather than let one - * abandoned slot wedge the ring. - */ -void mk_ipi_ring_drop_pending(void) -{ - struct mk_ipi_ring *ring; - unsigned int head, tail; - - if (!mk_self || !mk_self->ipi_data) - return; - - ring = &mk_self->ipi_data->ring; - head = mk_ring_idx(atomic_read(&ring->head)); - - for (tail = mk_ring_idx(atomic_read(&ring->tail)); tail != head; - tail = mk_ring_idx(tail + 1)) - ring->entries[tail].data_size = 0; - - atomic_set(&ring->tail, head); -} - /** * multikernel_register_handler - Register a callback for multikernel IPI * @callback: Function to call when IPI is received @@ -85,8 +59,10 @@ struct mk_ipi_handler *multikernel_register_handler(mk_ipi_callback_t callback, handler->ipi_type = ipi_type; raw_spin_lock_irqsave(&mk_handlers_lock, flags); - handler->next = mk_handlers; - mk_handlers = handler; + RCU_INIT_POINTER(handler->next, + rcu_dereference_protected(mk_handlers, + lockdep_is_held(&mk_handlers_lock))); + rcu_assign_pointer(mk_handlers, handler); raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); return handler; @@ -96,10 +72,15 @@ EXPORT_SYMBOL(multikernel_register_handler); /** * multikernel_unregister_handler - Unregister a multikernel IPI callback * @handler: Handler pointer returned from multikernel_register_handler + * + * Process context only; no callback-owned locks may be held. Removal excludes + * new lookups; the grace period drains old readers and their callbacks before + * context release. Old readers may still invoke the callback during the wait. */ void multikernel_unregister_handler(struct mk_ipi_handler *handler) { - struct mk_ipi_handler **pp, *p; + struct mk_ipi_handler __rcu **pp; + struct mk_ipi_handler *p; unsigned long flags; if (!handler) @@ -107,19 +88,69 @@ void multikernel_unregister_handler(struct mk_ipi_handler *handler) raw_spin_lock_irqsave(&mk_handlers_lock, flags); pp = &mk_handlers; - while ((p = *pp) != NULL) { + while ((p = rcu_dereference_protected(*pp, + lockdep_is_held(&mk_handlers_lock))) != NULL) { if (p == handler) { - *pp = p->next; + rcu_assign_pointer(*pp, + rcu_dereference_protected(p->next, + lockdep_is_held(&mk_handlers_lock))); break; } pp = &p->next; } raw_spin_unlock_irqrestore(&mk_handlers_lock, flags); - kfree(p); + if (p) { + synchronize_rcu(); + kfree(p); + } } EXPORT_SYMBOL(multikernel_unregister_handler); +void mk_instance_ipi_deactivate(struct mk_instance *instance) +{ + unsigned long flags; + + lockdep_assert_held(&mk_overlay_mutex); + /* Acquiring the lock drains every already-entered ring producer. */ + raw_spin_lock_irqsave(&instance->ipi_lock, flags); + instance->ipi_data = NULL; + raw_spin_unlock_irqrestore(&instance->ipi_lock, flags); +} + +void mk_instance_ipi_publish(struct mk_instance *instance) +{ + unsigned long flags; + + lockdep_assert_held(&mk_overlay_mutex); + raw_spin_lock_irqsave(&instance->ipi_lock, flags); + instance->ipi_target = mk_cpu_set_first(instance->cpus); + instance->ipi_data = phys_to_virt(instance->ipi_phys); + raw_spin_unlock_irqrestore(&instance->ipi_lock, flags); +} + +void mk_instance_ipi_reset(struct mk_instance *instance) +{ + lockdep_assert_held(&mk_overlay_mutex); + mk_instance_ipi_deactivate(instance); + /* No producer can enter while closed; do the large clear with IRQs on. */ + memset(phys_to_virt(instance->ipi_phys), 0, sizeof(struct mk_shared_data)); + mk_instance_ipi_publish(instance); +} + +void mk_instance_ipi_retarget(struct mk_instance *instance) +{ + unsigned long flags; + + lockdep_assert_held(&mk_overlay_mutex); + /* The restored parent has a firmware-described, explicit doorbell CPU. */ + if (instance == host_instance) + return; + raw_spin_lock_irqsave(&instance->ipi_lock, flags); + instance->ipi_target = mk_cpu_set_first(instance->cpus); + raw_spin_unlock_irqrestore(&instance->ipi_lock, flags); +} + /** * mk_arm_force_halt - Post the force-halt marker for an instance * @instance: Instance about to be NMIed @@ -134,15 +165,21 @@ EXPORT_SYMBOL(multikernel_unregister_handler); */ int mk_arm_force_halt(struct mk_instance *instance) { - struct mk_shared_data *ipi_data = READ_ONCE(instance->ipi_data); - - if (!ipi_data) - return -ENODEV; + struct mk_shared_data *ipi_data; + unsigned long flags; + int ret = -ENODEV; + + raw_spin_lock_irqsave(&instance->ipi_lock, flags); + ipi_data = instance->ipi_data; + if (ipi_data) { + WRITE_ONCE(ipi_data->force_halt, 1); + /* The marker must be visible before the NMIs that test it. */ + smp_wmb(); + ret = 0; + } - WRITE_ONCE(ipi_data->force_halt, 1); - /* The marker must be visible before the NMIs that test it */ - smp_wmb(); - return 0; + raw_spin_unlock_irqrestore(&instance->ipi_lock, flags); + return ret; } /** @@ -170,46 +207,49 @@ int multikernel_send_ipi_data(int instance_id, void *data, size_t data_size, uns return ret; } -/* - * The same for a caller that holds a reference to @instance. Looking an - * instance up by ID takes a mutex; this does not sleep, so it is the one - * to use with interrupts off. - */ -int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, - size_t data_size, unsigned long type) +/* All producers retain the instance and publish under the same ring lock. */ +static int mk_send_ipi_data(struct mk_instance *instance, const void *data, + size_t data_size, unsigned long type, + const struct mk_message *message) { struct mk_shared_data *ipi_data; struct mk_ipi_data *slot; int instance_id = instance->id; unsigned int head, next_head, tail; mk_phys_cpu_t target; + unsigned long flags; + size_t header_size = message ? sizeof(*message) : 0; + size_t offset = 0; + int ret; - if (data_size > MK_MAX_DATA_SIZE) + if (data_size > MK_MAX_DATA_SIZE - header_size) return -EINVAL; + raw_spin_lock_irqsave(&instance->ipi_lock, flags); target = instance->ipi_target; - if (target == MK_PHYS_CPU_INVALID) - target = mk_cpu_set_first(instance->cpus); if (target == MK_PHYS_CPU_INVALID) { - pr_err("Instance %d has no CPU to receive the IPI\n", instance_id); - return -ENODEV; + ret = -ENODEV; + goto unlock; } /* The ring lives as long as the instance, which is pinned here */ ipi_data = READ_ONCE(instance->ipi_data); if (!ipi_data) { - pr_err("Multikernel IPI buffer not available for instance %d\n", instance_id); - return -ENODEV; + ret = -ENODEV; + goto unlock; } /* Try to enqueue the message in the ring buffer */ do { head = mk_ring_idx(atomic_read(&ipi_data->ring.head)); next_head = mk_ring_idx(head + 1); - tail = mk_ring_idx(atomic_read(&ipi_data->ring.tail)); + /* Pair with the consumer's release of cleared slot capacity. */ + tail = mk_ring_idx(atomic_read_acquire(&ipi_data->ring.tail)); /* Check if ring buffer is full */ if (next_head == tail) { + ret = -ENOSPC; + raw_spin_unlock_irqrestore(&instance->ipi_lock, flags); /* * Console output reaches this path, so a plain printk * here re-enters the console write that called us and @@ -230,38 +270,77 @@ int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, slot->sender_cpu = arch_cpu_physical_id(smp_processor_id()); slot->type = type; + if (message) { + /* The flexible payload can start before sizeof(*message). */ + offset = offsetof(struct mk_message, payload); + memset(slot->buffer, 0, header_size + data_size); + memcpy(slot->buffer, message, offset); + } if (data && data_size > 0) - memcpy(slot->buffer, data, data_size); + memcpy(slot->buffer + offset, data, data_size); /* * data_size publishes the slot: the reader treats a zero as "the * producer has claimed this slot but has not filled it yet" and * waits. Claiming the slot advanced head, so a reader can already - * be looking at it; everything above must be visible first. + * be looking at it; the header and payload must be visible first. */ - smp_store_release(&slot->data_size, data_size); + smp_store_release(&slot->data_size, header_size + data_size); + raw_spin_unlock_irqrestore(&instance->ipi_lock, flags); mk_arch_send_ipi(target); return 0; + +unlock: + raw_spin_unlock_irqrestore(&instance->ipi_lock, flags); + return ret; } -static void mk_ipi_drain_ring(void) +/* + * The same for a caller that already holds a reference to @instance. + * Both send routes serialize against ring reset without sleeping. + */ +int multikernel_send_ipi_data_to(struct mk_instance *instance, void *data, + size_t data_size, unsigned long type) +{ + return mk_send_ipi_data(instance, data, data_size, type, NULL); +} + +int mk_send_message_ipi(int instance_id, struct mk_instance *instance, + const struct mk_message *message, const void *payload) +{ + bool lookup = !instance; + int ret; + + if (lookup) + instance = mk_instance_find(instance_id); + if (!instance) + return -EINVAL; + + ret = mk_send_ipi_data(instance, payload, message->payload_len, + message->msg_type, message); + if (lookup) + mk_instance_put(instance); + return ret; +} + +static int mk_ipi_drain_ring(void) { struct mk_ipi_data *slot; struct mk_ipi_handler *handler; unsigned int head, tail, next_tail; size_t data_size; - int messages_processed = 0; if (!mk_self || !mk_self->ipi_data) - return; + return 0; + /* New peer traffic must not extend this bounded retirement snapshot. */ + head = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.head)); while (1) { tail = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.tail)); - head = mk_ring_idx(atomic_read(&mk_self->ipi_data->ring.head)); if (tail == head) - break; + return 0; slot = &mk_self->ipi_data->ring.entries[tail]; @@ -272,46 +351,45 @@ static void mk_ipi_drain_ring(void) * drop the message it is about to publish. Its own IPI, or * the next one, brings us back here. * - * A sender stopped before publishing leaves its slot zero - * forever; mk_ipi_ring_drop_pending() clears those out when - * the instance is re-spawned. + * A stopped sender may leave its slot zero permanently. Without + * per-slot ownership or a quiescence protocol, respawning one peer + * cannot safely clear this shared receive ring: other peers may + * still publish or this receiver may advance tail concurrently. + * Retain the slot rather than fabricate reusable capacity. */ data_size = smp_load_acquire(&slot->data_size); if (data_size == 0) - break; + return -EBUSY; if (data_size > MK_MAX_DATA_SIZE) { pr_warn_once("Multikernel IPI slot %u has bad size %zu\n", tail, data_size); slot->data_size = 0; next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); + /* Do not let a producer reuse the slot before its clear. */ + atomic_set_release(&mk_self->ipi_data->ring.tail, next_tail); continue; } /* Dispatch to registered handler */ - raw_spin_lock(&mk_handlers_lock); - for (handler = mk_handlers; handler; handler = handler->next) { + rcu_read_lock(); + for (handler = rcu_dereference(mk_handlers); handler; + handler = rcu_dereference(handler->next)) { if (handler->ipi_type == slot->type && handler->callback) { mk_ipi_callback_t cb = handler->callback; void *ctx = handler->context; - raw_spin_unlock(&mk_handlers_lock); cb(slot, ctx); - goto advance_tail; + break; } } - raw_spin_unlock(&mk_handlers_lock); + rcu_read_unlock(); -advance_tail: /* Mark consumed so the slot reads as unpublished again */ slot->data_size = 0; next_tail = mk_ring_idx(tail + 1); - atomic_set(&mk_self->ipi_data->ring.tail, next_tail); - messages_processed++; - - if (messages_processed >= MK_IPI_RING_SIZE) - break; + /* Pair with producers' acquire: clear precedes all future reuse. */ + atomic_set_release(&mk_self->ipi_data->ring.tail, next_tail); } } @@ -321,12 +399,15 @@ static void mk_ipi_drain_ring(void) * This function is called when a multikernel IPI is received. * Messages are drained here, in interrupt context. */ -static void multikernel_interrupt_handler(void) +static int multikernel_interrupt_handler(void) { unsigned long flags; + int ret; if (!mk_self || !mk_self->ipi_data) - return; + return 0; + if (mk_ipi_in_drain()) + return -EBUSY; /* * Drain here rather than from irq_work. We are already in interrupt @@ -343,9 +424,10 @@ static void multikernel_interrupt_handler(void) */ raw_spin_lock_irqsave(&mk_drain_lock, flags); this_cpu_write(mk_draining, true); - mk_ipi_drain_ring(); + ret = mk_ipi_drain_ring(); this_cpu_write(mk_draining, false); raw_spin_unlock_irqrestore(&mk_drain_lock, flags); + return ret; } /** @@ -366,10 +448,18 @@ bool mk_ipi_in_drain(void) * cannot be taken: with interrupts off on the doorbell CPU, or while * stop_machine() holds every CPU. Message handlers run in the caller's * context, which they are fit for, as they normally run in the interrupt. + * + * After stopping a peer's CPUs, a successful snapshot drain also ensures its + * old requests have reached their handlers before that peer's grants are reused. + * Deferred handlers must separately pin identity and reject revoked work. + * + * Returns 0 if the captured prefix was consumed, -EBUSY if an unpublished slot + * blocked it or the caller is already dispatching a message. Other peers may + * still be writing; success is not global ring quiescence. */ -void mk_ipi_poll(void) +int mk_ipi_poll(void) { - multikernel_interrupt_handler(); + return multikernel_interrupt_handler(); } /** diff --git a/kernel/multikernel/kernfs.c b/kernel/multikernel/kernfs.c index c8fa6270e1fe6b..96684b63e25a2e 100644 --- a/kernel/multikernel/kernfs.c +++ b/kernel/multikernel/kernfs.c @@ -132,7 +132,11 @@ static int root_device_tree_seq_show(struct seq_file *sf, void *v) size_t dtb_size = 0; int ret; + /* Teardown holds the resource lock while draining kernfs callbacks. */ + if (!mutex_trylock(&mk_overlay_mutex)) + return -EAGAIN; ret = mk_dt_generate_instance_dtb(mk_self, &dtb_data, &dtb_size); + mutex_unlock(&mk_overlay_mutex); if (ret) { pr_err("Failed to generate baseline DTB: %d\n", ret); return ret; @@ -153,7 +157,11 @@ static ssize_t instance_device_tree_read(struct kernfs_open_file *of, size_t to_read; int ret; + /* Never wait on a controller that may be draining this callback. */ + if (!mutex_trylock(&mk_overlay_mutex)) + return -EAGAIN; ret = mk_dt_generate_instance_dtb(instance, &dtb_data, &dtb_size); + mutex_unlock(&mk_overlay_mutex); if (ret) { pr_err("Instance '%s': Failed to generate device tree: %d\n", instance->name, ret); @@ -277,9 +285,9 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, } if (id < 0) - allocated_id = idr_alloc(&mk_instance_idr, instance, 1, INT_MAX, GFP_KERNEL); + allocated_id = mk_instance_idr_add(instance, 1, INT_MAX); else - allocated_id = idr_alloc(&mk_instance_idr, instance, id, id + 1, GFP_KERNEL); + allocated_id = mk_instance_idr_add(instance, id, id + 1); if (allocated_id < 0) { pr_err("Failed to register instance '%s' in IDR: %d\n", name, allocated_id); @@ -310,15 +318,20 @@ int mk_create_instance_from_dtb(const char *name, int id, const void *fdt, return 0; err_free_resources: - mk_instance_free_memory(instance); err_free_idr: - idr_remove(&mk_instance_idr, instance->id); + mk_instance_idr_remove(instance->id); err_remove_dir: + /* Inactive files still must not be drained under the registry mutex. */ + mutex_unlock(&mk_instance_mutex); kernfs_remove(kn); + instance->kn = NULL; + ret = mk_instance_free_memory(instance) ?: ret; mk_instance_put(instance); + mutex_lock(&mk_instance_mutex); err_free_config: mk_dt_config_free(&config); - mk_instance_free(instance); + /* A lookup may already have pinned the early IDR publication. */ + mk_instance_put(instance); return ret; } @@ -401,39 +414,51 @@ static int mk_kernfs_rmdir(struct kernfs_node *kn) * (CPUs, memory, PCI devices) are automatically returned to the root * instance pool, making them available for future instance allocations. * - * Caller must hold mk_instance_mutex. + * Caller must hold mk_overlay_mutex and mk_instance_mutex. The registry mutex + * is temporarily dropped for stop/repark confirmation and kernfs draining, + * then restored before return. Resources are returned before this call exits; + * outstanding lookup references defer only the record/boot-page lifetime. * * Returns 0 on success, negative error code on failure. */ int mk_instance_destroy(struct mk_instance *instance) { + int ret; + lockdep_assert_held(&mk_instance_mutex); + lockdep_assert_held(&mk_overlay_mutex); if (!instance) { pr_err("NULL instance passed to mk_instance_destroy\n"); return -EINVAL; } - if (instance->state == MK_STATE_ACTIVE) { - pr_err("Cannot remove active instance '%s' (ID: %d). Instance must be stopped first.\n", + if (instance->state != MK_STATE_READY || instance->kimage) { + pr_err("Cannot remove instance '%s' (ID: %d) before confirmed stop and unload.\n", instance->name, instance->id); return -EBUSY; } - if (instance->state == MK_STATE_LOADED) { - pr_err("Cannot remove instance '%s' (ID: %d) with loaded kernel. Unload it first.\n", - instance->name, instance->id); - return -EBUSY; - } + /* The resource mutex pins membership while firmware may sleep. */ + mutex_unlock(&mk_instance_mutex); + ret = mk_instance_prepare_destroy(instance); + if (!ret) + ret = mk_instance_retire_resources(instance); + mutex_lock(&mk_instance_mutex); + if (ret) + return ret; list_del(&instance->list); - idr_remove(&mk_instance_idr, instance->id); + mk_instance_idr_remove(instance->id); + mutex_unlock(&mk_instance_mutex); + if (instance->kn) { kernfs_remove(instance->kn); instance->kn = NULL; mk_instance_put(instance); } mk_instance_put(instance); + mutex_lock(&mk_instance_mutex); return 0; } @@ -599,16 +624,30 @@ void mk_kernfs_cleanup(void) struct mk_instance *instance, *tmp; /* Remove all instances */ + mutex_lock(&mk_overlay_mutex); mutex_lock(&mk_instance_mutex); + list_for_each_entry(instance, &mk_instance_list, list) { + /* No inbox/IRQ retirement contract exists for boot records here. */ + if (instance == mk_self || instance == host_instance || + instance->state != MK_STATE_READY || instance->kimage) { + pr_err("Bulk kernfs cleanup refused: instance %d is still live\n", + instance->id); + mutex_unlock(&mk_instance_mutex); + mutex_unlock(&mk_overlay_mutex); + return; + } + } list_for_each_entry_safe(instance, tmp, &mk_instance_list, list) { - list_del(&instance->list); - idr_remove(&mk_instance_idr, instance->id); - mk_instance_put(instance); + if (mk_instance_destroy(instance)) { + mutex_unlock(&mk_instance_mutex); + mutex_unlock(&mk_overlay_mutex); + return; + } } mutex_unlock(&mk_instance_mutex); + mutex_unlock(&mk_overlay_mutex); - /* Clean up IDR */ - idr_destroy(&mk_instance_idr); + /* The remaining initialized IDR is also safe for RCU lookups. */ /* Remove sysfs mount point */ sysfs_remove_mount_point(fs_kobj, "multikernel"); diff --git a/kernel/multikernel/mem.c b/kernel/multikernel/mem.c index 0b489c62ebc9b0..7a23c92c63cc11 100644 --- a/kernel/multikernel/mem.c +++ b/kernel/multikernel/mem.c @@ -794,12 +794,15 @@ int mk_pool_map(phys_addr_t start, size_t size) return ret; } -void mk_pool_unmap(phys_addr_t start, size_t size) +int mk_pool_unmap(phys_addr_t start, size_t size) { + int ret; + guard(mutex)(&mk_pool_map_lock); get_online_mems(); - mk_arch_pool_unmap(start, size); + ret = mk_arch_pool_unmap(start, size); put_online_mems(); + return ret; } static void mk_pool_chunk_return(struct mk_pool_chunk *chunk) diff --git a/kernel/multikernel/messaging.c b/kernel/multikernel/messaging.c index c0bdfdd429cd85..168c678c16e54e 100644 --- a/kernel/multikernel/messaging.c +++ b/kernel/multikernel/messaging.c @@ -12,20 +12,28 @@ #include #include #include +#include +#include +#include +#include #include +#include "internal.h" /* Pending message tracking for request-response pattern */ struct mk_pending_msg { + u64 msg_id; /* Nonzero, never reused during this kernel boot */ u32 msg_type; /* Message type (e.g., MK_MSG_RESOURCE) */ u32 operation; /* Operation subtype (e.g., MK_RES_CPU_ADD) */ u64 resource_id; /* Resource identifier (physical CPU ID, PFN, etc.) */ int result; /* Operation result */ + bool completed; /* First reply wins, even after the waiter wakes */ struct completion done; /* Completion for waiting */ struct list_head list; /* List linkage */ }; static LIST_HEAD(mk_pending_msgs); -static DEFINE_SPINLOCK(mk_pending_msgs_lock); +static DEFINE_RAW_SPINLOCK(mk_pending_msgs_lock); +static u64 mk_last_msg_id; /* Per-type message handler registry */ struct mk_msg_type_handler { @@ -37,7 +45,15 @@ struct mk_msg_type_handler { }; static struct mk_msg_type_handler *mk_msg_type_handlers; -static raw_spinlock_t mk_msg_type_handlers_lock = __RAW_SPIN_LOCK_UNLOCKED(mk_msg_type_handlers_lock); +/* + * Process-context writers only. Callbacks use their immutable type_handler, + * never this registry or its mutex, and must not register/unregister handlers. + * Writers must not hold callback-owned locks while waiting for retirement. + */ +static DEFINE_MUTEX(mk_msg_type_handlers_lock); +static bool mk_messaging_initialized; +/* PCI registers at arch_initcall, before multikernel's subsys_initcall. */ +static bool mk_messaging_accepting = true; /** * mk_message_type_ipi_callback - IPI callback to handle incoming messages for a specific type @@ -71,6 +87,8 @@ static void mk_message_type_ipi_callback(struct mk_ipi_data *data, void *ctx) } msg = (struct mk_message *)data->buffer; + if (msg->msg_type != data->type) + return; /* Validate message structure */ if (msg->payload_len > (data->data_size - sizeof(struct mk_message))) { @@ -88,7 +106,8 @@ static void mk_message_type_ipi_callback(struct mk_ipi_data *data, void *ctx) msg_type, msg_subtype, payload_len, data->sender_cpu); /* Call the registered handler for this message type */ - type_handler->msg_handler(msg_type, msg_subtype, payload, payload_len, type_handler->context); + type_handler->msg_handler(msg_type, msg_subtype, msg->msg_id, + payload, payload_len, type_handler->context); } /* @@ -101,7 +120,8 @@ static void mk_message_type_ipi_callback(struct mk_ipi_data *data, void *ctx) * @operation: Operation subtype * @resource_id: Resource identifier * - * Returns pointer to pending message structure, or NULL on failure + * Returns a pending message, or ERR_PTR(-ENOMEM/-EOVERFLOW). + * The requester must consume it exactly once with wait or cancel. */ struct mk_pending_msg *mk_msg_pending_add(u32 msg_type, u32 operation, u64 resource_id) { @@ -110,7 +130,7 @@ struct mk_pending_msg *mk_msg_pending_add(u32 msg_type, u32 operation, u64 resou pending = kzalloc(sizeof(*pending), GFP_KERNEL); if (!pending) - return NULL; + return ERR_PTR(-ENOMEM); pending->msg_type = msg_type; pending->operation = operation; @@ -118,38 +138,67 @@ struct mk_pending_msg *mk_msg_pending_add(u32 msg_type, u32 operation, u64 resou pending->result = -ETIMEDOUT; /* Default to timeout */ init_completion(&pending->done); - spin_lock_irqsave(&mk_pending_msgs_lock, flags); + raw_spin_lock_irqsave(&mk_pending_msgs_lock, flags); + /* IDs are boot-local: exhaustion must fail, never wrap into an old ID. */ + if (mk_last_msg_id == U64_MAX) { + raw_spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + kfree(pending); + return ERR_PTR(-EOVERFLOW); + } + pending->msg_id = ++mk_last_msg_id; list_add(&pending->list, &mk_pending_msgs); - spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + raw_spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); return pending; } +u64 mk_msg_pending_id(const struct mk_pending_msg *pending) +{ + return pending->msg_id; +} + +/* Local cleanup only: does not cancel a request already delivered to a peer. */ +void mk_msg_pending_cancel(struct mk_pending_msg *pending) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&mk_pending_msgs_lock, flags); + list_del(&pending->list); + raw_spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + kfree(pending); +} + /** * mk_msg_pending_complete - Mark a pending operation as complete * @msg_type: Message type * @operation: Operation subtype * @resource_id: Resource identifier + * @msg_id: Nonzero request ID echoed by the responder * @result: Operation result (0 = success, negative = error) * * Called by response handlers to wake up waiting thread */ -void mk_msg_pending_complete(u32 msg_type, u32 operation, u64 resource_id, int result) +void mk_msg_pending_complete(u32 msg_type, u32 operation, u64 resource_id, + u64 msg_id, int result) { struct mk_pending_msg *pending; unsigned long flags; - spin_lock_irqsave(&mk_pending_msgs_lock, flags); + raw_spin_lock_irqsave(&mk_pending_msgs_lock, flags); list_for_each_entry(pending, &mk_pending_msgs, list) { - if (pending->msg_type == msg_type && + if (pending->msg_id == msg_id && + pending->msg_type == msg_type && pending->operation == operation && pending->resource_id == resource_id) { - pending->result = result; - complete(&pending->done); + if (!pending->completed) { + pending->result = result; + pending->completed = true; + complete(&pending->done); + } break; } } - spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + raw_spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); } /** @@ -163,78 +212,57 @@ int mk_msg_pending_wait(struct mk_pending_msg *pending, unsigned long timeout_ms { unsigned long timeout = msecs_to_jiffies(timeout_ms); unsigned long flags; + unsigned long completed; int result; - if (!wait_for_completion_timeout(&pending->done, timeout)) { + completed = wait_for_completion_timeout(&pending->done, timeout); + if (!completed) pr_err("Timeout waiting for operation 0x%x on resource %llu\n", pending->operation, pending->resource_id); - result = -ETIMEDOUT; - } else { - result = pending->result; - } - /* Remove from list and free */ - spin_lock_irqsave(&mk_pending_msgs_lock, flags); + /* A reply arriving after the wait timed out cannot undo the timeout. */ + raw_spin_lock_irqsave(&mk_pending_msgs_lock, flags); + result = completed ? pending->result : -ETIMEDOUT; list_del(&pending->list); - spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); + raw_spin_unlock_irqrestore(&mk_pending_msgs_lock, flags); kfree(pending); return result; } /** - * mk_send_message - Send a message to another multikernel instance + * mk_build_and_send - Build a message for an ID or referenced instance * @instance_id: Target multikernel instance ID + * @instance: Referenced target, or NULL for an ID lookup * @msg_type: Message type identifier * @subtype: Message subtype + * @msg_id: Request ID, or zero for an uncorrelated message * @payload: Pointer to payload data (can be NULL) * @payload_len: Length of payload data * * Returns 0 on success, negative error code on failure */ static int mk_build_and_send(int instance_id, struct mk_instance *instance, - u32 msg_type, u32 subtype, void *payload, + u32 msg_type, u32 subtype, u64 msg_id, void *payload, u32 payload_len) { - struct mk_message *msg; - size_t total_size; + struct mk_message msg = { + .msg_type = msg_type, + .msg_subtype = subtype, + .msg_id = msg_id, + .payload_len = payload_len, + }; int ret; - /* Calculate total message size */ - total_size = sizeof(struct mk_message) + payload_len; - /* Check if message fits in IPI buffer */ - if (total_size > MK_MAX_DATA_SIZE) { - pr_err("Multikernel message too large: %zu > %d bytes\n", - total_size, MK_MAX_DATA_SIZE); + if (payload_len > MK_MAX_DATA_SIZE - sizeof(msg)) { + pr_err("Multikernel message payload too large: %u bytes\n", + payload_len); return -EMSGSIZE; } - /* Allocate temporary buffer for message */ - msg = kzalloc(total_size, GFP_ATOMIC); - if (!msg) - return -ENOMEM; - - /* Fill in message header */ - msg->msg_type = msg_type; - msg->msg_subtype = subtype; - msg->msg_id = 0; /* Could be enhanced with unique IDs later */ - msg->payload_len = payload_len; - - /* Copy payload if provided */ - if (payload && payload_len > 0) - memcpy(msg->payload, payload, payload_len); - - /* Send via IPI using the message type as IPI type */ - if (instance) - ret = multikernel_send_ipi_data_to(instance, msg, total_size, - msg_type); - else - ret = multikernel_send_ipi_data(instance_id, msg, total_size, - msg_type); - - /* Clean up temporary buffer */ - kfree(msg); + /* Assemble in the reserved ring slot, including from raw-locked callbacks. */ + ret = mk_send_message_ipi(instance_id, instance, &msg, payload); if (ret < 0) { pr_err("Failed to send multikernel message: %d\n", ret); @@ -247,18 +275,25 @@ static int mk_build_and_send(int instance_id, struct mk_instance *instance, return 0; } +int mk_send_message_id(int instance_id, u32 msg_type, u32 subtype, u64 msg_id, + void *payload, u32 payload_len) +{ + return mk_build_and_send(instance_id, NULL, msg_type, subtype, msg_id, + payload, payload_len); +} + int mk_send_message(int instance_id, u32 msg_type, u32 subtype, void *payload, u32 payload_len) { - return mk_build_and_send(instance_id, NULL, msg_type, subtype, payload, - payload_len); + return mk_send_message_id(instance_id, msg_type, subtype, 0, + payload, payload_len); } EXPORT_SYMBOL(mk_send_message); int mk_send_message_to(struct mk_instance *instance, u32 msg_type, u32 subtype, void *payload, u32 payload_len) { - return mk_build_and_send(instance->id, instance, msg_type, subtype, + return mk_build_and_send(instance->id, instance, msg_type, subtype, 0, payload, payload_len); } EXPORT_SYMBOL(mk_send_message_to); @@ -370,31 +405,40 @@ EXPORT_SYMBOL_GPL(mk_request_answer); * @handler: Handler function * @ctx: Context pointer passed to handler * + * Process context only, never from a callback or with callback-owned locks held. * Returns 0 on success, negative error code on failure */ int mk_register_msg_handler(u32 msg_type, mk_msg_handler_t handler, void *ctx) { struct mk_msg_type_handler *type_handler; - unsigned long flags; + int ret; if (!handler) return -EINVAL; + if (WARN_ON_ONCE(in_interrupt())) + return -EWOULDBLOCK; + + mutex_lock(&mk_msg_type_handlers_lock); + if (!mk_messaging_accepting) { + ret = -ESHUTDOWN; + goto unlock; + } /* Check if handler for this type already exists */ - raw_spin_lock_irqsave(&mk_msg_type_handlers_lock, flags); for (type_handler = mk_msg_type_handlers; type_handler; type_handler = type_handler->next) { if (type_handler->msg_type == msg_type) { - raw_spin_unlock_irqrestore(&mk_msg_type_handlers_lock, flags); pr_warn("Handler for message type 0x%x already registered\n", msg_type); - return -EEXIST; + ret = -EEXIST; + goto unlock; } } - raw_spin_unlock_irqrestore(&mk_msg_type_handlers_lock, flags); /* Allocate new type handler entry */ type_handler = kzalloc(sizeof(*type_handler), GFP_KERNEL); - if (!type_handler) - return -ENOMEM; + if (!type_handler) { + ret = -ENOMEM; + goto unlock; + } type_handler->msg_type = msg_type; type_handler->msg_handler = handler; @@ -406,17 +450,19 @@ int mk_register_msg_handler(u32 msg_type, mk_msg_handler_t handler, void *ctx) if (!type_handler->ipi_handler) { pr_err("Failed to register IPI handler for message type 0x%x\n", msg_type); kfree(type_handler); - return -ENOMEM; + ret = -ENOMEM; + goto unlock; } /* Add to type handler list */ - raw_spin_lock_irqsave(&mk_msg_type_handlers_lock, flags); type_handler->next = mk_msg_type_handlers; mk_msg_type_handlers = type_handler; - raw_spin_unlock_irqrestore(&mk_msg_type_handlers_lock, flags); pr_debug("Registered multikernel message handler for type 0x%x\n", msg_type); - return 0; + ret = 0; +unlock: + mutex_unlock(&mk_msg_type_handlers_lock); + return ret; } EXPORT_SYMBOL(mk_register_msg_handler); @@ -425,18 +471,21 @@ EXPORT_SYMBOL(mk_register_msg_handler); * @msg_type: Message type to unregister * @handler: Handler function to remove * + * Process context only, never from a callback or with callback-owned locks held. + * Drains entered callbacks before the caller may release its context. * Returns 0 on success, negative error code on failure */ int mk_unregister_msg_handler(u32 msg_type, mk_msg_handler_t handler) { struct mk_msg_type_handler **pp, *type_handler; - unsigned long flags; int found = 0; if (!handler) return -EINVAL; + if (WARN_ON_ONCE(in_interrupt())) + return -EWOULDBLOCK; - raw_spin_lock_irqsave(&mk_msg_type_handlers_lock, flags); + mutex_lock(&mk_msg_type_handlers_lock); pp = &mk_msg_type_handlers; while ((type_handler = *pp) != NULL) { if (type_handler->msg_type == msg_type && type_handler->msg_handler == handler) { @@ -446,18 +495,19 @@ int mk_unregister_msg_handler(u32 msg_type, mk_msg_handler_t handler) } pp = &type_handler->next; } - raw_spin_unlock_irqrestore(&mk_msg_type_handlers_lock, flags); if (found) { - /* Unregister the IPI handler */ + /* Keep same-type insertion excluded until old callbacks have returned. */ if (type_handler->ipi_handler) { multikernel_unregister_handler(type_handler->ipi_handler); } kfree(type_handler); + mutex_unlock(&mk_msg_type_handlers_lock); pr_debug("Unregistered multikernel message handler for type 0x%x\n", msg_type); return 0; } + mutex_unlock(&mk_msg_type_handlers_lock); return -ENOENT; } EXPORT_SYMBOL(mk_unregister_msg_handler); @@ -470,6 +520,14 @@ EXPORT_SYMBOL(mk_unregister_msg_handler); */ int __init mk_messaging_init(void) { + mutex_lock(&mk_msg_type_handlers_lock); + if (mk_messaging_initialized) { + mutex_unlock(&mk_msg_type_handlers_lock); + return -EALREADY; + } + mk_messaging_initialized = true; + mutex_unlock(&mk_msg_type_handlers_lock); + /* No global IPI handler needed anymore - handlers are registered per message type */ pr_info("Multikernel messaging system initialized\n"); return 0; @@ -478,19 +536,24 @@ int __init mk_messaging_init(void) /** * mk_messaging_cleanup - Cleanup the messaging system * - * Called during multikernel cleanup + * Process context only; no callback-owned locks may be held. + * Closes registration permanently, then drains entered callbacks. */ void mk_messaging_cleanup(void) { struct mk_msg_type_handler *type_handler, *next; - unsigned long flags; + + if (WARN_ON_ONCE(in_interrupt())) + return; /* Clean up all registered message type handlers */ - raw_spin_lock_irqsave(&mk_msg_type_handlers_lock, flags); + mutex_lock(&mk_msg_type_handlers_lock); + mk_messaging_initialized = true; + mk_messaging_accepting = false; type_handler = mk_msg_type_handlers; mk_msg_type_handlers = NULL; - raw_spin_unlock_irqrestore(&mk_msg_type_handlers_lock, flags); + /* The mutex is never callback-owned; concurrent cleanup also waits here. */ while (type_handler) { next = type_handler->next; @@ -502,6 +565,7 @@ void mk_messaging_cleanup(void) kfree(type_handler); type_handler = next; } + mutex_unlock(&mk_msg_type_handlers_lock); pr_info("Multikernel messaging system cleaned up\n"); } diff --git a/kernel/multikernel/overlay.c b/kernel/multikernel/overlay.c index 3ee26597620c57..4324b43990ab8a 100644 --- a/kernel/multikernel/overlay.c +++ b/kernel/multikernel/overlay.c @@ -6,7 +6,7 @@ * * Provides /sys/fs/multikernel/overlays/ for dynamic resource adjustments * via Device Tree overlays. Each overlay is tracked as an independent - * transaction that can be applied and rolled back atomically. + * transaction. Failed or partial transactions retain their records. */ #include @@ -77,7 +77,9 @@ static LIST_HEAD(mk_overlay_tx_list); /* List of all transactions */ /* * Serializes every user-driven reconfiguration of this kernel's resources: * the transaction list, overlay application and rollback, and the baseline - * write in kernfs.c. Two writers would otherwise run pool moves at once. + * write in kernfs.c. MK image load, exec and cleanup take it too, so an image + * cannot use the instance pool while a controller resource writer changes it. + * Two writers would otherwise run pool moves at once. */ DEFINE_MUTEX(mk_overlay_mutex); static atomic_t mk_overlay_next_id = ATOMIC_INIT(1); /* Next transaction ID */ @@ -1407,17 +1409,6 @@ static int mk_overlay_apply(const void *dtbo_data, size_t dtbo_size) guard(mutex)(&mk_overlay_mutex); - ret = mk_overlay_parse_and_apply(tx, dtbo_data); - - if (ret < 0) { - pr_err("Failed to apply overlay transaction %d: %d\n", tx->id, ret); - tx->status = MK_OVERLAY_TX_FAILED; - /* Continue to create sysfs entry so user can see failure */ - } else { - tx->status = MK_OVERLAY_TX_APPLIED; - pr_info("Overlay transaction %d applied successfully\n", tx->id); - } - snprintf(tx_name, sizeof(tx_name), "tx_%d", tx->id); tx->dir_kn = kernfs_create_dir(mk_overlay_root_kn, tx_name, 0755, tx); if (IS_ERR(tx->dir_kn)) { @@ -1439,38 +1430,22 @@ static int mk_overlay_apply(const void *dtbo_data, size_t dtbo_size) return ret; } - kernfs_activate(tx->dir_kn); - - list_add_tail(&tx->list, &mk_overlay_tx_list); - - return tx->id; -} - -static int mk_overlay_remove_tx(struct mk_overlay_tx *tx) -{ - int ret = 0; - - lockdep_assert_held(&mk_overlay_mutex); - - pr_info("Removing overlay transaction %d\n", tx->id); - - ret = mk_overlay_parse_and_rollback(tx, tx->dtbo_data); + /* The inactive record must exist before any resource can change. */ + ret = mk_overlay_parse_and_apply(tx, dtbo_data); if (ret < 0) { - pr_err("Failed to rollback overlay transaction %d: %d\n", tx->id, ret); - /* Continue with cleanup even if rollback failed */ + pr_err("Failed to apply overlay transaction %d: %d\n", tx->id, ret); + tx->status = MK_OVERLAY_TX_FAILED; } else { - pr_info("Overlay transaction %d rolled back successfully\n", tx->id); + tx->status = MK_OVERLAY_TX_APPLIED; + pr_info("Overlay transaction %d applied successfully\n", tx->id); } - tx->status = MK_OVERLAY_TX_REMOVED; + kernfs_activate(tx->dir_kn); - list_del(&tx->list); - /* The whole overlays directory goes away with the caller */ - kfree(tx->dtbo_data); - kfree(tx); + list_add_tail(&tx->list, &mk_overlay_tx_list); - return 0; + return tx->id; } /** @@ -1526,7 +1501,7 @@ static ssize_t mk_overlay_new_write(struct kernfs_open_file *of, char *buf, * directories are removable, everything else keeps returning -EPERM. * * A rollback that fails leaves the transaction in place with status - * "failed", so the user can see what happened and retry. + * "failed". There is no per-operation journal to safely retry an inverse. */ int mk_overlay_rmdir(struct kernfs_node *kn) { @@ -1545,6 +1520,9 @@ int mk_overlay_rmdir(struct kernfs_node *kn) scoped_guard(mutex, &mk_overlay_mutex) { if (list_empty(&tx->list)) return -ENOENT; + /* Never inverse an unapplied item or repeat a partial inverse. */ + if (tx->status != MK_OVERLAY_TX_APPLIED) + return -EBUSY; pr_info("Removing overlay transaction %d\n", tx->id); @@ -1604,21 +1582,3 @@ int mk_overlay_init(void) pr_info("Multikernel overlay subsystem initialized\n"); return 0; } - -void mk_overlay_exit(void) -{ - struct mk_overlay_tx *tx, *tmp; - - pr_info("Cleaning up multikernel overlay subsystem\n"); - - mutex_lock(&mk_overlay_mutex); - list_for_each_entry_safe(tx, tmp, &mk_overlay_tx_list, list) { - mk_overlay_remove_tx(tx); - } - mutex_unlock(&mk_overlay_mutex); - - if (mk_overlay_root_kn) { - kernfs_remove(mk_overlay_root_kn); - mk_overlay_root_kn = NULL; - } -} diff --git a/kernel/multikernel/pciback.c b/kernel/multikernel/pciback.c index 669084f155cb50..5369325fa9a0ae 100644 --- a/kernel/multikernel/pciback.c +++ b/kernel/multikernel/pciback.c @@ -39,6 +39,7 @@ #define pr_fmt(fmt) "mk_pciback: " fmt #include +#include #include #include #include @@ -60,6 +61,8 @@ struct mk_pciback_dev { refcount_t ref; struct mk_instance *instance; struct pci_dev *pdev; + struct mutex io_lock; /* Slow config accesses versus device retirement */ + bool revoked; unsigned long sizing; /* BARs written with something else than their value */ u16 sriov; /* SR-IOV capability, or 0 */ }; @@ -71,8 +74,12 @@ struct mk_pciback_work { struct work_struct work; struct mk_pci_cfg_payload req; u32 subtype; + struct mk_pciback_dev *dev; }; +/* dev is the busy marker, protected by mk_pciback_lock. */ +static struct mk_pciback_work mk_pciback_requests[MK_IPI_RING_SIZE]; + static struct mk_pciback_dev *mk_pciback_find(int instance_id, u16 domain, u8 bus, u8 devfn) { @@ -300,71 +307,70 @@ static void mk_pciback_answer(struct mk_instance *instance, sizeof(*ack)); } -/* - * The slow way: the requester is not known here (it was lent nothing), - * or this kernel's own config access goes over the ring. - */ +/* This kernel's own config access goes over the ring and may wait. */ static void mk_pciback_work_fn(struct work_struct *work) { struct mk_pciback_work *w = container_of(work, struct mk_pciback_work, work); struct mk_pci_cfg_payload *ack = &w->req; - struct mk_instance *instance; - struct mk_pciback_dev *d; - - instance = mk_instance_find(ack->sender_instance_id); - if (!instance) - goto out; - - scoped_guard(raw_spinlock_irqsave, &mk_pciback_lock) { - d = mk_pciback_find(instance->id, ack->domain, ack->bus, - ack->devfn); - if (d) - refcount_inc(&d->ref); - } + struct mk_pciback_dev *d = w->dev; - if (d) { + /* A pin preserves identity; revocation forbids both access and stale ACKs. */ + mutex_lock(&d->io_lock); + if (!READ_ONCE(d->revoked)) { mk_pciback_do(d, w->subtype, ack); - mk_pciback_put(d); - } else { - ack->result = PCIBIOS_DEVICE_NOT_FOUND; - ack->val = ~0U; + mk_pciback_answer(d->instance, ack); } - - mk_pciback_answer(instance, ack); - mk_instance_put(instance); -out: - kfree(w); + mutex_unlock(&d->io_lock); + mk_pciback_put(d); + /* No request access after release; callback and queue remain unchanged. */ + scoped_guard(raw_spinlock_irqsave, &mk_pciback_lock) + w->dev = NULL; } -static void mk_pciback_defer(const struct mk_pci_cfg_payload *req, u32 subtype) +static void mk_pciback_defer(const struct mk_pci_cfg_payload *req, u32 subtype, + struct mk_pciback_dev *d) { - struct mk_pciback_work *w = kzalloc(sizeof(*w), GFP_ATOMIC); + struct mk_pciback_work *w; + int i; - if (!w) + lockdep_assert_held(&mk_pciback_lock); + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + w = &mk_pciback_requests[i]; + if (w->dev) + continue; + refcount_inc(&d->ref); + w->dev = d; + w->req = *req; + w->subtype = subtype; + schedule_work(&w->work); return; - w->req = *req; - w->subtype = subtype; - INIT_WORK(&w->work, mk_pciback_work_fn); - schedule_work(&w->work); + } } static void mk_pciback_request(const struct mk_pci_cfg_payload *req, u32 subtype) { struct mk_pci_cfg_payload ack = *req; + struct mk_instance *instance; struct mk_pciback_dev *d; - if (mk_manifest_phys()) { - mk_pciback_defer(req, subtype); - return; - } - guard(raw_spinlock_irqsave)(&mk_pciback_lock); d = mk_pciback_find(req->sender_instance_id, req->domain, req->bus, req->devfn); if (!d) { - mk_pciback_defer(req, subtype); + /* RCU-pinned lookup and a negative reply need no config access. */ + instance = mk_instance_find(req->sender_instance_id); + if (instance) { + ack.result = PCIBIOS_DEVICE_NOT_FOUND; + ack.val = ~0U; + mk_pciback_answer(instance, &ack); + mk_instance_put(instance); + } + return; + } + if (mk_manifest_phys()) { + mk_pciback_defer(req, subtype, d); return; } @@ -372,20 +378,23 @@ static void mk_pciback_request(const struct mk_pci_cfg_payload *req, mk_pciback_answer(d->instance, &ack); } -static void mk_pciback_msg_handler(u32 msg_type, u32 subtype, void *payload, - u32 payload_len, void *ctx) +static void mk_pciback_msg_handler(u32 msg_type, u32 subtype, u64 msg_id, + void *payload, u32 payload_len, void *ctx) { - if (payload_len < sizeof(struct mk_pci_cfg_payload)) + struct mk_pci_cfg_payload req; + + if (payload_len < sizeof(req)) return; + /* The wire payload need not align its 64-bit sequence number. */ + memcpy(&req, payload, sizeof(req)); switch (subtype) { case MK_PCI_CFG_READ: case MK_PCI_CFG_WRITE: - mk_pciback_request(payload, subtype); + mk_pciback_request(&req, subtype); break; case MK_PCI_CFG_ACK: - mk_request_answer(((struct mk_pci_cfg_payload *)payload)->seq, - payload, sizeof(struct mk_pci_cfg_payload)); + mk_request_answer(req.seq, &req, sizeof(req)); break; } } @@ -423,6 +432,7 @@ int mk_pciback_lend_one(struct mk_instance *instance, u16 domain, u8 bus, pm_runtime_get_sync(&pdev->dev); refcount_set(&d->ref, 1); + mutex_init(&d->io_lock); d->pdev = pdev; d->sriov = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); d->instance = instance; @@ -438,8 +448,11 @@ static void mk_pciback_free(struct list_head *gone) struct mk_pciback_dev *d, *tmp; list_for_each_entry_safe(d, tmp, gone, list) { + /* Admission is closed; wait outside the registry spinlock. */ + mutex_lock(&d->io_lock); if (pci_reset_function(d->pdev)) pci_clear_master(d->pdev); + mutex_unlock(&d->io_lock); mk_pciback_put(d); } } @@ -455,8 +468,10 @@ void mk_pciback_unlend_one(struct mk_instance *instance, u16 domain, u8 bus, scoped_guard(raw_spinlock_irqsave, &mk_pciback_lock) { d = mk_pciback_find(instance->id, domain, bus, devfn); - if (d) + if (d) { + WRITE_ONCE(d->revoked, true); list_move(&d->list, &gone); + } } mk_pciback_free(&gone); } @@ -472,8 +487,10 @@ void mk_pciback_unlend(struct mk_instance *instance) scoped_guard(raw_spinlock_irqsave, &mk_pciback_lock) { list_for_each_entry_safe(d, tmp, &mk_pciback_devs, list) { - if (d->instance == instance) + if (d->instance == instance) { + WRITE_ONCE(d->revoked, true); list_move(&d->list, &gone); + } } } mk_pciback_free(&gone); @@ -510,6 +527,11 @@ int mk_pciback_lend(struct mk_instance *instance) /* Answers reach a spawn's front end from its first bus scan, at subsys_initcall */ static int __init mk_pciback_init(void) { + int i; + + for (i = 0; i < MK_IPI_RING_SIZE; i++) + INIT_WORK(&mk_pciback_requests[i].work, mk_pciback_work_fn); + return mk_register_msg_handler(MK_MSG_PCI, mk_pciback_msg_handler, NULL); } arch_initcall(mk_pciback_init); diff --git a/net/vmw_vsock/mk_transport.c b/net/vmw_vsock/mk_transport.c index 434e67b5291d00..0f73d1ce5482b6 100644 --- a/net/vmw_vsock/mk_transport.c +++ b/net/vmw_vsock/mk_transport.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -29,9 +30,9 @@ static struct vsock_transport mk_transport; struct mk_vsock { struct workqueue_struct *workqueue; - struct sk_buff_head pkt_queue; + STRUCT_KFIFO_REC_2(MK_IPI_RING_SIZE * MK_MAX_DATA_SIZE) rx_fifo; struct work_struct rx_work; - spinlock_t rx_lock; + raw_spinlock_t rx_lock; }; static struct mk_vsock mk_vsock_dev; @@ -118,17 +119,6 @@ static void mk_vsock_send_rst(struct virtio_vsock_hdr *hdr) kfree_skb(skb); } -static void mk_queue_rx_pkt(struct sk_buff *skb) -{ - struct mk_vsock *mk = &mk_vsock_dev; - - spin_lock(&mk->rx_lock); - skb_queue_tail(&mk->pkt_queue, skb); - spin_unlock(&mk->rx_lock); - - queue_work(mk->workqueue, &mk->rx_work); -} - static void mk_vsock_rx_pkt(struct sk_buff *skb) { struct virtio_vsock_hdr *hdr = (struct virtio_vsock_hdr *)skb->data; @@ -282,44 +272,60 @@ static void mk_vsock_rx_pkt(struct sk_buff *skb) static void mk_vsock_rx_work(struct work_struct *work) { struct mk_vsock *mk = container_of(work, struct mk_vsock, rx_work); - struct sk_buff_head pkts; + unsigned int budget = MK_IPI_RING_SIZE, len, copied; struct sk_buff *skb; unsigned long flags; - skb_queue_head_init(&pkts); + while (budget--) { + raw_spin_lock_irqsave(&mk->rx_lock, flags); + len = kfifo_peek_len(&mk->rx_fifo); + raw_spin_unlock_irqrestore(&mk->rx_lock, flags); + if (!len) + return; - spin_lock_irqsave(&mk->rx_lock, flags); - skb_queue_splice_init(&mk->pkt_queue, &pkts); - spin_unlock_irqrestore(&mk->rx_lock, flags); + /* This work is the only consumer, so the head survives allocation. */ + skb = alloc_skb(len, GFP_KERNEL); + raw_spin_lock_irqsave(&mk->rx_lock, flags); + if (skb) + copied = kfifo_out(&mk->rx_fifo, skb_put(skb, len), len); + else + kfifo_skip(&mk->rx_fifo); + raw_spin_unlock_irqrestore(&mk->rx_lock, flags); + if (skb) { + if (copied == len) + mk_vsock_rx_pkt(skb); + else + kfree_skb(skb); + } + } - while ((skb = __skb_dequeue(&pkts))) - mk_vsock_rx_pkt(skb); + raw_spin_lock_irqsave(&mk->rx_lock, flags); + if (!kfifo_is_empty(&mk->rx_fifo)) + queue_work(mk->workqueue, &mk->rx_work); + raw_spin_unlock_irqrestore(&mk->rx_lock, flags); } -static void mk_vsock_ipi_handler(u32 msg_type, u32 subtype, +static void mk_vsock_ipi_handler(u32 msg_type, u32 subtype, u64 msg_id, void *payload, u32 payload_len, void *ctx) { - struct sk_buff *skb; - struct virtio_vsock_hdr *hdr; + struct mk_vsock *mk = &mk_vsock_dev; + struct virtio_vsock_hdr hdr; + unsigned long flags; if (msg_type != MK_MSG_NETWORK || subtype != MK_NET_VSOCK_PKT) return; - if (payload_len < sizeof(struct virtio_vsock_hdr)) { - pr_warn("mk_vsock: packet too small (%u bytes)\n", payload_len); + if (payload_len < sizeof(hdr) || payload_len > MK_VSOCK_MAX_PKT_SIZE) return; - } - - skb = alloc_skb(payload_len, GFP_ATOMIC); - if (!skb) { - pr_err("mk_vsock: failed to allocate skb\n"); + memcpy(&hdr, payload, sizeof(hdr)); + if (le32_to_cpu(hdr.len) > payload_len - sizeof(hdr)) return; - } - - skb_put_data(skb, payload, payload_len); - hdr = (struct virtio_vsock_hdr *)skb->data; - mk_queue_rx_pkt(skb); + /* The dispatcher releases the shared message slot on callback return. */ + raw_spin_lock_irqsave(&mk->rx_lock, flags); + if (kfifo_in(&mk->rx_fifo, payload, payload_len)) + queue_work(mk->workqueue, &mk->rx_work); + raw_spin_unlock_irqrestore(&mk->rx_lock, flags); } /* @@ -747,8 +753,8 @@ static int __init mk_vsock_transport_init(void) struct mk_vsock *mk = &mk_vsock_dev; int ret; - spin_lock_init(&mk->rx_lock); - skb_queue_head_init(&mk->pkt_queue); + raw_spin_lock_init(&mk->rx_lock); + INIT_KFIFO(mk->rx_fifo); INIT_WORK(&mk->rx_work, mk_vsock_rx_work); mk->workqueue = alloc_workqueue("mk_vsock_wq", WQ_UNBOUND, 0); @@ -757,17 +763,17 @@ static int __init mk_vsock_transport_init(void) return -ENOMEM; } - ret = mk_register_msg_handler(MK_MSG_NETWORK, mk_vsock_ipi_handler, NULL); + ret = vsock_core_register(&mk_transport, VSOCK_TRANSPORT_F_MULTIKERNEL); if (ret < 0) { - pr_err("mk_vsock: failed to register message handler: %d\n", ret); + pr_err("mk_vsock: failed to register transport: %d\n", ret); destroy_workqueue(mk->workqueue); return ret; } - ret = vsock_core_register(&mk_transport, VSOCK_TRANSPORT_F_MULTIKERNEL); + ret = mk_register_msg_handler(MK_MSG_NETWORK, mk_vsock_ipi_handler, NULL); if (ret < 0) { - pr_err("mk_vsock: failed to register transport: %d\n", ret); - mk_unregister_msg_handler(MK_MSG_NETWORK, mk_vsock_ipi_handler); + pr_err("mk_vsock: failed to register message handler: %d\n", ret); + vsock_core_unregister(&mk_transport); destroy_workqueue(mk->workqueue); return ret; } @@ -780,7 +786,6 @@ static void __exit mk_vsock_transport_exit(void) { struct mk_vsock *mk = &mk_vsock_dev; - vsock_core_unregister(&mk_transport); mk_unregister_msg_handler(MK_MSG_NETWORK, mk_vsock_ipi_handler); if (mk->workqueue) { @@ -788,7 +793,7 @@ static void __exit mk_vsock_transport_exit(void) destroy_workqueue(mk->workqueue); } - skb_queue_purge(&mk->pkt_queue); + vsock_core_unregister(&mk_transport); pr_info("Multikernel vsock transport unregistered\n"); } diff --git a/tools/testing/selftests/multikernel/README.ack b/tools/testing/selftests/multikernel/README.ack new file mode 100644 index 00000000000000..900fb74bb8752a --- /dev/null +++ b/tools/testing/selftests/multikernel/README.ack @@ -0,0 +1,114 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +The message builder no longer allocates a temporary wire buffer. The transport +assembles in its reserved slot; ACK tests require send success under allocator +exhaustion and zero hard-IPI allocation attempts. Transport publication and wire +padding are checked separately by ipi_writer_test.sh. This does not remove the +remaining resource/PCI work allocations or VSOCK receive allocation. + +ACK correlation checks +====================== + +From the repository root, with a host C compiler and awk: + + sh tools/testing/selftests/multikernel/ack_test.sh + +Optional modes: regression, characterization, completion, workers, failure. +Each selected mode must exit 0; an invalid argument exits 2. CC and CFLAGS +select the compiler and instrumentation, for example: + + CC=clang CFLAGS='-O1 -g -fsanitize=address -fno-omit-frame-pointer' \ + sh tools/testing/selftests/multikernel/ack_test.sh + +What actually runs +------------------ + +The runner extracts complete definitions verbatim from the current checkout: +pending allocation/ID/completion/wait/cancel, message encoder and dispatch, +resource dispatch and all six receive handlers/workers, system dispatch and +shutdown worker/notification/parking entry. The repository's tools/ list and +error-pointer helpers are used. No alternate implementation of this logic is +compiled. Production changes that break extraction fail compilation. + +Allocation, spinlocks, completions, transport, work scheduling, hardware +mutation and final CPU parking are deterministic host boundaries. No real CPU, +memory or PCI ownership transition occurs. The six sending resource helpers +and multikernel_halt_by_id are checked by source review and kernel builds, not +executed by this harness. Cancellation tests cover the pending/encoder APIs, +not those callers' resource rollback behavior. + +Checks include: + +* Delayed CPU add/remove and shutdown ACK A cannot complete retry B. +* IDs and type/operation/resource must all match; zero is not a pending ID. +* Conflicting duplicates preserve the first result, even after the waiter + consumes its completion token. A wait that reported timeout stays timed out + if an ACK arrives before list removal. +* Real workers retain IDs after the receive buffer is overwritten, echo both + zero and full-width nonzero IDs, preserve results and route replies to the + requester. Allocated-ID round trips pass through both real dispatch layers. +* Allocation/send failures, cancellation before/after completion, concurrent + outstanding same-tuple requests, the last u64 ID and exhaustion are checked. +* Voluntary HALTED and the uncorrelated sender retain their zero-ID behavior. +* Requested shutdown allocates no work item in hard IPI context, even under + allocator exhaustion. A short request cannot consume admission; duplicates + cannot replace the first requester/ID or requeue work after dequeue. Admission + stays closed until this kernel parks forever; each new kernel has its own slot. +* PCI add/remove ACKs match requester keys for domains 0x8000 and 0xffff; + signed promotion must not sign-extend the worker's u64 resource key. +* Receive decoding preserves wire bytes and sizes, accepts maximum-size trailing + data without overrunning local storage, and rejects short/empty requests. + HALTED retains its instance ID without an unaligned payload access. + +This is controlled interleaving, not a threaded race, kernel memory-ordering, +transport-delay, boot or physical-runtime test. Pending entries must be removed +and the lock released after wait/cancel. Sanitizers can check local memory use +but do not qualify these substituted kernel boundaries. + +The allocation checks cover requested shutdown and HALTED admission. Resource, +PCI and network receive allocation contexts require separate qualification. +HALTED pins the exact instance before queueing its embedded work item. Short +payloads, non-shutdown operations, nonzero results, overflowing instance IDs and +unknown IDs cannot queue it. halt_settlement_test exercises worker identity, +duplicate coalescing and reference release; start_lifetime_test checks that a +queued/running notice prevents a new execution generation. + +Protocol limits +--------------- + +The existing header and payload layouts are unchanged. Qualify matched-revision +host/spawn kernels; differing configurations are allowed, mixed revisions are +not qualified. Legacy zero-ID requests still execute and echo zero, but their +replies cannot satisfy a new pending request. An old responder can execute a +new request and return an unusable zero-ID ACK: rejecting it is not rollback. + +IDs are unique only during one requester kernel lifetime. They are not peer +authentication or session identity. Requester restart with surviving peers, +queued work across restart, remote cancellation, CPU ownership recovery and +safe image/ring reclamation require separate work. A shutdown ACK means the +peer is about to park, not proof that all its CPUs have stopped. + +Payload alignment regression +---------------------------- + +The original characterization test at 11efe070fdd57c9e3f71218c92989697723f9a84 +and ACK tip 99f395007d84 reported unaligned resource/ACK accesses under UBSan: +mk_message.payload starts at offset 20 on rv64/x86_64 while several decoded +structs require 8-byte alignment. Resource and system dispatch now copy bounded +payloads into aligned local storage. The existing header, payload offsets and +payload sizes remain unchanged, and workers retain their own data before the +callback returns. The vsock handler already copies its payload into an skb. + +Run the full suite with alignment checking enabled: + + CC=clang CFLAGS='-O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer' \ + sh tools/testing/selftests/multikernel/ack_test.sh + +The test itself also uses memcpy to decode captured wire ACKs. Sanitizer success +applies to these exercised production paths and deterministic boundaries. + +Historical RED evidence: the original test at 11efe070 (on upstream 347841a29a94) +failed eight regression assertions and passed ten characterizations. Worker +echo tests additionally reproduced loss of nonzero IDs in all seven workers +before ID plumbing. These are not expected-failure wrappers; current logical +checks and ASan must pass. diff --git a/tools/testing/selftests/multikernel/README.callback-writers b/tools/testing/selftests/multikernel/README.callback-writers new file mode 100644 index 00000000000000..6a5a7f8bb2da34 --- /dev/null +++ b/tools/testing/selftests/multikernel/README.callback-writers @@ -0,0 +1,103 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Local callback writers: messaging registry and host mktty +======================================================== + +Run from the repository root (no kernel build): + + sh tools/testing/selftests/multikernel/messaging_registry_test.sh + sh tools/testing/selftests/multikernel/mktty_lifetime_test.sh + +Both scripts extract the current production functions verbatim into a temporary +header, compile a host harness, and remove their temporary directory on exit. +CC and CFLAGS select the compiler/options; for example repeat both commands with +CC=gcc CFLAGS=-O2, then CC=clang +CFLAGS='-O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer'. +Assertions must remain enabled. mktty's harness suppresses unused-parameter and +sign-compare warnings present in the extracted baseline file operations. + +Messaging registry +------------------ + +The mutex serializes duplicate check, lower-level callback registration and list +insertion. Unregister holds it through lower-level callback retirement and context +freeing, so a replacement of the same type cannot overlap the old callback's +retirement. Cleanup closes admission before detaching the list and retains the +mutex until every detached handler is retired. A concurrent cleanup therefore +cannot report completion while another cleanup still drains callbacks. + +Admission begins open for PCI's arch-init handler registration. Messaging init +preserves those handlers; cleanup permanently closes admission even if called +before init. Registration while closed returns -ESHUTDOWN; another init returns +-EALREADY. Allocation failure and duplicate/missing-handler errors leave +the mutex usable. No message layout, peer/session protocol or API version changes. + +Register, unregister and cleanup require sleepable process context and no +callback-owned locks. Interrupt-context attempts are warned and rejected +(-EWOULDBLOCK for register/unregister; cleanup returns without changing state). +Other atomic process-context callers remain forbidden by the mutex contract. +The callback accesses its immutable per-type context, never the registry mutex; +callbacks must not register/unregister or clean up messaging. Owners retain their +callback context until successful unregister or completed cleanup. Production +multikernel_unregister_handler() supplies the existing RCU grace-period guarantee. + +Eight controlled scenarios exercise competing same-type writers, unregister vs +replacement, cleanup vs registration, concurrent cleanups, closed admission and +one-shot init, early PCI registration, allocation/error paths, and interrupt- +context rejection. An entered +callback runs the extracted message wrapper while a remover waits at the modeled +RCU boundary; its context must not be freed before callback return. Checkpoints +use condition variables and observed lock contention, not scheduling sleeps. + +Host mktty +---------- + +Open/release now use IRQ-save locking of the same connection-list lock held by +the hardIRQ receive callback. This excludes same-CPU interrupt reentry. The +callback retains the list lock through RX use and wakeup; release unlinks under +that lock before freeing the RX buffer and connection. Concurrent file operations +continue to rely on VFS file references delaying final .release(). + +First writes validate the instance outside the spinlock, then publish the target +once under the connection-list lock with WRITE_ONCE. A competing first write +that also observed an unconnected FD returns -EISCONN after losing publication. +Later read/poll/write paths use READ_ONCE; a data write snapshots the immutable +target once for all chunks. Callback lookup reads the target under the publication +lock. This does not pin a remote instance for the connection's entire lifetime or +solve remote restart/ID reuse; each send retains its existing lookup behavior. + +Four scenarios check IRQ exclusion at open/release lock-acquisition checkpoints, +release waiting for a callback paused during wakeup, simultaneous first-target +writers plus read/poll/data writes, and failed target validation/nonblocking I/O. +The IRQ checkpoint records the baseline's potential reentrant deadlock rather +than deliberately hanging a host thread. Baseline cross-thread callback/release +lifetime and validation scenarios already passed; their lifetime scheme remains. + +Evidence and limits +------------------- + +Tests were written and run before production changes at c5f7c18e1229: +messaging's initial six scenarios failed (duplicate successes, retirement +overlap, surviving post-cleanup registration, open-before-init/reopen, missing +context constraint); mktty failed two of four (IRQ exclusion and competing target +publication/marked reads). Concurrent-cleanup and broader context checks were +then added to messaging's suite. + +Final checks: all seven messaging and four mktty scenarios passed with plain cc, +GCC -O2 and Clang -O1 ASan+UBSan (including alignment). The existing extracted +ipi_lifetime_test.sh passed with plain cc, and git diff --check passed. No kernel +build was run. + +These are production-extraction behavioral tests with NON-NATIVE boundaries. +pthread mutexes stand in for kernel locks; registry callback registration/RCU +retirement, local IRQ masking, scheduler, allocation and VFS/instance operations +are controlled substitutes. They are not a native kernel, lockdep, KCSAN, physical +IPI, RCU or hardware qualification. The existing ipi_lifetime_test.sh separately +extracts the lower-level registration/removal/dispatch path. + +Remaining gates: native non-RT IRQ/lockdep and callback-drain stress on controller +and worker paths; actual multikernel/TTY use and remote retirement/restart evidence. +PREEMPT_RT remains explicitly UNQUALIFIED: ordinary spinlocks in callback pending, +mktty RX and wait/wakeup paths remain baseline limitations. Host compiler/sanitizer +success must not be interpreted as full RT safety, complete driver/DMA safety, +foreign-writer retirement or a wire/session lifetime protocol. diff --git a/tools/testing/selftests/multikernel/README.cpu-departure b/tools/testing/selftests/multikernel/README.cpu-departure new file mode 100644 index 00000000000000..8b42901cc24682 --- /dev/null +++ b/tools/testing/selftests/multikernel/README.cpu-departure @@ -0,0 +1,43 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +CPU departure transaction checks +=============================== + +Run from a source checkout: + + bash tools/testing/selftests/multikernel/cpu_departure_test.sh + +The harness compiles the checkout's depart_cpu(), cpu_down(), target_store(), +CPUHP_DEPARTED step, physical-to-logical lookup and mk_do_cpu_remove(). Native +teardown, device notification and locks are test doubles. No live CPU, sysfs +control or kernel failure injection is used. + +The contract is that successful departure leaves the CPU offline, departed and +absent before native hotplug serialization ends. Only then can the caller update +ownership. A CPU that was ordinarily offlined cannot be treated as donated: +its physical stop path differs from departure into the multikernel park area. + +Checks cover: + +* Continuous device-hotplug and CPU-map exclusion through withdrawal and device + bookkeeping, without recursively taking the CPU write lock around teardown. +* Successful departure, CPU zero, absent/impossible/out-of-range CPUs, missing + devices, ordinary offline and intermediate CPUHP states. +* Native teardown error propagation and rejection of a bad success postcondition. +* Native state-control rejection of DEPARTED, with ordinary offline/online targets + still routed through their normal entry points. +* Shared multikernel removal accounting on success, offline rejection and error. + +The fake native teardown returns EBUSY, EOPNOTSUPP and EIO unchanged; this checks +propagation, not the native last-CPU, hotplug-disable or callback implementations. +The optional hotplug operation-log allocation fails in this harness; operation +logging is not ownership rollback and is not the property under test. + +Limits +------ + +These are userspace control-flow and lock-contract checks, not scheduler, +lockdep, architecture park-loop or physical quiescence evidence. Live x86 +lockdep and final-stack x86/RISC-V validation remain separate gates. CPU +handback and mailbox quarantine are separate changes: this check does not +qualify either of them. diff --git a/tools/testing/selftests/multikernel/README.cpu-handback b/tools/testing/selftests/multikernel/README.cpu-handback new file mode 100644 index 00000000000000..bd41fb1e3d6faf --- /dev/null +++ b/tools/testing/selftests/multikernel/README.cpu-handback @@ -0,0 +1,70 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +CPU handback and ownership +========================== + +Run from this worktree: + + bash tools/testing/selftests/multikernel/cpu_handback_test.sh + +Actual x86 Kconfig and affected-object build checks: + + bash tools/testing/selftests/multikernel/cpu_hotplug_config_test.sh + +The configuration test also requires OF and OF_FLATTREE with MULTIKERNEL: +the current manifest restore path consumes the boot device tree unconditionally. + +This rejects UP multikernel without CPU hotplug, builds ordinary UP CPU +objects, then restores SMP/hotplug multikernel and builds its resource-hotplug +object. It does not boot a guest or execute CPU transitions. + +Optional userspace sanitizers: + + CC=clang CFLAGS='-fsanitize=address,undefined -fno-omit-frame-pointer -g' \ + bash tools/testing/selftests/multikernel/cpu_handback_test.sh + +The script compiles the actual reclaim_cpu(), mk_do_cpu_add(), mapping and +CPU-set functions with mocked native CPUHP/device operations. Its nineteen +scenarios check caller ordering, not a real CPU transition or concurrency. + +Contract +-------- + +- The resource controller lock covers handback and ownership updates. +- Validate the logical CPU before indexing per-CPU storage. Reserve local + ownership before device registration or native bringup can run. +- Register an absent enumerated CPU under the device-hotplug lock. Presence + restoration, bringup and failure rollback are serialized by the native + device-hotplug/device/CPU-map locks. The CPU write lock is taken separately + for mask changes; _cpu_up() manages its own write-lock section. +- OFFLINE and DEPARTED are the supported starting CPUHP states. Native hotplug + disable, SMT policy, possible-CPU and NUMA checks still apply. Successful + return requires the CPU to be online at CPUHP_ONLINE, with device bookkeeping. +- Known preflight rejection undoes only newly added local ownership; it does + not remove free-pool membership. The attempted output reports entry into + native bringup, not whether firmware definitely started the hart/CPU. +- Preserve pool membership during the wake callback, which selects the x86 + local park slot. Never hold the pool lock across native bringup. After an + attempted failure, retain local ownership and remove free-pool eligibility + before releasing the resource controller lock, even for -EBUSY. +- Restore the original presence mask after a failed absent-CPU bringup only + if the CPU is offline. An incomplete rollback must not make an online CPU + absent. No automatic retry follows a failed attempt; CPUHP rollback and + an errno are not proof that physical execution has stopped. +- Ordinary add_cpu() remains unchanged. The CPU-device NUMA bookkeeping is + shared with the ordinary online callback and runs outside the CPU-map lock. + +Coverage and limits +------------------- + +The fixture checks successful and first-local bringup; allocation and device +registration failure; native hotplug/SMT/NUMA rejection; invalid CPUHP state; +attempted failure, attempted -EBUSY, incomplete rollback and false success; +impossible/out-of-range mappings; successful registration; preexisting +ownership; originally present failure; and already-online idempotence. + +The native _cpu_up() implementation, scheduler, NUMA sysfs link changes and +firmware are not executed by this fixture. Mock lock assertions do not replace +live lockdep. Final x86 and RISC-V stack builds and native lifecycle tests are +separate gates. Mailbox quarantine and park-memory lifetime protection must +also land before claiming safe recovery from an uncertain x86 wake attempt. diff --git a/tools/testing/selftests/multikernel/README.cpu-ownership b/tools/testing/selftests/multikernel/README.cpu-ownership new file mode 100644 index 00000000000000..f8316cb757af0d --- /dev/null +++ b/tools/testing/selftests/multikernel/README.cpu-ownership @@ -0,0 +1,50 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +CPU ownership and x86 handoff regression checks +============================================== + +Run from a checkout without building the kernel: + + sh tools/testing/selftests/multikernel/cpu_ownership_test.sh + CC=gcc CFLAGS=-O2 sh tools/testing/selftests/multikernel/cpu_ownership_test.sh + CC=clang CFLAGS='-O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer' \ + sh tools/testing/selftests/multikernel/cpu_ownership_test.sh + +The runner extracts verbatim production CPU set and transfer functions, CPU +add/remove senders, generic parked confirmation, and the x86 first-boot, +secondary, hot-add and parked-confirmation paths. +Changes to ownership or publication ordering therefore change the tested code. +Mailbox/firmware execution, allocation, pending waits, instance lookup and lock +scheduling are controlled host boundaries. The separate ack_test.sh exercises +the real pending list, message transport framing and deferred workers. + +Before the fix, the first 81 checks reported 25 failures: ownership/tracking +lagged publication, ambiguous handoffs remained assignable, failed or timed-out +requests attempted automatic return, and waiter allocation happened after the +handoff. Expanded coverage includes partial secondary failure, allocation and +mapping failure, confirmed removal and sparse full-width nonrunning transfers. + +Review exposed a consumer interaction: x86 host-slot confirmation deletes an +uncertain member, so forward iteration could skip the next running CPU and +falsely permit image overwrite. An additional production-path regression first +failed with success instead of -EBUSY and zero running-CPU probes. Reverse +iteration now checks that CPU and remains safe when the current member is removed. + +Ownership is reserved before handoff, and uncertain outcomes remain target-owned +and tracked. This deliberately prefers retention to unsafe pool reuse. A failed +tracking allocation after generic reservation can also retain an unmoved CPU; +this is conservative, not a proof that the CPU entered the instance. + +Timeout/cancel removes only a local waiter. It does not cancel queued remote +CPU-start work. Do not use a stopped snapshot or a successful repark alone to +recover an outstanding timed-out add: quiesce such work first. The successful +remove check models an independent acknowledged remove with no outstanding add. + +The post-wake allocation case checks ownership/tracking and waiter storage, +not all allocations: real message framing can still allocate and fail after +handoff, covered here by an injected transport error. + +These deterministic checks do not prove firmware execution, memory ordering, +resource/image lock serialization, teardown, or actual concurrent CPU hotplug. +Those require kernel runtime and failure-injection tests. This patch alone does +not make the lifecycle or reclamation series ready for publication. diff --git a/tools/testing/selftests/multikernel/README.cpu-work b/tools/testing/selftests/multikernel/README.cpu-work new file mode 100644 index 00000000000000..5730ab37101b4e --- /dev/null +++ b/tools/testing/selftests/multikernel/README.cpu-work @@ -0,0 +1,156 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Deferred CPU-request ordering checks +=================================== + +Run from this source tree, without building or booting a kernel: + + sh tools/testing/selftests/multikernel/cpu_work_test.sh + CC=gcc CFLAGS=-O2 sh tools/testing/selftests/multikernel/cpu_work_test.sh + CC=clang CFLAGS='-O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer' \ + sh tools/testing/selftests/multikernel/cpu_work_test.sh + +The runner extracts verbatim CPU/memory/device receive handlers and workers, +admission/quiescence helpers, hotplug init/cleanup, core init and the requested +shutdown worker. It tests real admission, error/free +paths and ACK fields against controlled scheduler, CPU mutation, registration +and transport boundaries. A legal adverse scheduler runs unordered requests +newest-first; the ordered-queue boundary follows the kernel's FIFO contract. +This is not a real Linux workqueue or concurrent hotplug execution test. + +The lock boundary also models hard-IPI context with PREEMPT_RT semantics: +ordinary spin locks may sleep, raw locks may not. Before the review fix, that +check reported 12 invalid lock acquisitions (44 checks, one failure). It passes +with raw submission/detachment/publication locks. This models the API choice; +it is not a reproduced PREEMPT_RT kernel failure or full RT qualification. + +Before the source fix: 35 checks, 14 failures, including a remove ACK preceding +an older queued add and the CPU becoming online afterward. Queue initialization, +registration failure, sparse full-width CPU IDs, worker errors, allocation +failure, short payloads, teardown admission and draining are also exercised. +Cleanup injects an already-entered callback at unregister to check that it +cannot submit after the CPU queue is detached. Kernel callback-object lifetime +and actual IRQ synchronization are not simulated or qualified by that check. + +Behavior and limits +------------------- + +Incoming CPU adds/removes now share alloc_ordered_workqueue, which guarantees +at most one running work item and FIFO queue order. Do not substitute a bound +queue with max_active=1: the in-tree workqueue API documents why that is not a +global ordering guarantee. A remove worker and its ACK follow CPU requests +already admitted ahead of it. The separate ack_test retains real pending/ID +and worker echo coverage with its single-item scheduler boundary. + +Submission and admission closure share an IRQ-safe raw spinlock. Cleanup closes +admission and waits before detaching the CPU queue and unregistering; +destroy_workqueue finishes releasing the CPU queue. Init failure does not +publish a resource handler without its +CPU queue. Rejected requests return -ESHUTDOWN to the receive dispatch; +no success ACK is fabricated. The existing dispatcher logs that error rather +than replying, so a requester may time out and must retain uncertain ownership. + +Requested shutdown now closes incoming resource admission and waits for every +accepted CPU, memory and device worker to finish its mutation, ACK attempt and +request-slot retirement before sending SHUTDOWN_ACK and entering the park path. +Cleanup also drains these jobs before freeing tracking. Memory/device jobs +remain on their existing system queue; CPU jobs retain FIFO ordering. Handler +registration exposes shutdown only after hotplug admission is initialized. + +The shutdown regression schedules shutdown ahead of three older resource jobs. +Before this fix, 68 checks reported 16 failures: shutdown ACK preceded those jobs, +late requests were accepted, and the system handler was exposed before hotplug +initialization. The fixture controls wait progression and injects late receives +during drain. It does not exercise native wait-queue wakeups or kernel scheduling. +Registration/unwind and mixed-resource cleanup with lost ACKs are also covered. + +Admission closure and the in-flight count share the raw lock. Completion is +published only after retiring the request slot; the idle predicate takes the same lock. +Quiescence waits without controller, registry, tracking or admission locks held. +If an accepted operation never returns, no successful shutdown ACK is sent. +No timeout is used to pretend the operation was cancelled. + +Workers also exclude local controller/pool mutations with mk_overlay_mutex. +They use trylock: a controller can hold its local resource lock while waiting +for another kernel's ACK. Waiting for that same lock in incoming workers could +make two such controllers depend on each other indefinitely. A contended worker +returns -EBUSY in its correlated ACK without mutating any resource. ACK delivery +and work retirement happen after releasing the resource lock. Retry is explicit; +this is not a successful mutation or recovery of uncertain ownership. + +The fixture covers all six busy worker paths and a successful retry after the +controller unlocks. It also requires every substituted mutation to run under +the resource lock. All traced direct send/pool callers are within baseline or +overlay resource critical sections. After incoming quiescence, requested +shutdown takes that mutex and retains it until every local CPU is stopped, +excluding local controllers from passing shutdown. Before taking that mutex, +it also calls cpu_hotplug_disable() to drain native CPU hotplug and close native +CPU-up admission, as the normal reboot path already does. A stuck native +hotplug operation must prevent the ACK, not authorize reclamation. The fixture +checks invocation before the ACK, after incoming drain and outside the resource +lock; it substitutes the native helper and does not qualify its kernel locking. +All-hart parked confirmation remains required after acknowledgment. +Voluntary halt, poweroff/reboot, panic and force stop do not use the sleeping +barrier above. The x86 machine-stop path now receives a separate syscore permit +only during normal terminal shutdown, after migrate_to_reboot_cpu() disabled +native CPU hotplug and a nonblocking resource-mutex acquisition succeeded. +It holds that mutex through final parking; admitted resource workers cannot +perform mutations while it is held. A panic, ordinary kexec or contended exit +does not gain permission to send HALTED. Panic/kexec are rechecked immediately +before consuming a previously published permit. Unqualified exits still park +locally and leave the host retaining the instance; they are not recovery. +Requested shutdown remains on its existing sleeping drain-and-ACK path. + +Run the extracted architecture-to-generic path checks with: + + sh tools/testing/selftests/multikernel/voluntary_shutdown_test.sh + +Each case runs in a fresh process and uses the actual x86 reboot registration, +syscore callback, machine operations and generic notification/parking bodies. +Checks cover halt/poweroff/restart, direct emergency, panic and kexec before +and after qualification, contention, nonterminal state and host exclusion. +The controlled native-hotplug and SMP boundaries do not exercise actual +cpu_hotplug_disable(), remote CPUs, syscore ordering or hardware memory +ordering. Fresh native x86 shutdown/retention and lockdep tests remain required. +No generic syscore hook is registered on RISC-V; its #28 transport stays separate. +No wire layout or request-ID change is introduced; no success ACK is fabricated +for late rejected resource requests. + +Full halt/HALTED/force-halt settlement, destruction/admission, callback and +ring/image lifetime, other shutdown quiescence and cross-kernel dependencies remain +release gates. All parked CPUs or a reset ring alone are not a queued-work +cancellation proof. Kernel lockdep, x86 runtime/concurrency, QEMU/OpenSBI and +same/different-Image 50-cycle failure/stress tests remain required. + +Allocation-free receive admission +--------------------------------- + +CPU and memory/device requests each have MK_IPI_RING_SIZE preinitialized slots. +They retain their ordered CPU queue and existing system queue respectively. +Under the admission raw lock, a handler claims a free slot, copies the complete +payload and request ID, counts the request and queues it. All six receive paths +and their rejection paths avoid allocation, freeing and work initialization. +Short payloads return -EINVAL; a closed subsystem returns -ESHUTDOWN; a full +class returns -ENOMEM without modifying or overwriting an admitted request. +No success ACK is synthesized for rejection: the existing sender timeout and +uncertain-ownership retention behavior applies. Capacity is deliberately bounded +at one receive-ring length per class; this is not unlimited burst admission. + +Each slot remains busy through mutation and its correlated ACK attempt, even +when delivery fails. Retirement drops busy/in-flight under the admission lock, +then wakes waiters in process context. No worker reads its payload afterward. +A requeued slot keeps the same callback and queue and is never reinitialized in +receive context; this satisfies the workqueue non-reentrancy contract even when +submission races the tail of the previous callback. Storage is static and is +not freed while workqueue bookkeeping can still reference the work item. +The pool is initialized once before publishing the receive handler. + +The cpu_work fixture additionally covers all six receives with allocation +unavailable, copied payload/sender lifetime, both full classes, no overwrite on +rejection, reuse on the same queue, FIFO request IDs, and saturated cleanup with +lost ACKs. It instruments allocator entry in modeled hard-IPI context, including +closed-admission paths. The queue scheduling and non-reentrancy guarantee are +modeled boundaries; native lockdep and actual resource hotplug remain separate +qualification requirements. The retained process-context tracking allocations +are not removed by this change. Deferred PCI and VSOCK receive paths are also +outside this change. diff --git a/tools/testing/selftests/multikernel/README.halt-teardown b/tools/testing/selftests/multikernel/README.halt-teardown new file mode 100644 index 00000000000000..8b04d94f597811 --- /dev/null +++ b/tools/testing/selftests/multikernel/README.halt-teardown @@ -0,0 +1,101 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Confirmed halt and fail-safe teardown +==================================== + +Run halt_settlement_test.sh, teardown_test.sh and memory_teardown_test.sh with +bash. CC and CFLAGS select host compiler/sanitizers. Runners extract production +functions; firmware, transport, kernel scheduler and resource allocation are +controlled boundaries. These are not native kernel or QEMU lifecycle tests. + +Halt settlement requires confirmation of every assigned CPU and orphaned park +tracking before publishing LOADED. Requested, voluntary and supported force +halt failures keep the existing non-reusable state. A lost ACK is not remote +cancellation. Unsupported architectural force stop returns -EOPNOTSUPP; marker +arming failure must not send force-stop interrupts. CPU ownership tests also +cover an assigned CPU missing from park tracking and the pre-first-spawn host +park slot. + +After CPU confirmation, mk_ipi_poll dispatches one captured local-inbox prefix +before permitting reuse. A claimed but unpublished slot returns -EBUSY and +retains resources; it is never skipped or cleared. Existing PCI admission pins +and revocation then protect dispatched requests across grant/ID reuse. See +README.pciback-lifetime for the bounded-drain tests and remaining assumptions. + +Repark retirement additionally requires a destination-loop self-repark claim. +The old mailbox clears ready before jumping away, so its claim alone permits +premature reuse of the old park page/page table. repark_arrival_test.sh exercises +the production publication and tracking paths with delayed destination arrival: +source claim plus destination timeout must retain tracking, while successful +arrival needs both claims. This is a controlled mailbox boundary, not native +assembly execution or a reproduced physical race. The park ABI is unchanged. + +Image unload admission is centralized before kimage_free() unlinks anything. +ACTIVE is refused. FAILED retains its Image until stop confirmation succeeds. +Uncertain published memory/device operations also pin the instance even after +CPU stop; README.resource-transfers describes this reboot-only quarantine. +The Image locking fixture covers these paths plus failed load and manifest +finalization. Normal/crash callers remain outside the multikernel resource +lock, and their successful free returns zero. + +FAILED is not evidence that no CPU executed: an unsuccessful spawn may have +run preparation code or moved CPUs onto instance control memory. An architecture +must not return a failed spawn with an uncancelled child-Image launch that can +arrive after confirmation. The current x86 boot publication revokes an unclaimed +wake on timeout and treats a concurrent claim as success; the RISC-V path returns +the final SBI start result directly, relying on accurate firmware responses. +Neither has a later error point after reported acceptance of the boot launch. +All-owned/tracked-CPU confirmation and separate control-memory retirement remain +required. This contract assumes exclusive ownership and a quiescent previous run; +it does not authorize retrying an ACTIVE instance after a lost shutdown ACK. + +Destruction requires READY with no Image. Stop/repark proof precedes IDR or +kernfs removal. Registry locking is dropped for firmware waits and kernfs +draining, while the resource lock excludes competing controllers. Resource +retirement is synchronous; only the final record and persistent boot pages +await lookup reference release and an RCU grace period. A refused memory release +keeps the pool, region metadata and control allocation intact. The memory test +can characterize a saved production revision with MK_CORE_REV=. + +Run kernfs_snapshot_test.sh with sh to check the production root and instance +device_tree read callbacks. Each generation holds the resource mutex; contended +reads return -EAGAIN so a controller holding that mutex can drain kernfs without +waiting on a reader blocked behind it. Retry after contention clears. The mutex +is released before copying the generated buffer. Separate partial reads still +generate separate snapshots, not a stable snapshot for the whole open file. +The fixture checks exclusion at a stubbed DT generator, controller-held-lock +contention, retry, generation errors, offsets and buffer release using a host +pthread mutex. It does not exercise the real DT emitter or native kernfs drain. + +The unused bulk kernfs cleanup entry point refuses live boot records, loaded +Images and uncertain states. It does not supply a new inbox/IRQ retirement +contract. Unexpected final-reference release without stop proof retains +allocations rather than reclaiming executing memory; that diagnostic fallback +is not a recoverable management API or proof of leak-free error handling. + +Voluntary HALTED work is allocated with the instance record, before publication. +IRQ admission takes a reference to that exact record and coalesces duplicates +through an atomic pending flag; it does not allocate or resolve the ID later. +The pending flag remains set while the worker holds the resource mutex and +attempts stop confirmation. It excludes a new start, including a start after +another controller has settled the same record. Completion clears the flag and +drops the reference. Failed confirmation retains ACTIVE/FAILED and allocations; +a queue rejection drops the reference without publishing a reusable state. +The execute caller must still confirm stop and drain the inbox before copying +an Image, as before. This is not a wire session or peer-authentication protocol. + +Remaining qualification gates include native x86 concurrency/lockdep, QEMU +OpenSBI HSM transitions, foreign ring producers, driver/DMA quiescence, stale +queued notifications across re-spawns, and 50-cycle same/different Image reuse. +No #7 session protocol, wire-layout change or mixed-peer compatibility claim +is introduced. RISC-V runtime additionally needs the explicitly separate +modern #26 loader and #27/#28 test integration dependencies. + +Pool unmapping is fallible. Control-block unmap failure retains memory-region +metadata, the control allocation and the instance pool, and propagates through +re-reservation and baseline replacement. Destroy attempts retirement before +removing the record from kernfs/IDR; memory readers remain excluded by the +controller mutex. CPU/device return can already have completed on an error. +Unexpected failure after the final reference retains the record/allocations +as a diagnostic leak; it is not a recovery API. The x86 pool-control wrapper +also refuses to free physical memory after an unexpected unmap error. diff --git a/tools/testing/selftests/multikernel/README.image-lock b/tools/testing/selftests/multikernel/README.image-lock new file mode 100644 index 00000000000000..702af57fa887c1 --- /dev/null +++ b/tools/testing/selftests/multikernel/README.image-lock @@ -0,0 +1,67 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Controller resource/image serialization checks +============================================== + +Run from a checkout without building or booting the kernel: + + sh tools/testing/selftests/multikernel/image_lock_test.sh + CC=gcc CFLAGS=-O2 sh tools/testing/selftests/multikernel/image_lock_test.sh + CC=clang CFLAGS='-O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer' \ + sh tools/testing/selftests/multikernel/image_lock_test.sh + +The runner extracts verbatim production file-load, allocation/error cleanup, +MK image release, lookup, exec, instance lookup/state and overlay-rmdir functions. +It uses real pthread mutexes and a second thread entering the actual overlay +rmdir control flow during load, exec, unload or failed-load cleanup. The +contender announces its lock attempt and cannot mutate until the image operation +unlocks. Without the patch, the initial harness reported 166 failures in 842 +checks, including actual contender progress inside an image operation. + +Assertions check kexec -> resource -> registry acquisition, reverse release, +no image work under the registry lock, cleanup before unlocking and exclusion +from overlay rollback. Load/exec failure stages, unknown IDs, empty CPU sets, +mapping/allocation failure and early permission/flag/busy exits are covered. +Normal/crash images retain their existing absence of a resource lock; cleanup +is now inside kexec_lock, as it already is for legacy kexec_load. + +Command-line cleanup uses the extracted kimage_file_free_cmdline helper and a +small tracked allocation supplied by the segment-preparation boundary. Early +preparation, validation and control/swap-page failures must free it for normal, +crash and multikernel images (crash images have no swap page). Successful loads +must retain it until unload. Post-load cleanup intentionally retains this buffer; +the early allocation unwind must release it before freeing the image directly. +This is a host cleanup regression check, not a kernel syscall reproduction. + +Kernel allocation, segment copying, firmware, pool mutation, instance registry +storage and kernfs removal are controlled host boundaries. The real MK release +and can-free helpers are exercised; the general kimage_free dispatcher is a +boundary. The +fixture contains only the fields reached by the extracted functions. Kernel +sign-compare and unused-argument diagnostics from those functions are excluded +from the host compiler's -Werror policy, not from kernel build validation. +The host mutexes check control-flow locking, not kernel lockdep or scheduling. + +Remaining release gates +---------------------- + +This mutex covers local controller overlay/baseline resource changes and MK +image load, exec and cleanup. Incoming resource workers now refuse a contended +resource lock with -EBUSY rather than mutate through a controller operation. +Active Image unload and execution are refused. An execution request cannot +bypass halt settlement merely because individual CPUs look parked; it returns +-EBUSY before confirmation, Image writes, manifest updates or ring resets. +The ACTIVE rejection is covered even when the substituted confirmation would +succeed. An uncertain failed start must first confirm +all assigned/tracked CPUs stopped. Manifest-finalization failure prevents +starting the Image, and a partial start failure retains FAILED state rather +than publishing a reusable LOADED state. These paths are covered here with +controlled stop/start boundaries, not actual firmware execution. + +Halt/HALTED/force-halt settlement and destruction are tested separately by +halt_settlement_test.sh, teardown_test.sh and memory_teardown_test.sh. They do +not cancel queued remote work or qualify cross-kernel producer retirement. + +A substituted successful firmware start does not prove CPU execution or safe +reclamation after a partial start. Local threading, sanitizers and kernel links +do not replace x86 or QEMU/OpenSBI runtime, failure-injection and re-spawn tests. diff --git a/tools/testing/selftests/multikernel/README.ipi-lifetime b/tools/testing/selftests/multikernel/README.ipi-lifetime new file mode 100644 index 00000000000000..110d5335be3a1b --- /dev/null +++ b/tools/testing/selftests/multikernel/README.ipi-lifetime @@ -0,0 +1,55 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +IPI callback and local writer lifetime +====================================== + +Run ipi_lifetime_test.sh, instance_lookup_test.sh, instance_release_test.sh +and ipi_writer_test.sh with bash. CC and CFLAGS select the host compiler and +sanitizers. Each runner extracts the production functions, not another +implementation. Kernel RCU, scheduler, IDR, physical IPI and hardware boundaries +are substituted; these tests do not qualify kernel IRQ/RCU behavior. + +The callback test holds an entered callback while another pthread unregisters +it. The baseline freed its handler before that callback returned. The fixed +path unlinks under the writer lock, then waits outside that lock for an RCU +grace period covering lookup and callback invocation. The test requires an +active protected reader, not merely entry into the grace-period shim. Caller +context remains live until unregister returns. Unregister is process-context +only, must not run inside a callback, and cannot hold callback-owned locks. + +Lookup now pins the instance in an RCU read-side section with +kref_get_unless_zero(), without the sleeping registry mutex in a hard-IPI +sender. IDR writers reserve an empty slot under the registry mutex and publish +only after the instance ID is initialized. No allocator runs under a raw +membership lock. Normal destruction returns resources synchronously, so later +fragments of the same overlay can reuse them. The last reference defers record +and persistent boot-page reclamation to process context after an RCU grace +period. Early-publication failure also drops its reference instead of freeing +an object another lookup can have pinned. + +Local producers hold the per-instance raw ring lock through payload publication, +but release it before ringing a doorbell or logging. Reset closes publication +under that lock, clears the persistent ring with IRQs enabled, then republishes +it and a cached physical target. Atomic senders do not walk a mutable CPU set. +The fixture exercises send, closed admission and reset/republication ordering. +Message sends now build the header and payload directly in the reserved slot, +under the same producer lock, without slab allocation or a full-size stack +buffer. Publication follows both copies. Raw IPI callers keep their existing +interface. The wire size remains sizeof(struct mk_message) plus payload length, +with payload at its original flexible-array offset and zero tail padding. +NULL payloads retain the previous zero-fill behavior. A held-instance send does +not acquire or drop the caller's reference; an ID send pins through publication. +The fixture checks 16 size/NULL-payload/route combinations, full-size messages, +overflow rejection, queue/deactivation errors and bytes at release publication. +The target cache is refreshed at publication/reset and local CPU ownership +changes, except for the firmware-described parent doorbell target. Runtime +boot-hart reassignment remains a hardware qualification gate. + +The host-owned ring allocation now outlives the Image grant on this upstream +base. Host instance references therefore pin its storage. These local locks +do not fence foreign kernels writing the same physical ring: remote producer +retirement, requester restart and the shared receive-ring reset remain protocol +limitations, not solved by resetting indices or confirming receiver CPUs. +No #7 protocol dependency, new message layout or mixed-peer claim is introduced. +mktty IRQ locking and higher-level messaging registration admission are separate +baseline gates; callback retirement alone does not qualify them. diff --git a/tools/testing/selftests/multikernel/README.mailbox b/tools/testing/selftests/multikernel/README.mailbox new file mode 100644 index 00000000000000..e070f4872e7132 --- /dev/null +++ b/tools/testing/selftests/multikernel/README.mailbox @@ -0,0 +1,97 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +x86 mailbox ownership and quarantine +==================================== + +Run these host-side checks from the worktree: + + bash tools/testing/selftests/multikernel/mailbox_test.sh + bash tools/testing/selftests/multikernel/mailbox_lifetime_test.sh + bash tools/testing/selftests/multikernel/mailbox_reader_test.sh + bash tools/testing/selftests/multikernel/pool_return_test.sh + +The reader test requires x86-64 userspace and returns kselftest skip status 4 +on other architectures. C checks also support CC=clang and sanitizer CFLAGS. + +Protocol +-------- + +The existing 32-bit ready field carries these states, without moving fields: + + IDLE -> BUSY (publisher reservation) -> PENDING + -> BUSY (reader claim) -> ACK -> IDLE (publisher consumption) + +Every writer reserves the slot before changing dispatch fields. Cross-kernel +writers use this atomic reservation in addition to local park_lock ordering. +The reader's initial APIC comparison is only a filter: after PENDING -> BUSY +succeeds it re-reads the target and all dispatch data. A wrong-target reader +returns BUSY -> PENDING with compare/exchange. It cannot undo quarantine. + +The reader stages all dispatch registers before BUSY -> ACK. It jumps only +if that compare/exchange succeeds. A publisher may reuse the dispatch fields +after consuming ACK, because no successful reader loads them afterward. +Atomic compare/exchange orders preceding stores before PENDING publication. + +ACK is not proof that a CPU has left its old park code or page table. A repark +to another context still requires an independent self-repark acknowledgment +from the destination before old-context tracking can be removed. + +The bounded wait is one second. After its final ACK-consumption attempt fails, +the publisher exchanges the state to QUARANTINED. There is no retry loop racing +wrong-target readers. A just-arrived ACK can lose this last race; the result is +conservative retention, not a false success. A reader that already ACKed may +still dispatch, so timeout never authorizes reclaiming its code or data. + +Fail-closed scope +----------------- + +Timeout quarantines the selected slot, this kernel's own context and its local +pool slot when present. A remote wake may still reference this kernel's +trampoline or pool page tables; an empty free-CPU set does not prove those +resources idle. This deliberately sacrifices further pool use after uncertainty. +A spawn's own-context marker is also visible to its parent. + +No new writer reserves a quarantined slot. An already-reserved publisher can +finish stores after a concurrent quarantine, but its final compare/exchange +cannot republish or reset the marker. Resources remain pinned regardless. +Setup, spawn, confirmation, repark and pool teardown reject non-idle slots. +Instance teardown checks the context even if CPU tracking is empty. +Pool retirement also requires locally owned CPUs to be online: CPUHP may fail +after the mailbox ACK while the CPU still references pool code. This check +also conservatively blocks retirement for ordinary offline local CPUs until +they are onlined. It does not take CPUHP locks under the park lock; resource +serialization excludes new donations, and ordinary native offlining does not +send CPUs into pool memory. +Confirmation selects the recorded slot once; it does not retry on another +slot after timeout or repair uncertain tracking by guessing a location. + +Recovery is a full host reboot. There is no sysfs reset, automatic retry, +generation reset, or old-peer compatibility mode. CPUHP rollback, absence +from a CPU set and a late shutdown ACK are not mailbox retirement proofs. + +Compatibility and landing +------------------------- + +This changes shared protocol semantics, even though context offsets and size +are unchanged. Host, spawn and nested kernels must all use the updated reader +and writers. Different configurations are permitted only with compatible +context layout and this same protocol; mixing old/new kernels is unsupported. +The C writers, assembly reader and shared state definitions must land together. + +Evidence limits +--------------- + +mailbox_test extracts the actual writer/setup functions, using simulated poll +outcomes. mailbox_lifetime_test executes real release/teardown paths against +counted allocation boundaries, including empty tracking. mailbox_reader_test +executes actual primary/secondary reader instructions in userspace, with a +capture trampoline; it excludes CR3, descriptor setup, MONITOR/MWAIT and repark. +pool_return_test checks the actual pool retirement predicate with retained +offline or invalid local CPU ownership, despite empty free/child sets. +Ownership and repark-arrival fixtures mock the mailbox primitive and cannot +prove this protocol independently. + +These checks and object builds are not live lockdep, native halt, physical CPU +retirement or end-to-end spawn qualification. Final-stack tests must use fresh +VMs for quarantined failure cases, and verify retention rather than in-place +recovery. Earlier v21 runtime results do not qualify this protocol change. diff --git a/tools/testing/selftests/multikernel/README.pciback-lifetime b/tools/testing/selftests/multikernel/README.pciback-lifetime new file mode 100644 index 00000000000000..f7c3505f01b875 --- /dev/null +++ b/tools/testing/selftests/multikernel/README.pciback-lifetime @@ -0,0 +1,102 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +PCI backend request retirement +============================== + +Run: bash tools/testing/selftests/multikernel/pciback_lifetime_test.sh + +The fixture extracts the production request, deferred worker, reference release, +unlend and inbox-draining functions. It uses pthread locks and controlled +condition variables where the kernel uses locks and a workqueue. PCI config +access, reset, IRQ delivery and message decoding are modeled. + +Four scenarios check: + +* A queued request pins its original device grant. Unlend followed by a new grant + with the same instance ID and BDF cannot redirect that request, or send its ACK + into a later boot. +* A negative reply requires no config access and does not queue an ID lookup. +* Unlend revokes admission under the registry spinlock, then waits outside that + lock for a slow config access and its reply before resetting the function. +* Direct and deferred config writes preserve the request value; single-function + retirement releases the same pins as whole-instance retirement. + +The existing startup and image-lock tests also check PCI retirement before +publishing a reusable instance state and before freeing Image segments. + +Requests still in the shared ring +--------------------------------- + +Pinning a deferred worker cannot protect a request that has not reached its +handler. Generic parked confirmation therefore drains a captured prefix of the +local inbox after confirming every owned/tracked CPU stopped. With the controller +resource mutex held, the retiring peer can no longer publish requests or acquire +a replacement grant. The consumer lock also waits for any entered callback. +PCI requests reach the old grant before unlend revokes it; queued workers retain +that grant and cannot access or reply through a later one. + +A claimed but unpublished slot returns -EBUSY, leaving the slot and the following +traffic intact. Callers retain the Image, grants and instance ID. If a live peer +publishes later, retry can succeed; an abandoned slot can block retirement until +controller reboot. The shared ring is never reset to recover one peer. + +The captured prefix contains at most ring-size-minus-one entries, including +malformed ones. New traffic from other peers cannot extend this drain; it remains +queued for a later poll or doorbell. This bounds entries, not callback duration +or lock wait. Successful polling is not global ring quiescence. Deferred handlers +still need their own admission/identity and retirement guarantees. + +Additional scenarios exercise old PCI requests before unlend/re-lend, unrelated +messages, unpublished slots, late publication, partial prefixes, wraparound, +continuous valid/malformed arrivals and callback reentry. cpu_ownership_test +checks that failed CPU confirmation prevents draining and a blocked drain cannot +report reusable confirmation. + +Limits +------ + +These checks do not prove native lockdep behavior, PCI reset/DMA quiescence, or +RISC-V PCI assignment. Real cross-kernel producers, memory ordering, interrupt +delivery and CPU-stop proof need runtime qualification. A stopped peer must not +be able to publish via an untracked CPU or another kernel. Native device-remove +ACKs must mean the remote driver stopped issuing config requests. Delayed +resource-hotplug ACKs, partial overlay rollback and stale deferred non-PCI work +are separate lifetime gates; a passing fixture is not end-to-end safety. + +Bounded deferred requests +------------------------- + +Nested kernels use MK_IPI_RING_SIZE preinitialized system-workqueue slots. +A free slot has no device pin. Under mk_pciback_lock, admission takes the grant +reference and copies the request/subtype into a slot before queueing it. The +worker still serializes access and its ACK with revocation via the grant's +io_lock. It releases the grant before marking its slot free under the raw lock. +No request field is used afterward. Each work item keeps its callback and queue +and is never reinitialized in receive context, preserving workqueue +non-reentrancy if another request arrives before the old callback returns. +The slots have static lifetime and are initialized before the early PCI handler +registration. Registration errors are still propagated. + +Full capacity drops the new request without taking a reference or sending a +success reply, matching the former allocation-failure behavior. The requester +can time out; no reset, grant reuse or recovery is implied. Direct config access +and negative replies remain synchronous because frontends may poll with +interrupts off. Process-context grant allocations and final reference release +remain; only IRQ-side work allocation/free/init is removed. + +The handler copies the wire payload into aligned local storage before reading +its 64-bit sequence number or dispatching requests. The packed message layout +is unchanged. A deliberately misaligned ACK reproduces undefined behavior +before this copy; short messages are rejected as before. + +The fixture now extracts the real payload/work definitions and early init, +checks allocation exhaustion without IRQ allocator entry, saturation without +overwrite or extra pins, copied request values, slot reuse, saturated revocation +and replacement-grant isolation, and unaligned request/ACK decoding. Existing +pthread retirement races and bounded ring-prefix tests remain. Run ASan/UBSan: + + CC=clang CFLAGS='-O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all' \ + bash tools/testing/selftests/multikernel/pciback_lifetime_test.sh + +The scheduler remains controlled; PCI reset/DMA and actual nested-kernel config +forwarding are not qualified by the host fixture. diff --git a/tools/testing/selftests/multikernel/README.resource-transfers b/tools/testing/selftests/multikernel/README.resource-transfers new file mode 100644 index 00000000000000..7cda2651f503de --- /dev/null +++ b/tools/testing/selftests/multikernel/README.resource-transfers @@ -0,0 +1,62 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Uncertain resource transfer retention +==================================== + +Run resource_transfer_test.sh and overlay_retention_test.sh with sh. They +extract the actual controller senders and overlay lifetime functions. Transport, +native resource operations and kernfs are controlled boundaries. No test here +boots a kernel, runs a delayed remote worker or proves PCI/DMA quiescence. + +All six CPU/memory/device controller senders require mk_overlay_mutex, as do +their overlay callers and Image operations. A successful ring enqueue is +publication; transport errors currently occur before a slot is published. +Cancelling a pending waiter does not cancel a published request. + +Memory and device ownership must be reserved before publication. Device add +also reserves ownership before admitting config access. Before publication, +allocation or send errors undo the reservation; failed cleanup retains it and +reports the cleanup error. After publication, any unsuccessful wait retains +ownership and marks the instance resource_uncertain. A negative ACK cannot +certify rollback: the receiver can have changed one memory piece, failed the +next, and failed to restore the first. A timeout permits later worker execution. +Memory removal therefore keeps its range reserved against Image allocations; +device failures retain lending and do not reset a potentially busy function. +The free-pool device-add path checks all other instance grants before touching +the local function. Otherwise a second overlay could make a retained function +assignable again without addressing the quarantined instance at all. + +The flag blocks further CPU/memory/device controller changes, start, Image +reuse/unload and destruction through the common confirmation gate. Freeing all +instance memory also checks it. Even confirmed stopped CPUs cannot prove a +resource rollback or DMA settlement. There is no automatic flag clearing: +recovery requires rebooting the controller. Graceful/force halt can still request +that CPUs stop, but settlement returns -EBUSY and does not publish LOADED. +This deliberately sacrifices availability, including for a negative ACK that +happened before any remote change. A future recovery protocol must distinguish +that case and prove all admitted resource work has retired before granting reuse. + +Overlay records are allocated and populated while inactive, before resource +operations begin. Record allocation failure cannot strand applied changes. +An apply failure can leave a partially applied transaction; it remains visible +as failed. rmdir only inverses an applied transaction, and the first inverse +failure permanently marks it failed. Repeating a failed inverse could undo +operations that never ran, or undo an already-reversed item twice. The unused +overlay exit/discard path was removed; it could free records after failed +rollback and before draining attribute readers. No per-operation transaction +journal or automatic retry is provided. Successful removal still drains kernfs +readers outside mk_overlay_mutex before freeing the transaction. + +Coverage: negative ACKs and timeouts for all four memory/device operations, +ownership before lending/publication, competing reservations, attempts to +reinsert a retained PCI grant through the free-pool path, unpublished error +cleanup, cleanup failures, resource quarantine across state changes, metadata +allocation failure before overlay effects, partial apply/inverse and repeated +removal. cpu_ownership_test, memory_teardown_test and start_lifetime_test exercise +the actual reuse gates. image_lock_test checks controller serialization with +real host mutexes; its transaction begins in APPLIED state. + +Remaining qualification: actual delayed/partial receiver work, live lockdep, +hardware PCI reset/DMA and native/QEMU resource cycles. README.pciback-lifetime +describes the separate stopped-peer inbox drain before PCI grant reuse. It does +not establish retirement of deferred resource workers or cancel published work. diff --git a/tools/testing/selftests/multikernel/README.vsock-rx b/tools/testing/selftests/multikernel/README.vsock-rx new file mode 100644 index 00000000000000..cfdf0afa1f4c61 --- /dev/null +++ b/tools/testing/selftests/multikernel/README.vsock-rx @@ -0,0 +1,49 @@ +SPDX-License-Identifier: GPL-2.0-only + +Multikernel VSOCK receive context +================================ + +MK_MSG_NETWORK callbacks run under the raw IPI drain lock. Allocating an skb +with GFP_ATOMIC or taking an ordinary skb-queue spinlock there can violate +the lock nesting rules. The callback now validates the message/header lengths +and copies the complete packet into a preallocated record FIFO under a raw +lock. It does not retain the shared IPI slot, allocate/free an skb, or enter +the socket stack. The payload format and stream operations are unchanged. + +The FIFO occupies MK_IPI_RING_SIZE * MK_MAX_DATA_SIZE bytes (256 KiB), with a +two-byte prefix per record. It holds 64 maximum-size packets or more smaller +packets. Full capacity drops a whole new packet without overwriting older +records or claiming success. A send accepted by the IPI transport still does +not guarantee remote socket delivery; overflow or allocation failure can cause +a connection to time out. This change does not add stream retransmission or +repair the transport's simplified flow control. + +The existing work item is the sole consumer. It peeks the head length under +the raw lock, allocates in process context, then copies/removes that same head. +Producers only append; they cannot change it during allocation. Allocation +failure discards that record and continues. Each invocation consumes at most +64 records and requeues if needed. The fixed callback and workqueue preserve +workqueue non-reentrancy, including a receive racing the worker's final empty +check. No packet-sized kernel stack buffer is introduced. + +Initialization creates the queue and registers the transport before publishing +the message callback; failure unwinds those steps. Exit unregisters/drains the +callback first, cancels work and destroys the queue, then unregisters the +transport. No worker can enter the socket stack after transport removal. +Queued raw bytes own no skb or external reference and need no allocator cleanup. + +Run: + + bash tools/testing/selftests/multikernel/vsock_rx_test.sh + CC=clang CFLAGS='-O1 -g -fsanitize=address,undefined -fno-sanitize-recover=all' \ + bash tools/testing/selftests/multikernel/vsock_rx_test.sh + +The fixture extracts the actual receive/worker/init/exit functions and wire +header. It checks malformed input, copied data lifetime, FIFO saturation and +reuse, worker budget, allocation failure, an arrival during allocation and at +the final empty check, initialization errors, and entered callbacks during +unregister. FIFO, workqueue, IRQ and socket boundaries are modeled. Native +qualification must enable CONFIG_VSOCKETS and CONFIG_MULTIKERNEL_VSOCKETS; +passing lifecycle tests with those options disabled is not VSOCK evidence. +This fixture does not qualify socket protocol races, full streams, physical +hardware, peer identity/reuse, or real module unload concurrency. diff --git a/tools/testing/selftests/multikernel/ack_test.c b/tools/testing/selftests/multikernel/ack_test.c new file mode 100644 index 00000000000000..6a466aed6ddb71 --- /dev/null +++ b/tools/testing/selftests/multikernel/ack_test.c @@ -0,0 +1,907 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* See README.ack: real pending/dispatch code, deterministic kernel boundaries. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* tools/ uses uint64_t; production printk formats use the kernel's __u64. */ +#define u64 __u64 +#define U64_MAX (~(u64)0) +#define GFP_KERNEL 0 +#define GFP_ATOMIC 0 +static bool allocation_fails; +static bool test_hardirq; +static unsigned int irq_allocations; +#define kzalloc(n, flags) \ + ((void)(flags), irq_allocations += test_hardirq, \ + allocation_fails ? NULL : calloc(1, n)) +#define kmalloc(n, flags) \ + ((void)(flags), irq_allocations += test_hardirq, \ + allocation_fails ? NULL : malloc(n)) +#define kfree free +#define pr_err(...) fprintf(stderr, __VA_ARGS__) +#define pr_warn(...) fprintf(stderr, __VA_ARGS__) +#define pr_info(...) \ + do { \ + } while (0) +#define pr_debug(...) \ + do { \ + } while (0) +#define DEFINE_SPINLOCK(name) bool name +#define spin_lock_irqsave(lock, flags) \ + do { \ + assert(!test_hardirq); \ + assert(!*(lock)); \ + *(lock) = true; \ + (flags) = 0; \ + } while (0) +#define spin_unlock_irqrestore(lock, flags) \ + do { \ + assert(*(lock)); \ + *(lock) = false; \ + (void)(flags); \ + } while (0) +typedef struct { + bool held; +} test_raw_lock_t; +#define DEFINE_RAW_SPINLOCK(name) test_raw_lock_t name +static void test_raw_lock(test_raw_lock_t *lock) +{ + assert(!lock->held); + lock->held = true; +} +static void test_raw_unlock(test_raw_lock_t *lock) +{ + assert(lock->held); + lock->held = false; +} +#define raw_spin_lock_irqsave(lock, flags) \ + do { \ + test_raw_lock(lock); \ + (flags) = 0; \ + } while (0) +#define raw_spin_unlock_irqrestore(lock, flags) \ + do { \ + test_raw_unlock(lock); \ + (void)(flags); \ + } while (0) +#define msecs_to_jiffies(ms) (ms) +#define DECLARE_WAIT_QUEUE_HEAD(name) int name +#define wake_up_all(wait) ((void)(wait)) +/* This fixture executes one job at a time; the multi-job drain is tested separately. */ +#define wait_event(wait, condition) \ + do { \ + (void)(wait); \ + assert(condition); \ + } while (0) + +struct completion { + unsigned int done; +}; + +static void (*at_timeout)(void); +static void (*at_completion)(void); + +static void init_completion(struct completion *c) +{ + c->done = 0; +} + +static void complete(struct completion *c) +{ + c->done++; +} + +static unsigned long wait_for_completion_timeout(struct completion *c, + unsigned long timeout) +{ + if (c->done) { + c->done--; + if (at_completion) { + void (*hook)(void) = at_completion; + + at_completion = NULL; + hook(); + } + return timeout ? timeout : 1; + } + /* Inject an ACK after timeout is decided, before pending list removal. */ + if (at_timeout) { + void (*hook)(void) = at_timeout; + + at_timeout = NULL; + hook(); + } + return 0; +} + +struct work_struct { + void (*func)(struct work_struct *work); +}; +struct mk_instance { + int id, refs; + struct work_struct halted_work; + atomic_t halted_pending; +}; +#include "ack-types.h" + +static struct mk_ipi_data sent; +static int sent_to; +static int send_error; + +/* Transport boundary: capture real mk_send_message() output for replay. */ +static int mk_send_message_ipi(int instance_id, struct mk_instance *instance, + const struct mk_message *message, const void *payload) +{ + size_t size = sizeof(*message) + message->payload_len; + struct mk_message *msg = (struct mk_message *)sent.buffer; + + assert(size <= sizeof(sent.buffer)); + sent_to = instance ? instance->id : instance_id; + sent.type = message->msg_type; + sent.data_size = size; + memset(sent.buffer, 0, size); + memcpy(msg, message, offsetof(struct mk_message, payload)); + if (payload) + memcpy(msg->payload, payload, message->payload_len); + return send_error; +} + +#include "ack-messaging.h" + +/* Only the hardware mutation, scheduler and final parking are substituted. */ +struct workqueue_struct { + int unused; +}; +static struct workqueue_struct cpu_queue; + +static struct work_struct *queued; +static int mutation_result, mutations; +static jmp_buf parked; +static struct { + int id; +} self = { .id = 65537 }, *mk_self = &self; + +static int mutate(void) +{ + mutations++; + return mutation_result; +} + +static bool mk_overlay_mutex; +static void cpu_hotplug_disable(void) +{ + assert(!mk_overlay_mutex); +} + +#define mutex_lock(m) \ + do { \ + assert(!*(m)); \ + *(m) = true; \ + } while (0) +#define mutex_trylock(m) (*(m) ? false : (*(m) = true)) +#define mutex_unlock(m) \ + do { \ + assert(*(m)); \ + *(m) = false; \ + } while (0) + +#define mk_do_cpu_add(...) mutate() +#define mk_do_cpu_remove(...) mutate() +#define mk_do_mem_add(...) mutate() +#define mk_do_mem_remove(...) mutate() +#define mk_do_device_add(...) mutate() +#define mk_do_device_remove(...) mutate() +#define mk_manifest_phys() 0 +#define mk_pcifront_add_function(...) mutate() +#define PCI_SLOT(devfn) (((devfn) >> 3) & 0x1f) +#define PCI_FUNC(devfn) ((devfn) & 7) +#define strscpy(dst, src, size) snprintf(dst, size, "%s", src) +typedef u64 mk_phys_cpu_t; + +static void __noreturn mk_enter_pool_state(void *unused) +{ + longjmp(parked, 1); +} + +static void smp_call_function(void (*fn)(void *), void *arg, int wait) +{ + assert(fn == mk_enter_pool_state && !wait); +} + +static bool schedule_work(struct work_struct *work) +{ + assert(!queued); + queued = work; + return true; +} + +static bool queue_work(struct workqueue_struct *wq, struct work_struct *work) +{ + assert(wq == &cpu_queue); + schedule_work(work); + return true; +} + +static void unexpected_work(struct work_struct *work) +{ + abort(); +} + +#define INIT_WORK(work, fn) ((work)->func = (fn)) +#define mk_halted_work_fn unexpected_work + +static struct mk_instance halted_peer = { .id = 65537 }; + +static struct mk_instance *mk_instance_find(int id) +{ + if (id != halted_peer.id) + return NULL; + halted_peer.refs++; + return &halted_peer; +} + +static void mk_instance_put(struct mk_instance *instance) +{ + assert(instance->refs > 0); + instance->refs--; +} + +#include "ack-resource.h" +#include "ack-system.h" + +static int failures; + +static void expect(const char *kind, const char *name, int got, int want) +{ + if (got != want) { + fprintf(stderr, "FAIL %s: %s: got %d, want %d\n", kind, name, + got, want); + failures++; + } else { + printf("PASS %s: %s\n", kind, name); + } +} + +static u64 current_id; + +static struct mk_pending_msg *pending(u32 type, u32 op) +{ + struct mk_pending_msg *p = mk_msg_pending_add(type, op, 65537); + + assert(!IS_ERR(p)); + current_id = mk_msg_pending_id(p); + assert(current_id); + return p; +} + +static int finish(struct mk_pending_msg *p) +{ + int result = mk_msg_pending_wait(p, 0); + + assert(list_empty(&mk_pending_msgs)); + assert(!mk_pending_msgs_lock.held); + return result; +} + +static void dispatch(struct mk_ipi_data *data) +{ + struct mk_msg_type_handler handler = { + .msg_type = data->type, + .msg_handler = data->type == MK_MSG_SYSTEM ? + mk_system_msg_handler : + mk_resource_msg_handler, + }; + + mk_message_type_ipi_callback(data, &handler); +} + +static void reply(u32 type, u32 op, u64 resource, int result, u64 id, + bool short_ack) +{ + struct mk_resource_ack ack = { + .operation = op, + .result = result, + .resource_id = resource, + }; + struct mk_ipi_data data = { .sender_cpu = 7, .type = type }; + struct mk_message *msg = (struct mk_message *)data.buffer; + + msg->msg_type = type; + msg->msg_subtype = type == MK_MSG_SYSTEM ? MK_SYS_SHUTDOWN_ACK : + MK_RES_ACK; + msg->msg_id = id; + msg->payload_len = sizeof(ack) - short_ack; + memcpy(msg->payload, &ack, sizeof(ack)); + data.data_size = sizeof(*msg) + msg->payload_len; + dispatch(&data); +} + +static void cpu_ack(void) +{ + reply(MK_MSG_RESOURCE, MK_RES_CPU_ADD, 65537, 0, current_id, false); +} + +static void cpu_error_ack(void) +{ + reply(MK_MSG_RESOURCE, MK_RES_CPU_ADD, 65537, -EIO, current_id, false); +} + +static unsigned int callbacks; + +static void count_callback(u32 type, u32 subtype, u64 id, void *payload, + u32 len, void *ctx) +{ + callbacks++; +} + +static void completion_regressions(void) +{ + struct mk_pending_msg *p; + struct mk_ipi_data data = { .type = MK_MSG_RESOURCE }; + struct mk_message *msg = (struct mk_message *)data.buffer; + struct mk_msg_type_handler handler = { + .msg_type = MK_MSG_RESOURCE, + .msg_handler = count_callback, + }; + + /* A completion token is consumed by the waiter, not by duplicates. */ + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + cpu_ack(); + cpu_error_ack(); + expect("REGRESSION", "duplicate must preserve first success", finish(p), + 0); + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + cpu_error_ack(); + cpu_ack(); + expect("REGRESSION", "duplicate must preserve first error", finish(p), + -EIO); + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + cpu_ack(); + at_completion = cpu_error_ack; + expect("REGRESSION", "duplicate after waiter consumes token", finish(p), + 0); + + p = pending(MK_MSG_SYSTEM, MK_SYS_SHUTDOWN); + reply(MK_MSG_SYSTEM, MK_RES_CPU_ADD, 65537, 0, current_id, false); + expect("REGRESSION", "shutdown operation mismatch", finish(p), + -ETIMEDOUT); + + /* Reject a confused outer/inner family before calling any handler. */ + msg->msg_type = MK_MSG_SYSTEM; + msg->payload_len = 0; + data.data_size = sizeof(*msg); + mk_message_type_ipi_callback(&data, &handler); + expect("REGRESSION", "outer/inner family mismatch", callbacks, 0); + callbacks = 0; + msg->msg_type = MK_MSG_RESOURCE; + mk_message_type_ipi_callback(&data, &handler); + expect("REGRESSION", "matching family reaches handler", callbacks, 1); +} + +static void regressions(void) +{ + const struct { + const char *name; + u32 type, op; + } stale[] = { + { "delayed CPU_ADD A must not complete B", MK_MSG_RESOURCE, + MK_RES_CPU_ADD }, + { "delayed CPU_REMOVE A must not complete B", MK_MSG_RESOURCE, + MK_RES_CPU_REMOVE }, + { "delayed SHUTDOWN A must not complete B", MK_MSG_SYSTEM, + MK_SYS_SHUTDOWN }, + }; + struct mk_pending_msg *p; + u64 old_id; + size_t i; + + /* Fix: allocate and echo distinct request IDs, then match them. */ + for (i = 0; i < ARRAY_SIZE(stale); i++) { + p = pending(stale[i].type, stale[i].op); + assert(!mk_send_message_id(1, stale[i].type, stale[i].op, + current_id, NULL, 0)); + old_id = ((struct mk_message *)sent.buffer)->msg_id; + assert(finish(p) == -ETIMEDOUT); + p = pending(stale[i].type, stale[i].op); + assert(!mk_send_message_id(1, stale[i].type, stale[i].op, + current_id, NULL, 0)); + reply(stale[i].type, stale[i].op, 65537, 0, old_id, false); + expect("REGRESSION", stale[i].name, finish(p), -ETIMEDOUT); + } + + /* Zero replies from an older responder cannot complete a new request. */ + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + reply(MK_MSG_RESOURCE, MK_RES_CPU_ADD, 65537, 0, 0, false); + expect("REGRESSION", "zero msg_id must not authorize", finish(p), + -ETIMEDOUT); + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + reply(MK_MSG_RESOURCE, MK_RES_CPU_ADD, 65537, 0, U64_MAX, false); + expect("REGRESSION", "unissued msg_id must not authorize", finish(p), + -ETIMEDOUT); +} + +static void characterizations(void) +{ + const struct { + const char *name; + u32 type, op; + u64 resource; + bool short_ack; + } reject[] = { + { "resource operation mismatch", MK_MSG_RESOURCE, + MK_RES_CPU_REMOVE, 65537, false }, + { "resource ID mismatch", MK_MSG_RESOURCE, MK_RES_CPU_ADD, + 65538, false }, + { "short resource ACK", MK_MSG_RESOURCE, MK_RES_CPU_ADD, 65537, + true }, + }; + struct mk_pending_msg *p; + size_t i; + + /* Isolate type from dispatch validation. */ + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + mk_msg_pending_complete(MK_MSG_SYSTEM, MK_RES_CPU_ADD, 65537, + current_id, 0); + expect("CHARACTERIZATION", "message type mismatch", finish(p), + -ETIMEDOUT); + + /* Breaks if matching loses a tuple check or short-ACK validation. */ + for (i = 0; i < ARRAY_SIZE(reject); i++) { + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + reply(reject[i].type, reject[i].op, reject[i].resource, 0, + current_id, reject[i].short_ack); + expect("CHARACTERIZATION", reject[i].name, finish(p), + -ETIMEDOUT); + } + p = pending(MK_MSG_SYSTEM, MK_SYS_SHUTDOWN); + reply(MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, 65537, 0, current_id, true); + expect("CHARACTERIZATION", "short shutdown ACK", finish(p), -ETIMEDOUT); + + /* Breaks if wait forgets cleanup, ignores completion or changes its result. */ + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + expect("CHARACTERIZATION", "timeout removes pending A", finish(p), + -ETIMEDOUT); + cpu_ack(); /* Late reply after free, with no replacement: must be harmless. */ + assert(list_empty(&mk_pending_msgs)); + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + expect("CHARACTERIZATION", "ACK after cleanup is not saved for B", + finish(p), -ETIMEDOUT); + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + at_timeout = cpu_ack; + expect("CHARACTERIZATION", "ACK after timeout decision still cleans up", + finish(p), -ETIMEDOUT); + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + cpu_ack(); + cpu_ack(); + expect("CHARACTERIZATION", "identical duplicate returns success", + finish(p), 0); + cpu_ack(); /* Also harmless after a successful wait frees the request. */ + assert(list_empty(&mk_pending_msgs)); + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + reply(MK_MSG_RESOURCE, MK_RES_CPU_ADD, 65537, -EIO, current_id, false); + expect("CHARACTERIZATION", "matching error is returned", finish(p), + -EIO); +} + +/* Keep longjmp at the parking boundary, outside the test-case loop. */ +static void run_work(struct work_struct *work, bool parks) +{ + if (!setjmp(parked)) { + if (work) + work->func(work); + else + /* Wire checks only; qualification is tested separately. */ + mk_halt_to_pool(true); + assert(!parks); + } else { + assert(parks); + /* The parked fixture kernel is discarded before the next case. */ + mk_overlay_mutex = false; + } +} + +static void worker_regressions(void) +{ + const struct { + const char *name; + u32 type, op; + } cases[] = { + { "CPU add worker echo", MK_MSG_RESOURCE, MK_RES_CPU_ADD }, + { "CPU remove worker echo", MK_MSG_RESOURCE, + MK_RES_CPU_REMOVE }, + { "memory add worker echo", MK_MSG_RESOURCE, MK_RES_MEM_ADD }, + { "memory remove worker echo", MK_MSG_RESOURCE, + MK_RES_MEM_REMOVE }, + { "device add worker echo", MK_MSG_RESOURCE, + MK_RES_DEVICE_ADD }, + { "device remove worker echo", MK_MSG_RESOURCE, + MK_RES_DEVICE_REMOVE }, + { "shutdown worker echo", MK_MSG_SYSTEM, MK_SYS_SHUTDOWN }, + }; + const u64 ids[] = { 0, 0x123456789abcdef0ULL, 0 /* allocated below */ }; + struct mk_cpu_resource_payload cpu = { .cpu_id = 65537, + .sender_instance_id = 3 }; + struct mk_mem_resource_payload mem = { .start_pfn = 65537, + .sender_instance_id = 3 }; + struct mk_device_resource_payload dev = { + .domain = 1, + .devfn = 1, + .sender_instance_id = 3, + }; + struct mk_shutdown_payload shutdown = { .sender_instance_id = 3 }; + size_t i, j; + + /* The existing 64-bit wire ABI has an unaligned flexible payload. */ + assert(sizeof(struct mk_message) == 24); + assert(offsetof(struct mk_message, payload) == 20); + assert(sizeof(cpu) == 24 && sizeof(mem) == 32 && sizeof(dev) == 544); + assert(sizeof(struct mk_resource_ack) == 16 && sizeof(shutdown) == 8); + + for (i = 0; i < ARRAY_SIZE(cases); i++) { + for (j = 0; j < ARRAY_SIZE(ids); j++) { + struct mk_ipi_data request = { .type = cases[i].type }; + struct mk_ipi_data before; + struct mk_message *msg = (void *)request.buffer; + struct mk_resource_ack ack; + struct work_struct *work; + struct mk_pending_msg *p = NULL; + void *payload; + u32 len; + u64 id = ids[j]; + int result; + + /* Each case represents a fresh receiver, before shutdown closes it. */ + mk_hotplug_accepting = true; + atomic_set(&mk_shutdown_requested, 0); + + if (i < 2) { + payload = &cpu; + len = sizeof(cpu); + } else if (i < 4) { + payload = &mem; + len = sizeof(mem); + } else if (i < 6) { + payload = &dev; + len = sizeof(dev); + } else { + payload = &shutdown; + len = sizeof(shutdown); + } + if (j == 2) { + p = pending(cases[i].type, cases[i].op); + id = mk_msg_pending_id(p); + } + assert(!mk_send_message_id(1, cases[i].type, + cases[i].op, id, payload, + len)); + request = sent; + if (j == 1) { + /* A large extension must not overrun the local decoder. */ + msg = (void *)request.buffer; + memset(msg->payload + len, 0xa5, + MK_MAX_DATA_SIZE - sizeof(*msg) - len); + msg->payload_len = + MK_MAX_DATA_SIZE - sizeof(*msg); + request.data_size = MK_MAX_DATA_SIZE; + } + before = request; + mutations = 0; + mutation_result = j ? -EIO : 0; + result = i == 6 ? 0 : mutation_result; + dispatch(&request); + assert(queued); + expect("DECODE", "receive buffer unchanged", + memcmp(&request, &before, sizeof(request)) == 0, + 1); + work = queued; + queued = NULL; + /* A worker must retain the ID after the receive buffer is reused. */ + memset(&request, 0xa5, sizeof(request)); + memset(&sent, 0, sizeof(sent)); + run_work(work, i == 6); + msg = (void *)sent.buffer; + /* Test code must also decode the wire without unaligned reads. */ + memcpy(&ack, msg->payload, sizeof(ack)); + assert(sent_to == 3 && msg->msg_type == cases[i].type); + assert(msg->msg_subtype == + (i == 6 ? MK_SYS_SHUTDOWN_ACK : MK_RES_ACK)); + assert(msg->payload_len == sizeof(ack)); + assert(ack.operation == cases[i].op && + ack.resource_id == 65537); + assert((int)ack.result == result); + assert(mutations == (i == 6 ? 0 : 1)); + expect("WORKER", cases[i].name, msg->msg_id == id, 1); + if (p) { + dispatch(&sent); + expect("ROUNDTRIP", cases[i].name, finish(p), + result); + } + } + /* Short and empty requests must not queue work or read missing fields. */ + for (j = 0; j < 2; j++) { + u8 bytes[sizeof(mem)] = {}; + + assert(!queued); + assert(!mk_send_message_id(1, cases[i].type, + cases[i].op, 1, bytes, j)); + dispatch(&sent); + expect("DECODE", "short request queues no work", + queued == NULL, 1); + } + } + run_work(NULL, true); + expect("WORKER", "voluntary HALTED remains a zero-ID notification", + ((struct mk_message *)sent.buffer)->msg_id == 0 && + sent_to == 0 && + ((struct mk_message *)sent.buffer)->msg_subtype == + MK_SYS_HALTED, + 1); +} + +static void shutdown_admission_regressions(void) +{ + struct mk_shutdown_payload payload = { .sender_instance_id = 3 }; + struct mk_ipi_data request; + struct work_struct *work; + struct mk_shutdown_work *sw; + const u64 id = 0x123456789abcdef0ULL; + + assert(!queued); + /* Earlier worker cases discarded their parked fixture kernels. */ + atomic_set(&mk_shutdown_requested, 0); + mk_hotplug_accepting = true; + assert(!mk_send_message_id(1, MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, + id, &payload, sizeof(payload))); + request = sent; + irq_allocations = 0; + allocation_fails = test_hardirq = true; + /* Invalid input must not consume the one valid shutdown opportunity. */ + ((struct mk_message *)request.buffer)->payload_len = 0; + dispatch(&request); + expect("SHUTDOWN", "short request queues no work", queued == NULL, 1); + request = sent; + dispatch(&request); + allocation_fails = test_hardirq = false; + expect("SHUTDOWN", "hard IPI does not enter allocator", irq_allocations, 0); + expect("SHUTDOWN", "shutdown survives allocator exhaustion", queued != NULL, 1); + if (!queued) + return; + work = queued; + sw = container_of(work, struct mk_shutdown_work, work); + payload.sender_instance_id = 9; + payload.flags = 7; + assert(!mk_send_message_id(1, MK_MSG_SYSTEM, MK_SYS_SHUTDOWN, + id + 1, &payload, sizeof(payload))); + test_hardirq = true; + dispatch(&sent); + test_hardirq = false; + expect("SHUTDOWN", "duplicate preserves pending work", queued == work, 1); + expect("SHUTDOWN", "duplicate preserves first requester and ID", + sw->msg_id == id && sw->sender_instance_id == 3 && !sw->flags, 1); + queued = NULL; + /* A running work item is no longer pending, but must not be rearmed. */ + test_hardirq = true; + dispatch(&sent); + test_hardirq = false; + expect("SHUTDOWN", "duplicate cannot requeue dequeued work", queued == NULL, 1); + expect("SHUTDOWN", "duplicates do not enter allocator", irq_allocations, 0); + run_work(work, true); + expect("SHUTDOWN", "worker acknowledges original request", + sent_to == 3 && ((struct mk_message *)sent.buffer)->msg_id == id, 1); +} + +static void pci_key_regressions(void) +{ + const u16 domains[] = { 0x8000, 0xffff }; + const u32 ops[] = { MK_RES_DEVICE_ADD, MK_RES_DEVICE_REMOVE }; + struct mk_device_resource_payload payload = { .devfn = 1, + .sender_instance_id = 3 }; + size_t i, j; + + for (i = 0; i < ARRAY_SIZE(domains); i++) { + for (j = 0; j < ARRAY_SIZE(ops); j++) { + struct mk_pending_msg *p; + struct work_struct *work; + u32 resource_id = ((u32)domains[i] << 16) | + payload.devfn; + + payload.domain = domains[i]; + p = mk_msg_pending_add(MK_MSG_RESOURCE, ops[j], + resource_id); + assert(!IS_ERR(p)); + assert(!mk_send_message_id(1, MK_MSG_RESOURCE, ops[j], + mk_msg_pending_id(p), + &payload, sizeof(payload))); + dispatch(&sent); + assert(queued); + work = queued; + queued = NULL; + mutation_result = 0; + run_work(work, false); + dispatch(&sent); + expect("REGRESSION", "high PCI domain round trip", + finish(p), 0); + } + } +} + +static void halted_decode_regression(void) +{ + struct mk_resource_ack ack = { .operation = MK_SYS_SHUTDOWN, .resource_id = 65537 }; + struct mk_ipi_data before; + size_t len; + + INIT_WORK(&halted_peer.halted_work, mk_halted_work_fn); + assert(!mk_send_message(0, MK_MSG_SYSTEM, MK_SYS_HALTED, &ack, + sizeof(ack))); + before = sent; + irq_allocations = 0; + allocation_fails = test_hardirq = true; + dispatch(&sent); + allocation_fails = test_hardirq = false; + expect("HALTED", "hard IPI does not enter allocator", irq_allocations, 0); + expect("HALTED", "notification survives allocator exhaustion", queued != NULL, 1); + if (!queued) + return; + expect("DECODE", "HALTED pins exact instance work", + queued == &halted_peer.halted_work && halted_peer.refs == 1, 1); + expect("DECODE", "HALTED receive buffer unchanged", + memcmp(&sent, &before, sizeof(sent)) == 0, 1); + queued = NULL; + /* Only admission/decoding here; halt_settlement_test runs the real worker. */ + atomic_set(&halted_peer.halted_pending, 0); + mk_instance_put(&halted_peer); + + for (len = 0; len < sizeof(ack); len++) { + assert(!mk_send_message(0, MK_MSG_SYSTEM, MK_SYS_HALTED, &ack, + len)); + dispatch(&sent); + assert(!queued); + } + expect("DECODE", "short HALTED queues no work", queued == NULL, 1); + for (unsigned int invalid = 0; invalid < 4; invalid++) { + struct mk_resource_ack bad = ack; + + if (invalid == 0) + bad.operation++; + else if (invalid == 1) + bad.result = -EIO; + else if (invalid == 2) + bad.resource_id += 1ULL << 32; + else + bad.resource_id++; + assert(!mk_send_message(0, MK_MSG_SYSTEM, MK_SYS_HALTED, &bad, sizeof(bad))); + dispatch(&sent); + expect("DECODE", "invalid or unknown HALTED queues no work", + !queued && !halted_peer.refs, 1); + } +} + +static void failure_regressions(void) +{ + struct mk_pending_msg *p, *other; + u64 old_id; + int ret; + + allocation_fails = true; + p = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_ADD, 65537); + assert(IS_ERR(p)); + expect("FAILURE", "pending allocation error", PTR_ERR(p), -ENOMEM); + allocation_fails = false; + assert(list_empty(&mk_pending_msgs)); + + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + old_id = current_id; + other = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + assert(current_id != old_id); + reply(MK_MSG_RESOURCE, MK_RES_CPU_ADD, 65537, 0, old_id, false); + expect("REGRESSION", "same-tuple outstanding requests are distinct", + mk_msg_pending_wait(p, 0), 0); + expect("REGRESSION", "other outstanding request remains pending", + finish(other), -ETIMEDOUT); + + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + allocation_fails = true; + test_hardirq = true; + irq_allocations = 0; + ret = mk_send_message_id(1, MK_MSG_RESOURCE, MK_RES_CPU_ADD, current_id, + NULL, 0); + expect("REGRESSION", "send survives allocator exhaustion", ret, 0); + expect("REGRESSION", "hard-IPI send does not allocate", irq_allocations, 0); + test_hardirq = false; + allocation_fails = false; + mk_msg_pending_cancel(p); + cpu_ack(); + assert(list_empty(&mk_pending_msgs)); + + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + send_error = -EIO; + ret = mk_send_message_id(1, MK_MSG_RESOURCE, MK_RES_CPU_ADD, current_id, + NULL, 0); + expect("FAILURE", "transport error", ret, -EIO); + /* Even a reply racing local cleanup must not leave a dangling entry. */ + cpu_ack(); + mk_msg_pending_cancel(p); + cpu_ack(); + assert(list_empty(&mk_pending_msgs) && !mk_pending_msgs_lock.held); + send_error = 0; + expect("FAILURE", "oversized payload rejected before arithmetic", + mk_send_message(1, MK_MSG_NETWORK, MK_NET_VSOCK_PKT, NULL, UINT_MAX), + -EMSGSIZE); + assert(!mk_send_message(1, MK_MSG_NETWORK, MK_NET_VSOCK_PKT, NULL, 0)); + expect("REGRESSION", "uncorrelated sender keeps zero ID", + ((struct mk_message *)sent.buffer)->msg_id == 0, 1); + + /* Last: forcing exhaustion intentionally prevents all subsequent adds. */ + mk_last_msg_id = U64_MAX - 1; + p = pending(MK_MSG_RESOURCE, MK_RES_CPU_ADD); + expect("FAILURE", "last representable ID is usable", + current_id == U64_MAX, 1); + cpu_ack(); + expect("FAILURE", "full-width ID matches", finish(p), 0); + p = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_ADD, 65537); + assert(IS_ERR(p)); + expect("FAILURE", "ID exhaustion refuses reuse", PTR_ERR(p), + -EOVERFLOW); + assert(mk_last_msg_id == U64_MAX); + assert(list_empty(&mk_pending_msgs) && !mk_pending_msgs_lock.held); +} + +int main(int argc, char **argv) +{ + struct mk_pending_msg *irq_pending; + + /* Real ACK completion is dispatched by a hard IPI, not a threaded IRQ. */ + irq_pending = mk_msg_pending_add(MK_MSG_RESOURCE, MK_RES_CPU_ADD, 7); + assert(!IS_ERR(irq_pending)); + test_hardirq = true; + mk_msg_pending_complete(MK_MSG_RESOURCE, MK_RES_CPU_ADD, 7, + mk_msg_pending_id(irq_pending), 0); + test_hardirq = false; + assert(mk_msg_pending_wait(irq_pending, 1) == 0); + assert(list_empty(&mk_pending_msgs) && !mk_pending_msgs_lock.held); + + /* Queue creation/lifetime is exercised by cpu_work_test, not this fixture. */ + for (size_t slot = 0; slot < MK_IPI_RING_SIZE; slot++) { + INIT_WORK(&mk_cpu_requests[slot].work, mk_hotplug_work_fn); + INIT_WORK(&mk_other_requests[slot].work, mk_hotplug_work_fn); + } + mk_cpu_hotplug_wq = &cpu_queue; + mk_hotplug_accepting = true; + + if (argc > 2 || + (argc == 2 && strcmp(argv[1], "regression") && + strcmp(argv[1], "characterization") && + strcmp(argv[1], "completion") && strcmp(argv[1], "workers") && + strcmp(argv[1], "failure"))) { + fprintf(stderr, "usage: ack_test [mode] (see README.ack)\n"); + return 2; + } + if (argc == 1 || !strcmp(argv[1], "regression") || + !strcmp(argv[1], "completion")) + completion_regressions(); + if (argc == 1 || !strcmp(argv[1], "regression")) + regressions(); + if (argc == 1 || !strcmp(argv[1], "regression")) + pci_key_regressions(); + if (argc == 1 || !strcmp(argv[1], "characterization")) + characterizations(); + if (argc == 1 || !strcmp(argv[1], "workers")) + worker_regressions(); + if (argc == 1 || !strcmp(argv[1], "workers")) + halted_decode_regression(); + if (argc == 1 || !strcmp(argv[1], "workers")) + shutdown_admission_regressions(); + if (argc == 1 || !strcmp(argv[1], "failure")) + failure_regressions(); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/multikernel/ack_test.sh b/tools/testing/selftests/multikernel/ack_test.sh new file mode 100644 index 00000000000000..13d34be039a306 --- /dev/null +++ b/tools/testing/selftests/multikernel/ack_test.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# Compile verbatim production definitions; no kernel build or generated sources +# are kept in the worktree. See README.ack for historical RED evidence/limits. +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-ack.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM + +awk '/^#define MK_IPI_RING_SIZE / { print } + /^#define MK_MAX_DATA_SIZE / { print } + /^struct mk_ipi_data \{/ { copy = 1 } + copy { print } + copy && /^};/ { copy = 0 } + /^#define MK_MSG_IO / { types = 1 } + /^\/\* Opaque type/ { types = 0 } + types { print }' "$src/include/linux/multikernel.h" > "$build/ack-types.h" + +# Kernel top-level definitions end with a brace in column zero. Missing or +# incompatible definitions cause a compile failure, never a substitute model. +awk '/^struct mk_(pending_msg|msg_type_handler) \{/ { copy = 1 } + /^static (LIST_HEAD|DEFINE_(RAW_)?SPINLOCK)\(mk_pending_msgs/ { print } + /^static u64 mk_last_msg_id;/ { print } + /^(struct mk_pending_msg \*|void |int |u64 )mk_msg_pending_/ { copy = 1 } + /^static void mk_message_type_ipi_callback\(/ { copy = 1 } + /^static int mk_build_and_send\(/ { copy = 1 } + /^int mk_send_message(_id|_to)?\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/messaging.c" \ + > "$build/ack-messaging.h" +awk '/^struct mk_hotplug_work \{/ { copy = 1 } + /^static DEFINE_(RAW_)?SPINLOCK\(mk_cpu_hotplug_lock\);/ { print } + /^static struct workqueue_struct \*mk_cpu_hotplug_wq;/ { print } + /^static (bool mk_hotplug_accepting|unsigned int mk_hotplug_inflight)/ { print } + /^static DECLARE_WAIT_QUEUE_HEAD\(mk_hotplug_wait\);/ { print } + /^static int mk_queue_hotplug_request\(/ { copy = 1 } + /^static (void|bool) mk_hotplug_(work_done|idle)\(/ { copy = 1 } + /^static struct mk_hotplug_work mk_(cpu|other)_requests\[/ { print } + /^static void mk_hotplug_work_fn\(/ { copy = 1 } + /^void mk_hotplug_quiesce\(/ { copy = 1 } + /^static void mk_(cpu|mem|device)_(add|remove)_work_fn\(/ { copy = 1 } + /^(static )?int mk_handle_(cpu|mem|device)_(add|remove)\(/ { copy = 1 } + /^static void mk_resource_msg_handler\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/hotplug.c" \ + > "$build/ack-resource.h" +awk '/^struct mk_(shutdown_work|halted_work) \{/ { copy = 1 } + /^static struct mk_shutdown_work mk_shutdown;/ { print } + /^static atomic_t mk_shutdown_requested =/ { print } + /^static void __noreturn mk_notify_down_and_park\(/ { copy = 1 } + /^void __noreturn mk_halt_to_pool\(/ { copy = 1 } + /^static void mk_shutdown_work_fn\(/ { copy = 1 } + /^static void mk_system_msg_handler\(/ { copy = 1 } + /^static void mk_queue_halted\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/core.c" \ + > "$build/ack-system.h" + +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -Wno-unused-parameter \ + -I"$build" -I"$src/tools/include" "$test_dir/ack_test.c" -o "$build/ack_test" +"$build/ack_test" "$@" diff --git a/tools/testing/selftests/multikernel/cpu_departure_test.c b/tools/testing/selftests/multikernel/cpu_departure_test.c new file mode 100644 index 00000000000000..36e0c4cb6f8618 --- /dev/null +++ b/tools/testing/selftests/multikernel/cpu_departure_test.c @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* CPUHP entry-point contract checks, not a live race reproduction. */ +#include +#include +#include +#include +#include +#include +#include + +#define CONFIG_CPU_HOTPLUG_STATE_CONTROL 1 +#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) +enum cpuhp_state { + CPUHP_INVALID = -1, + CPUHP_OFFLINE, + CPUHP_DEPARTED, + CPUHP_INTERMEDIATE, + CPUHP_ONLINE, +}; + +struct cpuhp_cpu_state { + enum cpuhp_state state; + enum cpuhp_state target; +}; + +struct cpuhp_step { + const char *name; + struct { void *single; } startup, teardown; + bool cant_stop; +}; + +struct device { unsigned int id; }; +struct device_attribute { int unused; }; + +static const unsigned int nr_cpu_ids = 4; +static struct cpuhp_cpu_state cpuhp_state[4]; +static struct device devices[4]; +static bool possible[4], present[4], online[4], registered[4]; +static bool device_locked, maps_locked, write_locked; +static int down_error, down_calls, offline_calls, up_calls, failures; +static int departing_cpu = -1; +static bool bad_down_state; +static int cpuhp_state_mutex; +static const char *scenario; + +static struct cpuhp_step steps[] = { + [CPUHP_OFFLINE] = { .name = "offline" }, +#include "departure-step.h" + [CPUHP_INTERMEDIATE] = { .name = "intermediate" }, + [CPUHP_ONLINE] = { .name = "online" }, +}; + +#define per_cpu_ptr(ptr, cpu) (&(*(ptr))[cpu]) +#define WARN_ON_ONCE(condition) (!!(condition)) +#define WARN_ON(condition) (!!(condition)) + +static void check(bool condition, const char *reason) +{ + if (!condition) { + fprintf(stderr, "FAIL: %s: %s\n", scenario, reason); + failures++; + } +} + +static bool cpu_possible(unsigned int cpu) +{ + return cpu < nr_cpu_ids && possible[cpu]; +} + +static bool cpu_present(unsigned int cpu) +{ + return cpu < nr_cpu_ids && present[cpu]; +} + +static bool cpu_online(unsigned int cpu) +{ + return cpu < nr_cpu_ids && online[cpu]; +} + +static struct device *get_cpu_device(unsigned int cpu) +{ + return cpu < nr_cpu_ids && registered[cpu] ? &devices[cpu] : NULL; +} + +static void lock_device_hotplug(void) +{ + check(!device_locked && !maps_locked, "device lock order"); + device_locked = true; +} + +static void unlock_device_hotplug(void) +{ + check(device_locked && !maps_locked && !write_locked, + "device lock released last"); + device_locked = false; +} + +static void cpu_maps_update_begin(void) +{ + check(device_locked && !maps_locked, "maps lock order"); + maps_locked = true; +} + +static void cpu_maps_update_done(void) +{ + check(maps_locked && !write_locked, "maps lock balance"); + if (departing_cpu >= 0) + check(!present[departing_cpu], + "presence withdrawn before releasing native serialization"); + maps_locked = false; +} + +static void cpus_write_lock(void) +{ + check(maps_locked && !write_locked, "CPU write lock order"); + write_locked = true; +} + +static void cpus_write_unlock(void) +{ + check(write_locked, "CPU write lock balance"); + write_locked = false; +} + +static void set_cpu_present(unsigned int cpu, bool value) +{ + check(device_locked && maps_locked && write_locked, + "presence change excluded from native transitions"); + present[cpu] = value; +} + +static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target) +{ + check(device_locked && maps_locked && !write_locked, + "teardown called without recursive CPU write locking"); + down_calls++; + if (cpu >= nr_cpu_ids) + return -EINVAL; + if (down_error) + return down_error; + if (!bad_down_state) { + online[cpu] = false; + cpuhp_state[cpu].state = target; + cpuhp_state[cpu].target = target; + if (target == CPUHP_DEPARTED) + departing_cpu = cpu; + } + return 0; +} + +static void cpuhp_offline_cpu_device(unsigned int cpu) +{ + check(device_locked && maps_locked, "device bookkeeping within transaction"); + check(!cpu_present(cpu) && !cpu_online(cpu), + "withdrawn and offline before bookkeeping"); + check(get_cpu_device(cpu), "registered device required"); + offline_calls++; +} + +static int kstrtoint(const char *buf, int base, int *value) +{ + char *end; + + *value = strtol(buf, &end, base); + return *end ? -EINVAL : 0; +} + +static int lock_device_hotplug_sysfs(void) +{ + lock_device_hotplug(); + return 0; +} + +static void mutex_lock(int *lock) { } +static void mutex_unlock(int *lock) { } +static struct cpuhp_step *cpuhp_get_step(int state) { return &steps[state]; } + +static int cpu_up(unsigned int cpu, enum cpuhp_state target) +{ + up_calls++; + return 0; +} + +#include "departure-production.h" + +typedef unsigned long long u64; +typedef uint32_t u32; +typedef uint16_t u16; +typedef uint8_t u8; +typedef u64 phys_addr_t; +typedef u64 mk_phys_cpu_t; +struct list_head { struct list_head *next, *prev; }; +static struct list_head mk_hotplug_ops; +static int mk_hotplug_mutex, account_calls; + +#define pr_err(...) ((void)0) +#define pr_info(...) ((void)0) +#define pr_warn(...) ((void)0) +#define GFP_KERNEL 0 +/* Operation-log allocation is optional, unlike ownership accounting. */ +#define kzalloc(size, flags) ((void)(size), (void)(flags), NULL) +#define for_each_present_cpu(cpu) \ + for ((cpu) = next_present(0); (unsigned int)(cpu) < nr_cpu_ids; \ + (cpu) = next_present((cpu) + 1)) + +static unsigned int next_present(unsigned int cpu) +{ + while (cpu < nr_cpu_ids && !cpu_present(cpu)) + cpu++; + return cpu; +} + +static mk_phys_cpu_t arch_cpu_physical_id(unsigned int cpu) +{ + return 100 + cpu; +} + +static void list_add_tail(struct list_head *entry, struct list_head *head) { } + +static void mk_account_cpu_offline(mk_phys_cpu_t id) +{ + unsigned int cpu = id - 100; + + check(!cpu_present(cpu) && !cpu_online(cpu), + "ownership changed only after departure and withdrawal"); + account_calls++; +} + +#include "departure-hotplug.h" + +static void reset(const char *name) +{ + unsigned int cpu; + + scenario = name; + check(!device_locked && !maps_locked && !write_locked, + "previous operation released all locks"); + down_calls = 0; + offline_calls = 0; + up_calls = 0; + down_error = 0; + departing_cpu = -1; + account_calls = 0; + bad_down_state = false; + for (cpu = 0; cpu < nr_cpu_ids; cpu++) { + devices[cpu].id = cpu; + possible[cpu] = true; + present[cpu] = true; + online[cpu] = true; + registered[cpu] = true; + cpuhp_state[cpu].state = CPUHP_ONLINE; + cpuhp_state[cpu].target = CPUHP_ONLINE; + } +} + +static void expect_rejected(unsigned int cpu, int error) +{ + check(depart_cpu(cpu) == error, "eligibility error returned"); + check(!down_calls && !offline_calls, "ineligible CPU not handed off"); + check(!device_locked && !maps_locked && !write_locked, + "rejection released locks"); +} + +int main(void) +{ + static const int errors[] = { -EBUSY, -EOPNOTSUPP, -EIO }; + size_t i; + + reset("successful departure"); + check(depart_cpu(1) == 0, "success returned"); + check(!present[1] && !online[1], "CPU no longer present or online"); + check(cpuhp_state[1].state == CPUHP_DEPARTED, "departed state reached"); + check(down_calls == 1 && offline_calls == 1, "one departure and notification"); + + reset("ordinary offline CPU"); + online[1] = false; + cpuhp_state[1].state = CPUHP_OFFLINE; + expect_rejected(1, -EINVAL); + reset("intermediate CPUHP state"); + cpuhp_state[1].state = CPUHP_INTERMEDIATE; + expect_rejected(1, -EINVAL); + reset("CPU zero"); + expect_rejected(0, -EINVAL); + reset("out-of-range CPU"); + expect_rejected(nr_cpu_ids, -EINVAL); + reset("impossible CPU"); + possible[1] = false; + expect_rejected(1, -EINVAL); + reset("absent CPU"); + present[1] = false; + expect_rejected(1, -EINVAL); + reset("missing device"); + registered[1] = false; + expect_rejected(1, -ENODEV); + + for (i = 0; i < ARRAY_SIZE(errors); i++) { + reset("native teardown failure"); + down_error = errors[i]; + check(depart_cpu(1) == errors[i], "native errno preserved"); + check(present[1] && online[1] && !offline_calls, + "failed teardown did not withdraw presence or notify"); + } + + reset("unconfirmed departure state"); + bad_down_state = true; + check(depart_cpu(1) == -EIO, "bad postcondition rejected"); + check(present[1] && !offline_calls, "no withdrawal on bad postcondition"); + + reset("native target excludes departed"); + check(target_store(&devices[1], NULL, "1", 1) == -EINVAL, + "native target cannot donate a CPU"); + check(!down_calls && !up_calls, "rejected target never starts transition"); + + reset("native offline target unchanged"); + check(target_store(&devices[1], NULL, "0", 1) == 1, + "ordinary offline target accepted"); + check(down_calls == 1 && cpuhp_state[1].state == CPUHP_OFFLINE, + "ordinary target still takes native offline path"); + reset("native online target unchanged"); + online[1] = false; + cpuhp_state[1].state = CPUHP_OFFLINE; + check(target_store(&devices[1], NULL, "3", 1) == 1, + "ordinary online target accepted"); + check(up_calls == 1, "ordinary target still takes native online path"); + + reset("multikernel remove success"); + check(mk_do_cpu_remove(101) == 0, "removal succeeded"); + check(account_calls == 1, "successful removal updates ownership once"); + reset("multikernel remove rejects offline CPU"); + online[1] = false; + cpuhp_state[1].state = CPUHP_OFFLINE; + check(mk_do_cpu_remove(101) == -EINVAL, "offline removal rejected"); + check(!account_calls && !down_calls && present[1], + "offline removal preserves ownership and presence"); + reset("multikernel remove native failure"); + down_error = -EBUSY; + check(mk_do_cpu_remove(101) == -EBUSY, "removal error propagated"); + check(!account_calls && present[1], "failed removal preserves ownership"); + check(!device_locked && !maps_locked && !write_locked, + "final operation released locks"); + if (failures) + return EXIT_FAILURE; + puts("PASS: CPU departure and native target contract checks (18 scenarios)"); + return EXIT_SUCCESS; +} diff --git a/tools/testing/selftests/multikernel/cpu_departure_test.sh b/tools/testing/selftests/multikernel/cpu_departure_test.sh new file mode 100644 index 00000000000000..fe392435e89829 --- /dev/null +++ b/tools/testing/selftests/multikernel/cpu_departure_test.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-cpu-departure.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM + +# Compile the real CPUHP entry points, with native teardown replaced at its +# maps-locked boundary. No live CPUs or sysfs controls are touched. +awk '/^static int cpu_down\(/ { copy = 1 } + /^int depart_cpu\(/ { copy = 1 } + /^static ssize_t target_store\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/cpu.c" \ + > "$build/departure-production.h" +awk '/^\t\[CPUHP_DEPARTED\] = \{/ { copy = 1 } + copy { print } + copy && /^\t},/ { exit }' "$src/kernel/cpu.c" \ + > "$build/departure-step.h" +awk '/^struct mk_hotplug_op \{/ { copy = 1 } + /^static int mk_cpu_to_logical\(/ { copy = 1 } + /^static int mk_do_cpu_remove\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/hotplug.c" \ + > "$build/departure-hotplug.h" + +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror \ + -Wno-unused-parameter -Wno-unused-function -Wno-sign-compare -I"$build" \ + "$test_dir/cpu_departure_test.c" -o "$build/cpu_departure_test" +"$build/cpu_departure_test" diff --git a/tools/testing/selftests/multikernel/cpu_handback_test.c b/tools/testing/selftests/multikernel/cpu_handback_test.c new file mode 100644 index 00000000000000..939e7415b0adc1 --- /dev/null +++ b/tools/testing/selftests/multikernel/cpu_handback_test.c @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real CPUHP/MK entry points and CPU sets; no live CPU transition. */ +#include +#include +#include +#include +#include +#include +#include + +typedef unsigned long long u64; +typedef uint32_t u32; +typedef uint16_t u16; +typedef uint8_t u8; +typedef u64 phys_addr_t; +#include "handback-types.h" + +#define GFP_KERNEL 0 +#define NUMA_NO_NODE (-1) +#define KOBJ_ONLINE 1 +#define WARN_ON_ONCE(condition) (!!(condition)) +#define pr_err(...) ((void)0) +#define pr_warn(...) ((void)0) +#define pr_info(...) ((void)0) +#define per_cpu_ptr(ptr, cpu) (&(*(ptr))[cpu]) +#define per_cpu(name, cpu) ((name)[cpu]) +#define max_t(type, a, b) ((type)(a) > (type)(b) ? (type)(a) : (type)(b)) +/* Optional operation logging is deliberately not allocated in this fixture. */ +#define kzalloc(size, flags) ((void)(size), (void)(flags), NULL) +#define krealloc_array(ptr, n, size, flags) \ + ((void)(flags), fail_alloc ? NULL : realloc(ptr, (n) * (size))) + +static bool fail_alloc; +#include "handback-set.h" + +enum cpuhp_state { + CPUHP_OFFLINE, + CPUHP_DEPARTED, + CPUHP_INTERMEDIATE, + CPUHP_ONLINE +}; + +struct cpuhp_cpu_state { + enum cpuhp_state state, target; +}; + +struct device { + unsigned int id; + bool offline, locked; + int kobj; +}; + +struct cpu { + struct device dev; + bool hotpluggable; +}; + +struct list_head { + struct list_head *next, *prev; +}; + +struct mk_instance { + struct mk_cpu_set *cpus; +}; + +struct mk_pool { + struct mk_cpu_set *cpus; + bool cpus_lock; +}; + +static struct cpuhp_cpu_state cpuhp_state[4]; +static struct cpu cpu_devices[4]; +static const unsigned int nr_cpu_ids = 4; +static bool possible[4], present[4], online[4], registered[4]; +static int nodes[4]; +static bool device_hotplug_locked, maps_locked, write_locked; +static bool cpu_hotplug_disabled, bootable, partial_online, bad_success; +static bool initially_present; +static int mapped_cpu = 1; +static int native_error, node_error, registration_error; +static unsigned int native_calls, events, retargets, failures, scenarios; +static bool mk_overlay_mutex = true; +static int mk_hotplug_mutex; +static struct list_head mk_hotplug_ops; +static struct mk_cpu_set own_set, pool_set; +static struct mk_instance self = { .cpus = &own_set }, *mk_self = &self; +static struct mk_pool pool = { .cpus = &pool_set }, *mk_pool = &pool; +static const char *scenario; + +static void check(bool condition, const char *why) +{ + if (!condition) { + fprintf(stderr, "FAIL: %s: %s\n", scenario, why); + failures++; + } +} + +#define lockdep_assert_held(lock) check(*(lock), "resource lock held") +static void mutex_lock(void *lock) +{ +} + +static void mutex_unlock(void *lock) +{ +} + +static bool *pool_lock(bool *lock) +{ + check(!*lock && !maps_locked, + "pool lock not nested with native bringup"); + *lock = true; + return lock; +} + +static void pool_unlock(bool **lock) +{ + check(**lock, "pool lock balanced"); + **lock = false; +} + +#define guard(name) \ + bool *pool_guard_##name __attribute__((cleanup(pool_unlock))) = pool_lock + +static bool cpu_possible(unsigned int cpu) +{ + return cpu < 4 && possible[cpu]; +} + +static bool cpu_present(unsigned int cpu) +{ + return cpu < 4 && present[cpu]; +} + +static bool cpu_online(unsigned int cpu) +{ + return cpu < 4 && online[cpu]; +} + +static int cpu_to_node(unsigned int cpu) +{ + return nodes[cpu]; +} + +static bool cpu_bootable(unsigned int cpu) +{ + return bootable; +} + +static struct device *get_cpu_device(unsigned int cpu) +{ + return cpu < 4 && registered[cpu] ? &cpu_devices[cpu].dev : NULL; +} + +static void lock_device_hotplug(void) +{ + check(!device_hotplug_locked && !maps_locked, + "device-hotplug lock order"); + device_hotplug_locked = true; +} + +static void unlock_device_hotplug(void) +{ + check(device_hotplug_locked && !maps_locked, + "device-hotplug lock balance"); + device_hotplug_locked = false; +} + +static void device_lock(struct device *dev) +{ + check(device_hotplug_locked && !dev->locked && !maps_locked, + "device lock order"); + dev->locked = true; +} + +static void device_unlock(struct device *dev) +{ + check(dev->locked && !maps_locked, "device lock balance"); + dev->locked = false; +} + +static void cpu_maps_update_begin(void) +{ + check(device_hotplug_locked && !maps_locked, "maps lock order"); + maps_locked = true; +} + +static void cpu_maps_update_done(void) +{ + check(maps_locked && !write_locked, "maps lock balance"); + if (native_error && !online[1]) + check(present[1] == initially_present, + "failed handback restores original presence before unlock"); + maps_locked = false; +} + +static void cpus_write_lock(void) +{ + check(maps_locked && !write_locked, "CPU write lock order"); + write_locked = true; +} + +static void cpus_write_unlock(void) +{ + check(write_locked, "write lock balance"); + write_locked = false; +} + +static void set_cpu_present(unsigned int cpu, bool value) +{ + check(device_hotplug_locked && maps_locked && write_locked, + "presence changed within native transaction"); + present[cpu] = value; +} + +static int try_online_node(int node) +{ + return node_error; +} + +static void cpu_device_update_node(struct device *dev, int from) +{ + check(device_hotplug_locked && dev->locked, + "NUMA bookkeeping serialized"); +} + +static int kobject_uevent(int *kobj, int action) +{ + check(device_hotplug_locked && maps_locked, + "online event within transaction"); + events++; + return 0; +} + +static int _cpu_up(unsigned int cpu, int frozen, enum cpuhp_state target) +{ + check(maps_locked && !write_locked && present[cpu], + "native bringup preconditions"); + check(!pool.cpus_lock && cpu_devices[cpu].dev.locked, + "native bringup holds device lock, not pool lock"); + check(mk_cpu_set_contains(own_set.nr ? &own_set : NULL, 101), + "local ownership reserved before bringup"); + check(mk_cpu_set_contains(&pool_set, 101), + "wake-slot membership preserved during bringup"); + native_calls++; + if (native_error) { + online[cpu] = partial_online; + return native_error; + } + if (!bad_success) { + online[cpu] = true; + cpuhp_state[cpu].state = target; + cpuhp_state[cpu].target = target; + } + return 0; +} + +/* Original add_cpu path, to check the old caller without a kernel boot. */ +static int device_online(struct device *dev) +{ + int ret; + + if (!dev) + return -ENODEV; + device_lock(dev); + if (!cpu_possible(dev->id) || nodes[dev->id] == NUMA_NO_NODE || + node_error) { + ret = node_error ? node_error : -ENODEV; + goto out; + } + cpu_maps_update_begin(); + if (cpu_hotplug_disabled) + ret = -EBUSY; + else if (!bootable) + ret = -EPERM; + else + ret = _cpu_up(dev->id, 0, CPUHP_ONLINE); + cpu_maps_update_done(); + if (!ret) { + dev->offline = false; + events++; + } +out: + device_unlock(dev); + return ret; +} + +static void cpuhp_online_cpu_device(unsigned int cpu); +#include "handback-native.h" + +static int register_cpu(struct cpu *c, int cpu) +{ + check(device_hotplug_locked && !present[cpu], + "registration while absent and serialized"); + if (registration_error) + return registration_error; + registered[cpu] = true; + return 0; +} + +static unsigned int next_present(unsigned int cpu) +{ + while (cpu < nr_cpu_ids && !present[cpu]) + cpu++; + return cpu; +} + +#define for_each_present_cpu(cpu) \ + for ((cpu) = next_present(0); (unsigned int)(cpu) < nr_cpu_ids; \ + (cpu) = next_present((cpu) + 1)) +static u64 arch_cpu_physical_id(unsigned int cpu) +{ + return 100 + cpu; +} + +static int arch_cpu_from_physical_id(u64 id) +{ + return id == 101 ? mapped_cpu : -1; +} + +static void mk_instance_ipi_retarget(struct mk_instance *instance) +{ + retargets++; +} + +static void list_add_tail(struct list_head *entry, struct list_head *head) +{ +} + +#include "handback-hotplug.h" + +static void reset(const char *name) +{ + unsigned int cpu; + + scenario = name; + scenarios++; + check(!device_hotplug_locked && !maps_locked && !write_locked && + !pool.cpus_lock, + "previous locks released"); + free(own_set.ids); + free(pool_set.ids); + memset(&own_set, 0, sizeof(own_set)); + memset(&pool_set, 0, sizeof(pool_set)); + fail_alloc = false; + check(!mk_cpu_set_add(&pool_set, 101), "seed pool set"); + for (cpu = 0; cpu < 4; cpu++) { + possible[cpu] = true; + present[cpu] = false; + online[cpu] = false; + registered[cpu] = true; + cpu_devices[cpu].dev = + (struct device){ .id = cpu, .offline = true }; + cpuhp_state[cpu].state = CPUHP_DEPARTED; + cpuhp_state[cpu].target = CPUHP_DEPARTED; + nodes[cpu] = 0; + } + native_error = 0; + node_error = 0; + registration_error = 0; + native_calls = 0; + events = 0; + retargets = 0; + partial_online = false; + bad_success = false; + cpu_hotplug_disabled = false; + bootable = true; + initially_present = false; + mapped_cpu = 1; +} + +static void rejected(int expected) +{ + check(mk_do_cpu_add(101, 0, 0) == expected, + "preflight error propagated"); + check(!native_calls && !present[1] && !online[1], + "no bringup or presence exposure"); + check(!mk_cpu_set_contains(&own_set, 101) && + mk_cpu_set_contains(&pool_set, 101), + "known preflight rejection preserves original ownership"); +} + +int main(void) +{ + reset("successful handback"); + check(!mk_do_cpu_add(101, 0, 0), "successful return"); + check(present[1] && online[1] && native_calls == 1, "CPU online once"); + check(mk_cpu_set_contains(&own_set, 101) && retargets == 1, + "owned and retargeted"); + check(!cpu_devices[1].dev.offline && events == 1, + "device online event"); + reset("ownership allocation fails"); + fail_alloc = true; + rejected(-ENOMEM); + reset("CPU device registration fails"); + registered[1] = false; + registration_error = -ENOMEM; + rejected(-ENOMEM); + reset("native hotplug disabled"); + cpu_hotplug_disabled = true; + rejected(-EBUSY); + reset("SMT policy rejects CPU"); + bootable = false; + rejected(-EPERM); + reset("NUMA node online fails"); + node_error = -ENOMEM; + rejected(-ENOMEM); + reset("invalid absent CPU state"); + cpuhp_state[1].state = CPUHP_INTERMEDIATE; + rejected(-EINVAL); + reset("native bringup fails after attempt"); + native_error = -EIO; + check(mk_do_cpu_add(101, 0, 0) == -EIO, "native failure returned"); + check(!present[1] && !online[1], "absent mask restored"); + check(mk_cpu_set_contains(&own_set, 101) && + !mk_cpu_set_contains(&pool_set, 101), + "uncertain CPU retained, not assignable"); + check(!events && !retargets, "failure not advertised as online"); + reset("attempted EBUSY is not retried"); + native_error = -EBUSY; + check(mk_do_cpu_add(101, 0, 0) == -EBUSY && native_calls == 1, + "one native attempt"); + check(!mk_cpu_set_contains(&pool_set, 101), + "attempted EBUSY not mistaken for preflight"); + reset("incomplete rollback remains locally owned"); + native_error = -EIO; + partial_online = true; + check(mk_do_cpu_add(101, 0, 0) == -EIO, "incomplete rollback error"); + check(present[1] && online[1], "online CPU never marked absent"); + check(mk_cpu_set_contains(&own_set, 101) && + !mk_cpu_set_contains(&pool_set, 101), + "incomplete rollback not assigned elsewhere"); + reset("native false success"); + bad_success = true; + check(mk_do_cpu_add(101, 0, 0) == -EIO, "offline success rejected"); + check(!present[1] && !mk_cpu_set_contains(&pool_set, 101), + "false success retains safely"); + reset("never locally started CPU"); + cpuhp_state[1].state = CPUHP_OFFLINE; + check(!mk_do_cpu_add(101, 0, 0), "OFFLINE enumerated CPU can join"); + check(native_calls == 1 && present[1] && online[1], + "first local bringup completed"); + reset("impossible logical CPU"); + possible[1] = false; + registered[1] = false; + rejected(-ENODEV); + check(!registered[1], "impossible CPU not registered"); + reset("logical CPU outside possible map"); + mapped_cpu = nr_cpu_ids; + rejected(-ENODEV); + reset("missing NUMA mapping"); + nodes[1] = NUMA_NO_NODE; + rejected(-ENODEV); + reset("register absent enumerated CPU"); + registered[1] = false; + check(!mk_do_cpu_add(101, 0, 0) && registered[1], + "registered and brought up"); + check(native_calls == 1 && present[1] && events == 1, + "registration precedes online"); + reset("owned CPU rejected before attempt"); + check(!mk_cpu_set_add(&own_set, 101), "seed ownership"); + cpu_hotplug_disabled = true; + check(mk_do_cpu_add(101, 0, 0) == -EBUSY && !native_calls, + "preflight failed"); + check(mk_cpu_set_contains(&own_set, 101), + "preexisting ownership preserved"); + reset("present CPU fails native bringup"); + initially_present = true; + present[1] = true; + cpuhp_state[1].state = CPUHP_OFFLINE; + native_error = -EIO; + check(mk_do_cpu_add(101, 0, 0) == -EIO && present[1], + "original presence preserved"); + check(mk_cpu_set_contains(&own_set, 101) && + !mk_cpu_set_contains(&pool_set, 101), + "failed present CPU retained"); + reset("already online CPU"); + present[1] = true; + online[1] = true; + cpuhp_state[1].state = CPUHP_ONLINE; + cpu_devices[1].dev.offline = false; + check(!mk_do_cpu_add(101, 0, 0), "idempotent return"); + check(!native_calls && !events && retargets == 1, + "no duplicate online event or bringup"); + check(!device_hotplug_locked && !maps_locked && !write_locked, + "final locks released"); + free(own_set.ids); + free(pool_set.ids); + printf("CPU handback: %u scenarios, %u failures\n", scenarios, + failures); + return failures ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/tools/testing/selftests/multikernel/cpu_handback_test.sh b/tools/testing/selftests/multikernel/cpu_handback_test.sh new file mode 100644 index 00000000000000..5805efe17a3c70 --- /dev/null +++ b/tools/testing/selftests/multikernel/cpu_handback_test.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-cpu-handback.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM + +awk '/^int (add_cpu|reclaim_cpu)\(/ { copy = 1 } + /^static void cpuhp_online_cpu_device\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/cpu.c" > "$build/handback-native.h" +awk '/^struct mk_hotplug_op \{/ { copy = 1 } + /^static void mk_account_cpu_online\(/ { copy = 1 } + /^static int mk_(cpu_to_logical|do_cpu_add)\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/hotplug.c" \ + > "$build/handback-hotplug.h" +awk '/^typedef u64 mk_phys_cpu_t;/ { copy = 1 } + /^\/\*\*$/ && copy && seen { exit } + copy { print; seen = 1 }' "$src/include/linux/multikernel.h" \ + > "$build/handback-types.h" +awk '/^static int mk_cpu_set_index\(/ { copy = 1 } + /^(bool|int) mk_cpu_set_(contains|reserve|add|del)\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/cpuset.c" \ + > "$build/handback-set.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror \ + -Wno-unused-parameter -Wno-unused-function -Wno-unused-variable \ + -Wno-sign-compare -I"$build" \ + "$test_dir/cpu_handback_test.c" -o "$build/cpu_handback_test" +"$build/cpu_handback_test" diff --git a/tools/testing/selftests/multikernel/cpu_hotplug_config_test.sh b/tools/testing/selftests/multikernel/cpu_hotplug_config_test.sh new file mode 100644 index 00000000000000..b9388774342a80 --- /dev/null +++ b/tools/testing/selftests/multikernel/cpu_hotplug_config_test.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# Evaluate real x86 Kconfig; compile affected objects, not a guest runtime. +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=${MK_SOURCE:-$(CDPATH= cd -- "$test_dir/../../../.." && pwd)} +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-cpu-hotplug-config.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +configure() +{ + make -s -C "$src" ARCH=x86_64 O="$build" "$1" +} +configure x86_64_defconfig +"$src/scripts/config" --file "$build/.config" \ + -e KEXEC_FILE -e CONTIG_ALLOC -e MEMORY_HOTPLUG -e MEMORY_HOTREMOVE -e MULTIKERNEL +configure olddefconfig +if grep -qx 'CONFIG_MULTIKERNEL=y' "$build/.config"; then + echo 'FAIL: multikernel enabled without boot Device Tree support' + exit 1 +fi +"$src/scripts/config" --file "$build/.config" -e OF -e MULTIKERNEL +configure olddefconfig +grep -qx 'CONFIG_MULTIKERNEL=y' "$build/.config" +grep -qx 'CONFIG_HOTPLUG_CPU=y' "$build/.config" +grep -qx 'CONFIG_OF=y' "$build/.config" +grep -qx 'CONFIG_OF_FLATTREE=y' "$build/.config" +cp "$build/.config" "$build/supported.config" +"$src/scripts/config" --file "$build/.config" -d SMP -d HOTPLUG_CPU -e MULTIKERNEL +configure olddefconfig +grep -qx '# CONFIG_SMP is not set' "$build/.config" +if grep -qx 'CONFIG_MULTIKERNEL=y' "$build/.config"; then + echo 'FAIL: UP configuration permits multikernel without CPU hotplug' + exit 1 +fi +make -s -C "$src" ARCH=x86_64 O="$build" -j2 kernel/cpu.o drivers/base/cpu.o +echo 'PASS: UP excludes multikernel; ordinary CPU objects compile' +cp "$build/supported.config" "$build/.config" +"$src/scripts/config" --file "$build/.config" -e SMP -e MULTIKERNEL +configure olddefconfig +grep -qx 'CONFIG_MULTIKERNEL=y' "$build/.config" +grep -qx 'CONFIG_HOTPLUG_CPU=y' "$build/.config" +make -s -C "$src" ARCH=x86_64 O="$build" -j2 kernel/multikernel/hotplug.o +echo 'PASS: SMP/hotplug restores multikernel; resource-hotplug object compiles' diff --git a/tools/testing/selftests/multikernel/cpu_ownership_test.c b/tools/testing/selftests/multikernel/cpu_ownership_test.c new file mode 100644 index 00000000000000..d176c325bde15d --- /dev/null +++ b/tools/testing/selftests/multikernel/cpu_ownership_test.c @@ -0,0 +1,592 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real ownership/handoff code; firmware, pending waits and allocation injected. */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define u64 __u64 +#define GFP_KERNEL 0 +static bool fail_alloc, fail_set_alloc, fail_after_wake; +#define kzalloc(n, flags) ((void)(flags), fail_alloc ? NULL : calloc(1, n)) +#define krealloc_array(p, n, s, flags) \ + ((void)(flags), fail_alloc || fail_set_alloc ? NULL : realloc(p, (n) * (s))) +#define kfree free +#define max_t(type, a, b) ((type)(a) > (type)(b) ? (type)(a) : (type)(b)) +#define pr_err(...) fprintf(stderr, __VA_ARGS__) +#define pr_warn(...) fprintf(stderr, __VA_ARGS__) +#define pr_info(...) do { } while (0) + +static int scnprintf(char *buf, size_t size, const char *fmt, ...) +{ + va_list ap; + int n; + + va_start(ap, fmt); + n = vsnprintf(buf, size, fmt, ap); + va_end(ap); + return n < 0 ? n : (size_t)n < size ? n : size ? (int)size - 1 : 0; +} + +#include "cpu-types.h" +#include "cpu-set.h" + +struct mk_spawn_context { + unsigned long repark_park_phys, repark_park_cr3, repark_slot_phys; + unsigned long park_phys, park_cr3, identity_cr3, kernel_entry; + unsigned long trampoline_phys, self_phys, flags; +}; + +/* Only fields reached by the extracted functions; no alternate ownership model. */ +struct mk_instance { + int id, state; + const char *name; + struct mk_cpu_set *cpus, *cpus_on_slot; + mk_phys_cpu_t ipi_target; + bool ipi_lock, resource_uncertain; + struct { + struct mk_spawn_context *spawn_ctx; + unsigned long spawn_ctx_phys; + } arch; +}; + +struct mk_pool { + struct mk_cpu_set *cpus; + bool park_lock, cpus_lock; + struct { + struct mk_spawn_context *slot; + unsigned long park_phys, cr3, slot_phys; + } arch; +}; + +#define MK_STATE_ACTIVE 2 +#define MK_STATE_LOADED 1 +#define MK_STATE_FAILED 3 +#define MK_SPAWN_F_REPARK 1 +#define virt_to_phys(p) ((unsigned long)(p)) +#define per_cpu(name, cpu) name(cpu) + +static u32 x86_cpu_to_apicid(int cpu) +{ + return cpu; +} +static struct mk_instance self = { .id = 1 }, target; +static struct mk_instance *mk_self = &self; +static struct mk_instance *host_instance; +static bool mk_overlay_mutex = true; +static struct mk_pool pool, *mk_pool = &pool; +static struct mk_spawn_context host_slot, spawn_ctx; +static void *test_phys_to_virt(unsigned long phys) +{ + if (phys == 0x123000 || phys == (unsigned long)&spawn_ctx) + return &spawn_ctx; + assert(phys == 0x456000 || phys == (unsigned long)&host_slot); + return &host_slot; +} +#define __va(p) test_phys_to_virt(p) +static int references, wakes, wake_fail_at, send_error, wait_error; +static unsigned int live_pending, sends; +static bool owned_at_wake, tracked_at_wake, owned_at_send; +static bool confirm_mode; +static unsigned int running_probes; +static int drain_error; +static unsigned int drains; + +int mk_ipi_poll(void) +{ + assert(mk_overlay_mutex); + drains++; + return drain_error; +} +static mk_phys_cpu_t test_cpu; + +/* Deterministic lock boundary, not a concurrency test. */ +static void unlock(bool **lock) +{ + assert(**lock); + **lock = false; +} + +static bool *lock(bool *held) +{ + assert(!*held); + *held = true; + return held; +} + +#define guard(kind) bool *held_##kind __attribute__((cleanup(unlock))) = lock +#define lockdep_assert_held(held) assert(*(held)) +#define mutex_lock(held) ((void)lock(held)) +#define mutex_unlock(held) do { assert(*(held)); *(held) = false; } while (0) +#define raw_spin_lock_irqsave(m, flags) \ + do { assert(!*(m)); *(m) = true; (flags) = 0; } while (0) +#define raw_spin_unlock_irqrestore(m, flags) \ + do { assert(*(m)); *(m) = false; (void)(flags); } while (0) +#include "cpu-retarget.h" + +static int arch_cpu_from_physical_id(mk_phys_cpu_t cpu) +{ + return cpu == test_cpu || cpu == 7 ? 0 : -EINVAL; +} + +/* Check production callers, while setup below can construct unpublished sets. */ +static int checked_reserve(struct mk_cpu_set *set, unsigned int extra) +{ + assert(set != pool.cpus || pool.cpus_lock); + return mk_cpu_set_reserve(set, extra); +} + +static int checked_add(struct mk_cpu_set *set, mk_phys_cpu_t cpu) +{ + assert(set != pool.cpus || pool.cpus_lock); + return mk_cpu_set_add(set, cpu); +} + +static bool checked_del(struct mk_cpu_set *set, mk_phys_cpu_t cpu) +{ + assert(set != pool.cpus || pool.cpus_lock); + return mk_cpu_set_del(set, cpu); +} +#define mk_cpu_set_reserve checked_reserve +#define mk_cpu_set_add checked_add +#define mk_cpu_set_del checked_del + +#include "cpu-transfer.h" + +/* Mailbox publication can time out after a CPU has consumed the request. */ +static int mk_slot_begin(struct mk_spawn_context *ctx) +{ + return ctx ? 0 : -ENODEV; +} + +static bool mk_slot_idle(struct mk_spawn_context *ctx) +{ + return true; +} + +static int mk_slot_wake(struct mk_spawn_context *ctx, u32 cpu, const char *what) +{ + assert(pool.park_lock); + assert(!pool.cpus_lock); + wakes++; + owned_at_wake = mk_cpu_set_contains(target.cpus, cpu) && + !mk_cpu_set_contains(pool.cpus, cpu); + tracked_at_wake = mk_cpu_set_contains(target.cpus_on_slot, cpu); + if (confirm_mode) { + if (cpu == 7) { + running_probes++; + return -ETIMEDOUT; + } + /* The uncertain handoff never left the host park slot. */ + return ctx == &host_slot ? 0 : -ETIMEDOUT; + } + if (fail_after_wake) + fail_alloc = true; + return wakes == wake_fail_at ? -ETIMEDOUT : 0; +} + +#include "cpu-x86.h" +#include "cpu-confirm.h" + +static struct mk_instance *mk_instance_find(int id) +{ + if (id != target.id) + return NULL; + references++; + return ⌖ +} + +static void mk_instance_put(struct mk_instance *instance) +{ + assert(instance == &target && references > 0); + references--; +} + +struct mk_pending_msg { u64 id; }; + +static struct mk_pending_msg *mk_msg_pending_add(u32 type, u32 op, u64 resource) +{ + struct mk_pending_msg *p = kzalloc(sizeof(*p), GFP_KERNEL); + + if (!p) + return ERR_PTR(-ENOMEM); + p->id = 17; + live_pending++; + return p; +} + +static u64 mk_msg_pending_id(struct mk_pending_msg *p) +{ + return p->id; +} + +static void mk_msg_pending_cancel(struct mk_pending_msg *p) +{ + assert(live_pending == 1); + live_pending--; + free(p); +} + +static int mk_msg_pending_wait(struct mk_pending_msg *p, unsigned long timeout) +{ + assert(!pool.cpus_lock && !pool.park_lock); + mk_msg_pending_cancel(p); + return wait_error; +} + +static int mk_send_message_id(int id, u32 type, u32 op, u64 req_id, + const void *data, size_t size) +{ + const struct mk_cpu_resource_payload *p = data; + + assert(id == target.id && type == MK_MSG_RESOURCE && req_id == 17); + assert(op == MK_RES_CPU_ADD || op == MK_RES_CPU_REMOVE); + assert(size == sizeof(*p) && p->cpu_id == test_cpu); + owned_at_send = mk_cpu_set_contains(target.cpus, test_cpu) && + !mk_cpu_set_contains(pool.cpus, test_cpu); + sends++; + return send_error; +} + +static bool local_online; +static int hotplug_error; +static unsigned int hotplug_calls; +#define mk_cpu_to_logical(cpu) arch_cpu_from_physical_id(cpu) +#define cpu_online(cpu) ((void)(cpu), local_online) +static int mk_do_cpu_remove(mk_phys_cpu_t cpu) +{ + assert(!pool.cpus_lock && !pool.park_lock); + hotplug_calls++; + if (hotplug_error) + return hotplug_error; + local_online = false; + return 0; +} + +static int mk_do_cpu_add(mk_phys_cpu_t cpu, u32 node, u32 flags) +{ + assert(!pool.cpus_lock && !pool.park_lock); + hotplug_calls++; + /* CPUHP's wake callback can take both locks during local pool return. */ + mutex_lock(&pool.park_lock); + mutex_lock(&pool.cpus_lock); + assert(mk_cpu_set_contains(pool.cpus, cpu)); + mutex_unlock(&pool.cpus_lock); + mutex_unlock(&pool.park_lock); + if (hotplug_error) + return hotplug_error; + local_online = true; + return 0; +} +#include "cpu-send.h" +#undef mk_cpu_set_reserve +#undef mk_cpu_set_add +#undef mk_cpu_set_del + +static unsigned int failures, checks; + +static void expect(const char *name, int got, int want) +{ + checks++; + if (got != want) { + fprintf(stderr, "FAIL: %s: got %d, want %d\n", name, got, want); + failures++; + } else { + printf("PASS: %s\n", name); + } +} + +static void reset(void) +{ + mk_cpu_set_free(target.cpus); + mk_cpu_set_free(target.cpus_on_slot); + mk_cpu_set_free(pool.cpus); + assert(!references && !live_pending && !pool.park_lock && !pool.cpus_lock); + local_online = true; + hotplug_error = hotplug_calls = 0; + fail_alloc = fail_set_alloc = fail_after_wake = false; + wakes = wake_fail_at = send_error = wait_error = 0; + sends = 0; + owned_at_wake = tracked_at_wake = owned_at_send = false; + confirm_mode = false; + running_probes = 0; + drain_error = 0; + drains = 0; + test_cpu = 65537; + target = (struct mk_instance) { + .id = 2, .state = MK_STATE_ACTIVE, .name = "spawn", + .cpus = mk_cpu_set_alloc(), .cpus_on_slot = mk_cpu_set_alloc(), + .arch = { .spawn_ctx = &spawn_ctx, .spawn_ctx_phys = 0x123000 }, + }; + pool = (struct mk_pool) { + .cpus = mk_cpu_set_alloc(), + .arch = { .slot = &host_slot, .slot_phys = 0x456000 }, + }; + spawn_ctx = (struct mk_spawn_context) { .park_phys = 0x789000 }; + assert(!mk_cpu_set_add(pool.cpus, test_cpu)); +} + +static void retained(void) +{ + expect("CPU remains owned by target", + mk_cpu_set_contains(target.cpus, test_cpu), 1); + expect("uncertain CPU is not assignable", + mk_cpu_set_contains(pool.cpus, test_cpu), 0); + expect("uncertain handoff remains tracked", + mk_cpu_set_contains(target.cpus_on_slot, test_cpu), 1); + expect("pending waiter cleaned up", live_pending, 0); + expect("instance reference released", references, 0); +} + +int main(void) +{ + reset(); + mk_cpu_set_clear(pool.cpus); + pool.cpus->cap = 0; /* Force reserve to grow before the CPU departs. */ + fail_set_alloc = true; + expect("pool growth failure precedes CPU removal", mk_pool_cpu_add(test_cpu), -ENOMEM); + expect("failed growth leaves CPU online", local_online, 1); + expect("failed growth never calls CPUHP", hotplug_calls, 0); + fail_set_alloc = false; + hotplug_error = -EBUSY; + expect("failed departure does not donate", mk_pool_cpu_add(test_cpu), -EBUSY); + expect("failed departure leaves pool empty", pool.cpus->nr, 0); + hotplug_error = 0; + expect("local donation grows and publishes under set lock", mk_pool_cpu_add(test_cpu), 0); + expect("local donation records CPU", mk_cpu_set_contains(pool.cpus, test_cpu), 1); + hotplug_error = -EIO; + expect("failed local wake retains pool owner", mk_pool_cpu_remove(test_cpu, 0, 0), -EIO); + expect("failed local wake keeps membership", mk_cpu_set_contains(pool.cpus, test_cpu), 1); + hotplug_error = 0; + expect("local return lets CPUHP acquire pool locks", mk_pool_cpu_remove(test_cpu, 0, 0), 0); + expect("local return deletes under set lock", pool.cpus->nr, 0); + + reset(); + expect("initial add", mk_send_cpu_add(2, test_cpu, 0, 0), 0); + expect("ownership precedes handoff", owned_at_wake, 1); + expect("tracking precedes handoff", tracked_at_wake, 1); + expect("ownership precedes remote start request", owned_at_send, 1); + expect("CPU addition refreshes physical doorbell", target.ipi_target == test_cpu, 1); + retained(); + expect("repeat add is rejected", mk_send_cpu_add(2, test_cpu, 0, 0), -EBUSY); + expect("repeat add publishes nothing", sends, 1); + + reset(); + fail_alloc = true; + expect("pending allocation fails before handoff", + mk_send_cpu_add(2, test_cpu, 0, 0), -ENOMEM); + expect("allocation failure never wakes CPU", wakes, 0); + expect("allocation failure leaves pool ownership", + mk_cpu_set_contains(pool.cpus, test_cpu), 1); + expect("allocation failure leaves target empty", target.cpus->nr, 0); + + reset(); + fail_set_alloc = true; + expect("ownership reservation fails before handoff", + mk_send_cpu_add(2, test_cpu, 0, 0), -ENOMEM); + expect("reservation failure never wakes CPU", wakes, 0); + expect("reservation failure leaves pool ownership", + mk_cpu_set_contains(pool.cpus, test_cpu), 1); + expect("reservation failure cleans pending", live_pending, 0); + + reset(); + assert(!mk_cpu_set_reserve(target.cpus, 1)); + fail_set_alloc = true; + expect("tracking allocation fails before publication", + mk_send_cpu_add(2, test_cpu, 0, 0), -ENOMEM); + expect("tracking allocation failure never wakes CPU", wakes, 0); + expect("preowned CPU retained on tracking failure", + mk_cpu_set_contains(target.cpus, test_cpu), 1); + expect("tracking failure does not make CPU assignable", + mk_cpu_set_contains(pool.cpus, test_cpu), 0); + expect("tracking failure cancels pending", live_pending, 0); + + reset(); + wake_fail_at = 1; + expect("handoff timeout", mk_send_cpu_add(2, test_cpu, 0, 0), -ETIMEDOUT); + retained(); + expect("failed handoff sends no start request", sends, 0); + + reset(); + send_error = -ENOSPC; + expect("transport failure after handoff", + mk_send_cpu_add(2, test_cpu, 0, 0), -ENOSPC); + retained(); + expect("transport failure adds no wake beyond confirmed handoff", wakes, 2); + + reset(); + wait_error = -ETIMEDOUT; + expect("lost start ACK", mk_send_cpu_add(2, test_cpu, 0, 0), -ETIMEDOUT); + retained(); + expect("lost ACK adds no wake beyond confirmed handoff", wakes, 2); + + reset(); + wait_error = -EIO; + expect("error ACK retains ownership", mk_send_cpu_add(2, test_cpu, 0, 0), -EIO); + retained(); + + reset(); + fail_after_wake = true; + expect("no late ownership or waiter allocation", + mk_send_cpu_add(2, test_cpu, 0, 0), 0); + retained(); + + reset(); + assert(!mk_instance_transfer_cpus(&target, pool.cpus)); + assert(!mk_cpu_set_add(target.cpus_on_slot, test_cpu)); + wake_fail_at = 1; + expect("remove cannot reclaim on repark timeout", + mk_send_cpu_remove(2, test_cpu), -ETIMEDOUT); + retained(); + wake_fail_at = 0; + expect("confirmed repark permits removal", mk_send_cpu_remove(2, test_cpu), 0); + expect("removed CPU returned to pool", + mk_cpu_set_contains(pool.cpus, test_cpu), 1); + expect("removed CPU no longer target-owned", target.cpus->nr, 0); + expect("confirmed host return clears tracking", target.cpus_on_slot->nr, 0); + expect("CPU return removes obsolete doorbell", target.ipi_target == MK_PHYS_CPU_INVALID, 1); + + reset(); + assert(!mk_instance_transfer_cpus(&target, pool.cpus)); + wake_fail_at = 1; + expect("first boot wake timeout", + mk_spawn_cpu(&target, test_cpu, &spawn_ctx), -ETIMEDOUT); + retained(); + expect("boot tracking precedes publication", tracked_at_wake, 1); + + reset(); + assert(!mk_instance_transfer_cpus(&target, pool.cpus)); + assert(!mk_cpu_set_add(target.cpus, 7)); + wake_fail_at = 1; + expect("secondary handoff timeout", + mk_spawn_cpu(&target, 7, &spawn_ctx), -ETIMEDOUT); + retained(); + expect("secondary tracking precedes publication", tracked_at_wake, 1); + expect("boot is not woken after secondary failure", wakes, 1); + expect("unpublished boot CPU is not tracked", + mk_cpu_set_contains(target.cpus_on_slot, 7), 0); + + reset(); + assert(!mk_instance_transfer_cpus(&target, pool.cpus)); + assert(!mk_cpu_set_add(target.cpus, test_cpu + 1)); + assert(!mk_cpu_set_add(target.cpus, 7)); + /* First CPU claims source and destination; second source claim fails. */ + wake_fail_at = 3; + expect("partial secondary handoff failure", + mk_spawn_cpu(&target, 7, &spawn_ctx), -ETIMEDOUT); + retained(); + expect("second uncertain CPU remains tracked", + mk_cpu_set_contains(target.cpus_on_slot, test_cpu + 1), 1); + expect("partial handoff does not wake boot CPU", wakes, 3); + expect("unpublished boot stays untracked", + mk_cpu_set_contains(target.cpus_on_slot, 7), 0); + + reset(); + wake_fail_at = 1; + expect("uncertain add before confirmation", + mk_send_cpu_add(2, test_cpu, 0, 0), -ETIMEDOUT); + assert(!mk_cpu_set_add(target.cpus, 7)); + assert(!mk_cpu_set_add(target.cpus_on_slot, 7)); + confirm_mode = true; + expect("uncertain tracked CPU must not escape confirmation", + mk_instance_confirm_parked(&target), -EBUSY); + expect("running CPU probed only on recorded slot", running_probes, 1); + expect("failed tracked CPU never cleared by fallback", + mk_cpu_set_contains(target.cpus_on_slot, test_cpu), 1); + expect("running CPU remains tracked", + mk_cpu_set_contains(target.cpus_on_slot, 7), 1); + expect("host confirmation does not return ownership", + mk_cpu_set_contains(target.cpus, test_cpu), 1); + + reset(); + assert(!mk_instance_transfer_cpus(&target, pool.cpus)); + assert(!mk_cpu_set_add(target.cpus, test_cpu + 1)); + assert(!mk_cpu_set_add(target.cpus_on_slot, test_cpu)); + assert(!mk_cpu_set_add(target.cpus_on_slot, test_cpu + 1)); + confirm_mode = true; + expect("failed recorded-slot confirmations do not fall back", + mk_instance_confirm_parked(&target), -EBUSY); + expect("failed tracked CPUs stay tracked", target.cpus_on_slot->nr, 2); + expect("failed confirmation retains both owners", target.cpus->nr, 2); + mk_cpu_set_clear(target.cpus_on_slot); + expect("assigned CPUs confirm despite empty tracking", + mk_instance_confirm_parked(&target), 0); + + reset(); + assert(!mk_instance_transfer_cpus(&target, pool.cpus)); + assert(!mk_cpu_set_add(target.cpus, 7)); + confirm_mode = true; + expect("untracked assigned CPU must not escape confirmation", + mk_instance_confirm_parked(&target), -EBUSY); + expect("untracked running CPU probed only on selected slot", running_probes, 1); + expect("running peer cannot start inbox retirement", drains, 0); + expect("untracked running CPU retains ownership", mk_cpu_set_contains(target.cpus, 7), 1); + + reset(); + assert(!mk_instance_transfer_cpus(&target, pool.cpus)); + target.arch.spawn_ctx = NULL; + confirm_mode = true; + expect("before first spawn confirm host park slot", + mk_instance_confirm_parked(&target), 0); + expect("parked peer retires queued requests", drains, 1); + drain_error = -EBUSY; + expect("unpublished inbox slot prevents reusable confirmation", + mk_instance_confirm_parked(&target), -EBUSY); + drain_error = 0; + target.resource_uncertain = true; + expect("parked CPUs do not settle uncertain resource work", + mk_instance_confirm_parked(&target), -EBUSY); + expect("uncertain resource blocks CPU removal", + mk_send_cpu_remove(2, test_cpu), -EBUSY); + expect("quarantine retains parked CPU ownership", + mk_cpu_set_contains(target.cpus, test_cpu), 1); + expect("uncertain resource blocks CPU addition", + mk_send_cpu_add(2, test_cpu + 1, 0, 0), -EBUSY); + + reset(); + assert(!mk_instance_transfer_cpus(&target, pool.cpus)); + expect("successful first boot", + mk_spawn_cpu(&target, test_cpu, &spawn_ctx), 0); + expect("successful re-spawn uses existing tracking", + mk_spawn_cpu(&target, test_cpu, &spawn_ctx), 0); + expect("re-spawn tracking has no duplicates", target.cpus_on_slot->nr, 1); + + reset(); + mk_pool = NULL; + expect("active add without a pool is rejected", + mk_send_cpu_add(2, test_cpu, 0, 0), -EBUSY); + expect("missing pool publishes nothing", wakes, 0); + mk_pool = &pool; + + reset(); + assert(!mk_cpu_set_add(pool.cpus, test_cpu + 1)); + expect("unmapped physical CPU is rejected", + mk_send_cpu_add(2, test_cpu + 1, 0, 0), -EBUSY); + expect("mapping failure leaves CPU assignable", + mk_cpu_set_contains(pool.cpus, test_cpu + 1), 1); + expect("mapping failure publishes nothing", wakes, 0); + expect("mapping failure cleans pending", live_pending, 0); + + reset(); + mk_cpu_set_clear(pool.cpus); + test_cpu = 0x100010001ULL; + assert(!mk_cpu_set_add(pool.cpus, test_cpu)); + target.state = MK_STATE_LOADED; + expect("nonrunning transfer preserves sparse 64-bit ID", + mk_send_cpu_add(2, test_cpu, 0, 0), 0); + expect("full-width ID remains assigned", + mk_cpu_set_contains(target.cpus, 0x100010001ULL), 1); + expect("nonrunning transfer does not wake CPU", wakes, 0); + expect("return all CPUs under set lock", mk_instance_return_cpus(&target, target.cpus), 0); + expect("returned CPU is in the pool", mk_cpu_set_contains(pool.cpus, test_cpu), 1); + reset(); + mk_cpu_set_free(target.cpus); + mk_cpu_set_free(target.cpus_on_slot); + mk_cpu_set_free(pool.cpus); + printf("%u checks, %u failures\n", checks, failures); + return failures ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/tools/testing/selftests/multikernel/cpu_ownership_test.sh b/tools/testing/selftests/multikernel/cpu_ownership_test.sh new file mode 100644 index 00000000000000..f333c35fb196fb --- /dev/null +++ b/tools/testing/selftests/multikernel/cpu_ownership_test.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# Compile production CPU sets, transfers, senders and x86 handoff functions. +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-cpu-ownership.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM + +awk '/^typedef u64 mk_phys_cpu_t;/ { copy = 1 } + /^\/\*\*$/ && copy && seen { exit } + copy { print; seen = 1 }' "$src/include/linux/multikernel.h" \ + > "$build/cpu-types.h" +awk '/^#define MK_MSG_IO / { copy = 1 } + /^\/\* Opaque type/ { copy = 0 } + copy { print }' "$src/include/linux/multikernel.h" >> "$build/cpu-types.h" +sed '/^#include /d' "$src/kernel/multikernel/cpuset.c" > "$build/cpu-set.h" +awk '/^void mk_instance_ipi_retarget\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/ipi.c" \ + > "$build/cpu-retarget.h" +awk '/^int mk_instance_(transfer|return)_cpus\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/core.c" \ + > "$build/cpu-transfer.h" +awk '/^int mk_instance_confirm_parked\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/core.c" \ + > "$build/cpu-confirm.h" +awk '/^static struct mk_cpu_set \*mk_slot_set\(/ { copy = 1 } + /^static int mk_(publish_repark|repark_to_instance)\(/ { copy = 1 } + /^int mk_(spawn_cpu|arch_confirm_parked)\(/ { copy = 1 } + /^int mk_repark_cpu_to_(instance|host)\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/arch/x86/multikernel/spawn.c" \ + > "$build/cpu-x86.h" +awk '/^int mk_(send|pool)_cpu_(add|remove)\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/hotplug.c" \ + > "$build/cpu-send.h" + +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -Wno-unused-parameter \ + -I"$build" -I"$src/tools/include" "$test_dir/cpu_ownership_test.c" \ + -o "$build/cpu_ownership_test" +"$build/cpu_ownership_test" diff --git a/tools/testing/selftests/multikernel/cpu_work_test.c b/tools/testing/selftests/multikernel/cpu_work_test.c new file mode 100644 index 00000000000000..4298f08ad9c989 --- /dev/null +++ b/tools/testing/selftests/multikernel/cpu_work_test.c @@ -0,0 +1,707 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Production CPU receive/worker/init/cleanup paths; controlled scheduler. */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define u64 __u64 +typedef u64 mk_phys_cpu_t; +#define __init +#define GFP_ATOMIC 0 +#define WQ_MEM_RECLAIM 1 +#define pr_err(...) do { } while (0) +#define pr_warn(...) do { } while (0) +#define pr_info(...) do { } while (0) +#define DEFINE_MUTEX(name) bool name +struct irq_lock { bool held; }; +#define DEFINE_SPINLOCK(name) struct irq_lock name +#define DEFINE_RAW_SPINLOCK(name) struct irq_lock name +#define mutex_lock(m) do { assert(!*(m)); *(m) = true; } while (0) +#define mutex_unlock(m) do { assert(*(m)); *(m) = false; } while (0) +#define mutex_trylock(m) (*(m) ? false : (*(m) = true)) +static bool mk_overlay_mutex; +#define spin_lock_irqsave(m, flags) \ + do { lock_irq(m, false); (flags) = 0; } while (0) +#define spin_unlock_irqrestore(m, flags) \ + do { unlock_irq(m); (void)(flags); } while (0) +#define raw_spin_lock_irqsave(m, flags) \ + do { lock_irq(m, true); (flags) = 0; } while (0) +#define raw_spin_unlock_irqrestore(m, flags) spin_unlock_irqrestore(m, flags) +#define DECLARE_WAIT_QUEUE_HEAD(name) int name +#define wake_up_all(wait) ((void)(wait)) +#define wait_event(wait, condition) \ + do { (void)(wait); while (!(condition)) run_wait_jobs(); } while (0) +#define PCI_SLOT(devfn) (((devfn) >> 3) & 0x1f) +#define PCI_FUNC(devfn) ((devfn) & 7) +#define strscpy(dst, src, size) snprintf(dst, size, "%s", src) + +#include "work-types.h" + +static unsigned int checks, failures, allocations, queues_live, destroyed; +static bool allocation_fails, queue_allocation_fails, registered, closing_probe; +static int register_error, close_result; +static int system_register_error, kernfs_error; +static unsigned int messaging_cleanups; +static bool online[2]; +static int add_error, remove_error; +static int send_error; +static bool hard_irq; +static unsigned int rt_atomic_lock_errors, irq_allocator_calls; +static unsigned int unlocked_mutations; +static mk_phys_cpu_t ids[] = { 65537, 0x123456789abcdef0ULL }; +static bool memory_added, device_added, shutdown_sent, parked_cpus; +static bool native_hotplug_disabled; +static unsigned int shutdown_jobs_pending; +static bool probe_at_wait, probe_system_registration; +static int wait_probe_results[3], system_probe_result; +static jmp_buf parked; +static struct { int id; } self = { .id = 7 }, *mk_self = &self; + +static void expect(const char *name, int got, int want) +{ + checks++; + if (got != want) { + fprintf(stderr, "FAIL %s: got %d, want %d\n", name, got, want); + failures++; + } +} + +/* Model PREEMPT_RT: ordinary spin locks can sleep; raw locks cannot. */ +static void lock_irq(struct irq_lock *lock, bool raw) +{ + if (hard_irq && !raw) + rt_atomic_lock_errors++; + assert(!lock->held); + lock->held = true; +} + +static void unlock_irq(struct irq_lock *lock) +{ + assert(lock->held); + lock->held = false; +} + +static __maybe_unused void *kmalloc(size_t n, unsigned int flags) +{ + void *p; + + irq_allocator_calls += hard_irq; + if (allocation_fails) + return NULL; + p = malloc(n); + assert(p); + allocations++; + return p; +} + +static void kfree(void *p) +{ + irq_allocator_calls += hard_irq; + if (p) { + assert(allocations); + allocations--; + free(p); + } +} +#define kmalloc_obj(obj, flags) kmalloc(sizeof(obj), flags) + +struct work_struct { void (*func)(struct work_struct *work); }; +struct workqueue_struct { + bool ordered; + unsigned int nr; + struct work_struct *jobs[2 * MK_IPI_RING_SIZE + 8]; +}; +static struct workqueue_struct system_queue; +static struct workqueue_struct *cpu_queue; +#define INIT_WORK(work, fn) ((work)->func = (fn)) + +static __maybe_unused struct workqueue_struct *alloc_ordered_workqueue(const char *name, + unsigned int flags) +{ + if (queue_allocation_fails) + return NULL; + cpu_queue = calloc(1, sizeof(*cpu_queue)); + assert(cpu_queue); + cpu_queue->ordered = true; + queues_live++; + return cpu_queue; +} + +static bool queue_work(struct workqueue_struct *queue, struct work_struct *work) +{ + assert(queue && queue->nr < 2 * MK_IPI_RING_SIZE + 8); + queue->jobs[queue->nr++] = work; + return true; +} + +static __maybe_unused void schedule_work(struct work_struct *work) +{ + queue_work(&system_queue, work); +} + +/* A legal adverse schedule for unordered work; ordered queues must use FIFO. */ +static void run_one(struct workqueue_struct *queue) +{ + unsigned int i, selected; + struct work_struct *work; + + assert(queue->nr); + selected = queue->ordered ? 0 : queue->nr - 1; + work = queue->jobs[selected]; + for (i = selected; i + 1 < queue->nr; i++) + queue->jobs[i] = queue->jobs[i + 1]; + queue->nr--; + work->func(work); +} + +static void drain(struct workqueue_struct *queue) +{ + while (queue && queue->nr) + run_one(queue); +} + +static __maybe_unused void destroy_workqueue(struct workqueue_struct *queue) +{ + drain(queue); + assert(queues_live && queue == cpu_queue); + queues_live--; + destroyed++; + cpu_queue = NULL; + free(queue); +} + +static unsigned int cpu_index(mk_phys_cpu_t id) +{ + assert(id == ids[0] || id == ids[1]); + return id == ids[0] ? 0 : 1; +} + +static int mk_do_cpu_add(mk_phys_cpu_t id, u32 numa, u32 flags) +{ + unlocked_mutations += !mk_overlay_mutex; + if (!add_error) + online[cpu_index(id)] = true; + return add_error; +} + +static int mk_do_cpu_remove(mk_phys_cpu_t id) +{ + unlocked_mutations += !mk_overlay_mutex; + if (!remove_error) + online[cpu_index(id)] = false; + return remove_error; +} + +static int mk_do_mem_add(u64 pfn, u64 pages, u32 node, u32 type) +{ + unlocked_mutations += !mk_overlay_mutex; + memory_added = true; + return 0; +} + +static int mk_do_mem_remove(u64 pfn, u64 pages) +{ + unlocked_mutations += !mk_overlay_mutex; + memory_added = false; + return 0; +} + +static int mk_do_device_add(u16 domain, u8 bus, u8 devfn, const char *driver, u32 flags) +{ + unlocked_mutations += !mk_overlay_mutex; + device_added = true; + return 0; +} + +static int mk_do_device_remove(u16 domain, u8 bus, u8 devfn) +{ + unlocked_mutations += !mk_overlay_mutex; + device_added = false; + return 0; +} + +struct reply { + int to; + u64 id; + struct mk_resource_ack ack; +}; +static struct reply replies[4 * MK_IPI_RING_SIZE]; +static unsigned int nr_replies; + +static int mk_send_message_id(int to, u32 type, u32 subtype, u64 id, + const void *payload, u32 len) +{ + struct reply *reply; + + if (type == MK_MSG_SYSTEM) { + assert(subtype == MK_SYS_SHUTDOWN_ACK && id == 51 && to == 3); + expect("native CPU hotplug disabled before shutdown ACK", + native_hotplug_disabled, 1); + shutdown_sent = true; + shutdown_jobs_pending = (cpu_queue ? cpu_queue->nr : 0) + system_queue.nr; + return send_error; + } + reply = &replies[nr_replies++]; + assert(type == MK_MSG_RESOURCE && subtype == MK_RES_ACK); + assert(len == sizeof(reply->ack) && nr_replies <= 4 * MK_IPI_RING_SIZE); + reply->to = to; + reply->id = id; + memcpy(&reply->ack, payload, len); + return send_error; +} + +static void mk_resource_msg_handler(u32 type, u32 subtype, u64 id, + void *payload, u32 len, void *ctx) +{ + abort(); /* Dispatch is tested separately; this test calls receive handlers. */ +} + +static int mk_handle_cpu_add(struct mk_cpu_resource_payload *payload, u32 len, u64 id); +static int mk_handle_mem_add(struct mk_mem_resource_payload *payload, u32 len, u64 id); +static int mk_handle_device_add(struct mk_device_resource_payload *payload, u32 len, u64 id); +static void run_wait_jobs(void); +static int mk_register_msg_handler(u32 type, + void (*fn)(u32, u32, u64, void *, u32, void *), void *ctx) +{ + if (type == MK_MSG_SYSTEM && probe_system_registration) { + struct mk_cpu_resource_payload payload = { .cpu_id = 65537 }; + + system_probe_result = mk_handle_cpu_add(&payload, sizeof(payload), 61); + } + if (type == MK_MSG_RESOURCE) + registered = !register_error; + return type == MK_MSG_RESOURCE ? register_error : system_register_error; +} + +static int mk_unregister_msg_handler(u32 type, + void (*fn)(u32, u32, u64, void *, u32, void *)) +{ + struct mk_cpu_resource_payload payload = { .cpu_id = 65537 }; + + /* Model an already-entered callback attempting to queue during cleanup. */ + if (closing_probe) { + hard_irq = true; + close_result = mk_handle_cpu_add(&payload, sizeof(payload), 99); + hard_irq = false; + } + registered = false; + return 0; +} + +/* This fixture models host resource work, without a spawn PCI frontend. */ +#define mk_manifest_phys() 0 +#define mk_pcifront_add_function(...) (abort(), -EIO) +#include "work-production.h" + +static __maybe_unused void run_wait_jobs(void) +{ + struct mk_cpu_resource_payload cpu = { .cpu_id = 65537 }; + struct mk_mem_resource_payload mem = { .start_pfn = 65537 }; + struct mk_device_resource_payload dev = { .domain = 1 }; + + /* An already-entered receive callback submits after admission closes. */ + if (probe_at_wait) { + probe_at_wait = false; + hard_irq = true; + wait_probe_results[0] = mk_handle_cpu_add(&cpu, sizeof(cpu), 91); + wait_probe_results[1] = mk_handle_mem_add(&mem, sizeof(mem), 92); + wait_probe_results[2] = mk_handle_device_add(&dev, sizeof(dev), 93); + hard_irq = false; + } + assert((cpu_queue && cpu_queue->nr) || system_queue.nr); + drain(cpu_queue); + drain(&system_queue); +} + +static void __noreturn mk_enter_pool_state(void *arg) +{ + parked_cpus = true; + longjmp(parked, 1); +} + +static void smp_call_function(void (*fn)(void *), void *arg, int wait) +{ + assert(fn == mk_enter_pool_state && !wait); +} + +static int mk_messaging_init(void) +{ + return 0; +} + +static void mk_messaging_cleanup(void) +{ + messaging_cleanups++; +} + +static int mk_kernfs_init(void) +{ + return kernfs_error; +} + +static void mk_system_msg_handler(u32 type, u32 subtype, u64 id, + void *payload, u32 len, void *ctx) +{ +} + +static void cpu_hotplug_disable(void) +{ + expect("native hotplug drain outside resource lock", mk_overlay_mutex, 0); + expect("native hotplug drain in process context", hard_irq, 0); + expect("incoming admission closed before native drain", mk_hotplug_accepting, 0); + expect("incoming workers retired before native drain", mk_hotplug_idle(), 1); + native_hotplug_disabled = true; +} + +#include "work-shutdown.h" + +static int receive_cpu(bool add, struct mk_cpu_resource_payload *payload, u32 len, u64 id) +{ + int ret; + + hard_irq = true; + ret = add ? mk_handle_cpu_add(payload, len, id) : mk_handle_cpu_remove(payload, len, id); + hard_irq = false; + return ret; +} + +/* Test calls enter the real receive handlers with hard-IPI context modeled. */ +#define mk_handle_cpu_add(p, len, id) receive_cpu(true, p, len, id) +#define mk_handle_cpu_remove(p, len, id) receive_cpu(false, p, len, id) + +static struct workqueue_struct *active_queue(void) +{ + return cpu_queue ? cpu_queue : &system_queue; +} + +static void reset(void) +{ + drain(&system_queue); + assert(!cpu_queue && !allocations && !queues_live); + allocation_fails = queue_allocation_fails = registered = closing_probe = false; + register_error = close_result = add_error = remove_error = send_error = 0; + system_register_error = kernfs_error = 0; + messaging_cleanups = 0; + nr_replies = destroyed = 0; + online[0] = online[1] = false; + memory_added = device_added = shutdown_sent = parked_cpus = false; + native_hotplug_disabled = false; + probe_at_wait = probe_system_registration = false; + shutdown_jobs_pending = 0; + system_probe_result = 0; + memset(wait_probe_results, 0, sizeof(wait_probe_results)); +} + +static void shutdown_regressions(void) +{ + struct mk_cpu_resource_payload cpu = { .cpu_id = 65537, .sender_instance_id = 3 }; + struct mk_mem_resource_payload mem = { .start_pfn = 65537, .sender_instance_id = 3 }; + struct mk_device_resource_payload dev = { .domain = 1, .sender_instance_id = 3 }; + struct mk_shutdown_work shutdown, *sw = &shutdown; + + reset(); + expect("init for shutdown", mk_hotplug_init(), 0); + expect("CPU accepted before shutdown", mk_handle_cpu_add(&cpu, sizeof(cpu), 48), 0); + expect("memory accepted before shutdown", mk_handle_mem_add(&mem, sizeof(mem), 49), 0); + expect("device accepted before shutdown", mk_handle_device_add(&dev, sizeof(dev), 50), 0); + INIT_WORK(&sw->work, mk_shutdown_work_fn); + sw->sender_instance_id = 3; + sw->msg_id = 51; + schedule_work(&sw->work); + probe_at_wait = true; + /* Newest system job is shutdown; older resource jobs still exist. */ + if (!setjmp(parked)) + run_one(&system_queue); + expect("shutdown reaches parking boundary", parked_cpus, 1); + expect("shutdown ACK sent", shutdown_sent, 1); + expect("shutdown excludes local controllers until all CPUs park", mk_overlay_mutex, 1); + /* longjmp models disposal of this stopped kernel, not mutex unlocking. */ + mk_overlay_mutex = false; + expect("no accepted resource job remains at shutdown ACK", shutdown_jobs_pending, 0); + expect("CPU work completed before parking", online[0], 1); + expect("memory work completed before parking", memory_added, 1); + expect("device work completed before parking", device_added, 1); + expect("all resource ACKs precede shutdown", nr_replies, 3); + expect("accepted work freed before parking", allocations, 0); + expect("late CPU callback rejected during drain", wait_probe_results[0], -ESHUTDOWN); + expect("late memory callback rejected during drain", wait_probe_results[1], -ESHUTDOWN); + expect("late device callback rejected during drain", wait_probe_results[2], -ESHUTDOWN); + expect("post-shutdown CPU rejected", mk_handle_cpu_add(&cpu, sizeof(cpu), 52), -ESHUTDOWN); + expect("post-shutdown memory rejected", + mk_handle_mem_add(&mem, sizeof(mem), 53), -ESHUTDOWN); + expect("post-shutdown device rejected", + mk_handle_device_add(&dev, sizeof(dev), 54), -ESHUTDOWN); + expect("post-shutdown CPU remove rejected", + mk_handle_cpu_remove(&cpu, sizeof(cpu), 55), -ESHUTDOWN); + expect("post-shutdown memory remove rejected", + mk_handle_mem_remove(&mem, sizeof(mem), 56), -ESHUTDOWN); + expect("post-shutdown device remove rejected", + mk_handle_device_remove(&dev, sizeof(dev), 57), -ESHUTDOWN); + /* Keep baseline RED cleanup recoverable without claiming those jobs safe. */ + drain(cpu_queue); + drain(&system_queue); + mk_hotplug_cleanup(); + expect("shutdown cleanup releases allocations", allocations, 0); + reset(); + probe_system_registration = true; + expect("core init", multikernel_init(), 0); + expect("hotplug ready when system handler becomes visible", system_probe_result, 0); + mk_hotplug_cleanup(); + reset(); + /* Init unwinds both sides of the changed registration order. */ + system_register_error = -EIO; + expect("system registration error", multikernel_init(), -EIO); + expect("system failure destroys CPU queue", queues_live, 0); + expect("system failure closes resource admission", + mk_handle_mem_add(&mem, sizeof(mem), 62), -ESHUTDOWN); + expect("system failure unregisters resource handler", registered, 0); + expect("system failure cleans messaging", messaging_cleanups, 1); + reset(); + kernfs_error = -ENOMEM; + expect("kernfs init error", multikernel_init(), -ENOMEM); + expect("kernfs failure destroys CPU queue", queues_live, 0); + expect("kernfs failure cleans messaging", messaging_cleanups, 1); + expect("kernfs failure leaves no allocation", allocations, 0); + reset(); + + /* Cleanup must account for system-queue jobs as well as ordered CPU jobs. */ + expect("init for mixed cleanup", mk_hotplug_init(), 0); + expect("accept memory before cleanup", mk_handle_mem_add(&mem, sizeof(mem), 63), 0); + expect("accept device before cleanup", mk_handle_device_add(&dev, sizeof(dev), 64), 0); + send_error = -EIO; + mk_hotplug_cleanup(); + expect("cleanup completes memory despite lost ACK", memory_added, 1); + expect("cleanup completes device despite lost ACK", device_added, 1); + expect("cleanup releases mixed work despite lost ACK", allocations, 0); + expect("cleanup leaves no system jobs", system_queue.nr, 0); + expect("cleanup releases CPU queue", queues_live, 0); + reset(); +} + +static void controller_regressions(void) +{ + struct mk_cpu_resource_payload cpu = { .cpu_id = 65537, .sender_instance_id = 3 }; + struct mk_mem_resource_payload mem = { .start_pfn = 65537, .sender_instance_id = 3 }; + struct mk_device_resource_payload dev = { .domain = 1, .sender_instance_id = 3 }; + unsigned int i; + + reset(); + expect("init for controller contention", mk_hotplug_init(), 0); + expect("queue busy CPU add", mk_handle_cpu_add(&cpu, sizeof(cpu), 71), 0); + expect("queue busy CPU remove", mk_handle_cpu_remove(&cpu, sizeof(cpu), 72), 0); + expect("queue busy memory add", mk_handle_mem_add(&mem, sizeof(mem), 73), 0); + expect("queue busy memory remove", mk_handle_mem_remove(&mem, sizeof(mem), 74), 0); + expect("queue busy device add", mk_handle_device_add(&dev, sizeof(dev), 75), 0); + expect("queue busy device remove", mk_handle_device_remove(&dev, sizeof(dev), 76), 0); + mutex_lock(&mk_overlay_mutex); + drain(cpu_queue); + drain(&system_queue); + expect("busy workers leave controller lock held", mk_overlay_mutex, 1); + expect("busy workers do not mutate resources", online[0] || memory_added || device_added, 0); + expect("every busy worker attempts an ACK", nr_replies, 6); + for (i = 0; i < nr_replies; i++) + expect("busy ACK never claims success", replies[i].ack.result, -EBUSY); + mutex_unlock(&mk_overlay_mutex); + expect("retry admitted after controller unlock", mk_handle_cpu_add(&cpu, sizeof(cpu), 77), 0); + drain(cpu_queue); + expect("retry mutates after unlock", online[0], 1); + expect("retry ACK succeeds", replies[6].ack.result, 0); + mk_hotplug_cleanup(); + reset(); +} + +static int receive_resource(unsigned int op, u64 id) +{ + struct mk_cpu_resource_payload cpu = { .cpu_id = 65537, .sender_instance_id = 3 }; + struct mk_mem_resource_payload mem = { .start_pfn = 65537, .sender_instance_id = 3 }; + struct mk_device_resource_payload dev = { .domain = 1, .sender_instance_id = 3 }; + int ret; + + hard_irq = true; + switch (op) { + case 0: + ret = mk_handle_cpu_add(&cpu, sizeof(cpu), id); + break; + case 1: + ret = mk_handle_cpu_remove(&cpu, sizeof(cpu), id); + break; + case 2: + ret = mk_handle_mem_add(&mem, sizeof(mem), id); + break; + case 3: + ret = mk_handle_mem_remove(&mem, sizeof(mem), id); + break; + case 4: + ret = mk_handle_device_add(&dev, sizeof(dev), id); + break; + default: + ret = mk_handle_device_remove(&dev, sizeof(dev), id); + break; + } + hard_irq = false; + return ret; +} + +static void preallocation_regressions(void) +{ + struct mk_cpu_resource_payload cpu = { .cpu_id = 65537, .sender_instance_id = 3 }; + struct work_struct *slot; + unsigned int i; + + reset(); + expect("init for raw receive", mk_hotplug_init(), 0); + allocation_fails = true; + for (i = 0; i < 6; i++) + expect("all request types avoid allocation", receive_resource(i, 100 + i), 0); + drain(cpu_queue); + drain(&system_queue); + expect("all request types reply", nr_replies, 6); + for (i = 0; i < nr_replies; i++) + expect("copied sender survives callback return", replies[i].to, 3); + expect("first CPU request admitted", mk_handle_cpu_add(&cpu, sizeof(cpu), 200), 0); + cpu.cpu_id = ids[1]; + cpu.sender_instance_id = 9; + drain(cpu_queue); + expect("worker does not retain callback payload", replies[6].ack.resource_id == ids[0], 1); + expect("copied sender remains unchanged", replies[6].to, 3); + mk_hotplug_cleanup(); + for (i = 0; i < 6; i++) + expect("closed request avoids allocation", + receive_resource(i, 300 + i), -ESHUTDOWN); + expect("no IRQ allocator calls including rejected paths", irq_allocator_calls, 0); + reset(); + + expect("init for saturation", mk_hotplug_init(), 0); + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + expect("fill CPU slots", receive_resource(i % 2, 400 + i), 0); + expect("fill memory/device slots", receive_resource(2 + i % 4, 500 + i), 0); + } + for (i = 0; i < 6; i++) + expect("full class rejects without overwriting", + receive_resource(i, 600 + i), -ENOMEM); + expect("both classes counted through ACK", mk_hotplug_inflight, 2 * MK_IPI_RING_SIZE); + slot = cpu_queue->jobs[0]; + run_one(cpu_queue); + expect("CPU slot recycled after ACK", receive_resource(1, 700), 0); + expect("reused slot stays on same queue", cpu_queue->jobs[MK_IPI_RING_SIZE - 1] == slot, 1); + expect("callback remains initialized", slot->func == mk_hotplug_work_fn, 1); + send_error = -EIO; + mk_hotplug_cleanup(); + expect("cleanup drains saturated classes despite lost ACK", + nr_replies, 2 * MK_IPI_RING_SIZE + 1); + for (i = 0; i < MK_IPI_RING_SIZE; i++) + expect("CPU admission order and ID preserved", replies[i].id == 400 + i, 1); + expect("requeued CPU follows prior admitted requests", + replies[MK_IPI_RING_SIZE].id == 700, 1); + expect("quiescence retires all slots", mk_hotplug_inflight, 0); + reset(); +} + +int main(void) +{ + struct mk_cpu_resource_payload payload = { + .cpu_id = 65537, .sender_instance_id = 3, + }; + unsigned int i; + + reset(); + expect("init", mk_hotplug_init(), 0); + /* Leave the older add queued while its requester can time out. */ + expect("receive older add", mk_handle_cpu_add(&payload, sizeof(payload), 11), 0); + expect("receive newer remove", mk_handle_cpu_remove(&payload, sizeof(payload), 12), 0); + send_error = -EIO; + run_one(active_queue()); + expect("older add completes before remove", nr_replies && replies[0].id == 11, 1); + send_error = 0; + drain(active_queue()); + expect("newer remove leaves CPU offline", online[0], 0); + expect("remove ACK follows completed add", replies[1].id == 12, 1); + expect("workers release both allocations", allocations, 0); + mk_hotplug_cleanup(); + expect("cleanup destroys CPU queue", destroyed, 1); + expect("cleanup releases queue", queues_live, 0); + reset(); + + expect("init for remove/add", mk_hotplug_init(), 0); + online[0] = true; + expect("receive older remove", mk_handle_cpu_remove(&payload, sizeof(payload), 13), 0); + expect("receive newer add", mk_handle_cpu_add(&payload, sizeof(payload), 14), 0); + drain(active_queue()); + expect("newer add leaves CPU online", online[0], 1); + expect("older remove ACK first", replies[0].id == 13, 1); + remove_error = -EBUSY; + expect("queue failed remove", mk_handle_cpu_remove(&payload, sizeof(payload), 15), 0); + drain(active_queue()); + expect("failed remove leaves CPU online", online[0], 1); + expect("remove error echoed", (int)replies[2].ack.result, -EBUSY); + mk_hotplug_cleanup(); + reset(); + + expect("init for independent CPUs", mk_hotplug_init(), 0); + for (i = 0; i < 2; i++) { + payload.cpu_id = ids[i]; + expect("receive full-width CPU add", + mk_handle_cpu_add(&payload, sizeof(payload), 21 + i), 0); + } + drain(active_queue()); + expect("CPU additions both execute", online[0] && online[1], 1); + expect("first ACK preserves ID", replies[0].id == 21, 1); + expect("second ACK preserves sparse resource", replies[1].ack.resource_id == ids[1], 1); + expect("ACK targets requester", replies[1].to, 3); + payload.cpu_id = ids[0]; + allocation_fails = true; + expect("receive uses preallocated storage", + mk_handle_cpu_add(&payload, sizeof(payload), 23), 0); + drain(active_queue()); + /* Keep subsequent error checks independent of this new reply. */ + nr_replies = 2; + allocation_fails = false; + expect("short add", mk_handle_cpu_add(&payload, 0, 24), -EINVAL); + expect("short remove", mk_handle_cpu_remove(&payload, 0, 25), -EINVAL); + add_error = -EPERM; + expect("queue failed add", mk_handle_cpu_add(&payload, sizeof(payload), 26), 0); + drain(active_queue()); + expect("worker error echoed", (int)replies[2].ack.result, -EPERM); + expect("worker error preserves request ID", replies[2].id == 26, 1); + mk_hotplug_cleanup(); + reset(); + + expect("init for teardown", mk_hotplug_init(), 0); + expect("queue before cleanup", mk_handle_cpu_add(&payload, sizeof(payload), 31), 0); + closing_probe = true; + mk_hotplug_cleanup(); + expect("cleanup closes admission before unregister", close_result, -ESHUTDOWN); + expect("cleanup drains already admitted CPU work", online[0], 1); + expect("cleanup releases admitted/rejected work", allocations, 0); + expect("cleanup leaves no CPU queue", queues_live, 0); + expect("late callback refused", + mk_handle_cpu_add(&payload, sizeof(payload), 32), -ESHUTDOWN); + expect("late callback releases allocation", allocations, 0); + reset(); + + queue_allocation_fails = true; + expect("queue allocation failure", mk_hotplug_init(), -ENOMEM); + expect("failed queue never registers handler", registered, 0); + mk_hotplug_cleanup(); + reset(); + register_error = -EIO; + expect("handler registration failure", mk_hotplug_init(), -EIO); + expect("registration failure frees queue", queues_live, 0); + expect("registration failure closes CPU admission", + mk_handle_cpu_remove(&payload, sizeof(payload), 41), -ESHUTDOWN); + mk_hotplug_cleanup(); + reset(); + shutdown_regressions(); + controller_regressions(); + preallocation_regressions(); + expect("incoming mutation excludes direct controller/pool changes", unlocked_mutations, 0); + expect("hard-IPI admission never takes an RT sleeping lock", rt_atomic_lock_errors, 0); + expect("hard-IPI receive never enters allocator", irq_allocator_calls, 0); + printf("CPU work ordering: %u checks, %u failures\n", checks, failures); + return failures ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/tools/testing/selftests/multikernel/cpu_work_test.sh b/tools/testing/selftests/multikernel/cpu_work_test.sh new file mode 100644 index 00000000000000..8e7ad9ac9172de --- /dev/null +++ b/tools/testing/selftests/multikernel/cpu_work_test.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-cpu-work.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM + +awk '/^#define MK_IPI_RING_SIZE / { print } + /^#define MK_MSG_IO / { copy = 1 } + /^\/\* Opaque type/ { copy = 0 } + copy { print }' "$src/include/linux/multikernel.h" > "$build/work-types.h" +awk '/^struct mk_hotplug_op \{/ { copy = 1 } + /^static DEFINE_MUTEX\(mk_hotplug_mutex\);/ { print } + /^static LIST_HEAD\(mk_hotplug_ops\);/ { print } + /^static DEFINE_(RAW_)?SPINLOCK\(mk_cpu_hotplug_lock\);/ { print } + /^static struct workqueue_struct \*mk_cpu_hotplug_wq;/ { print } + /^static (bool mk_hotplug_accepting|unsigned int mk_hotplug_inflight)/ { print } + /^static DECLARE_WAIT_QUEUE_HEAD\(mk_hotplug_wait\);/ { print } + /^static (int|struct workqueue_struct \*)mk_cpu_hotplug_/ { copy = 1 } + /^static int mk_queue_hotplug_request\(/ { copy = 1 } + /^static (void|bool) mk_hotplug_(work_done|idle)\(/ { copy = 1 } + /^static struct mk_hotplug_work mk_(cpu|other)_requests\[/ { print } + /^static void mk_hotplug_work_fn\(/ { copy = 1 } + /^void mk_hotplug_quiesce\(/ { copy = 1 } + /^struct mk_hotplug_work \{/ { copy = 1 } + /^static void mk_(cpu|mem|device)_(add|remove)_work_fn\(/ { copy = 1 } + /^(static )?int mk_handle_(cpu|mem|device)_(add|remove)\(/ { copy = 1 } + /^int __init mk_hotplug_init\(/ { copy = 1 } + /^void mk_hotplug_cleanup\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/hotplug.c" \ + > "$build/work-production.h" +awk '/^struct mk_shutdown_work \{/ { copy = 1 } + /^static void __noreturn mk_notify_down_and_park\(/ { copy = 1 } + /^static void mk_shutdown_work_fn\(/ { copy = 1 } + /^static int __init multikernel_init\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/core.c" \ + > "$build/work-shutdown.h" + +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -Wno-unused-parameter \ + -I"$build" -I"$src/tools/include" "$test_dir/cpu_work_test.c" \ + -o "$build/cpu_work_test" +"$build/cpu_work_test" diff --git a/tools/testing/selftests/multikernel/halt_settlement_test.c b/tools/testing/selftests/multikernel/halt_settlement_test.c new file mode 100644 index 00000000000000..24928c1fc87269 --- /dev/null +++ b/tools/testing/selftests/multikernel/halt_settlement_test.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real host settlement paths; transport and physical stop boundaries injected. */ +#include +#include +#include +#include +#include + +typedef unsigned long long u64; +typedef unsigned int u32; +typedef u64 mk_phys_cpu_t; +#define MK_STATE_ACTIVE 3 +#define MK_STATE_LOADED 2 +#define MK_STATE_FAILED 4 +#define MK_SYS_SHUTDOWN 0x1001 +#define MK_MSG_SYSTEM 1 +#define MK_SHUTDOWN_GRACEFUL 1 +#define CONFIG_ARCH_HAS_MK_FORCE_STOP 1 +#define IS_ENABLED(config) force_supported +#define GFP_KERNEL 0 +#define pr_info(...) do { if (0) printf(__VA_ARGS__); } while (0) +#define pr_warn(...) do { } while (0) +#define pr_err(...) do { } while (0) +#define IS_ERR(p) false +#define PTR_ERR(p) (-ENOMEM) +#define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) +struct mk_cpu_set { unsigned int nr; mk_phys_cpu_t ids[2]; }; +#define mk_cpu_set_for_each(i, cpu, set) \ + for ((i) = 0; (i) < (set)->nr && (((cpu) = (set)->ids[i]), true); (i)++) +#define mk_cpu_set_empty(p) (!(p)->nr) +struct work_struct { int unused; }; +struct mk_instance { + int id, state; + const char *name; + unsigned int refs; + struct work_struct halted_work; + int halted_pending; +}; +#define atomic_set(p, v) (*(p) = (v)) +static int atomic_cmpxchg(int *p, int old, int new) +{ + int value = *p; + + if (value == old) + *p = new; + return value; +} +struct mk_shutdown_payload { u32 flags; int sender_instance_id; }; +struct mk_pending_msg { int unused; }; +static struct mk_instance instance = { .id = 7, .state = MK_STATE_ACTIVE, .name = "spawn" }; +static struct mk_instance replacement = { .id = 7, .state = MK_STATE_ACTIVE, .name = "new" }; +static struct mk_instance *lookup_instance = &instance; +static struct mk_instance self = { .id = 0, .state = MK_STATE_ACTIVE, .name = "host" }; +static struct mk_instance *mk_self = &self; +static struct mk_instance *host_instance; +static struct mk_pending_msg pending; +static struct mk_cpu_set targets; +static bool force_supported = true, mk_overlay_mutex; +static int confirm_error, send_error, wait_error, arm_error, fence_error; +static unsigned int refs, stopped, confirms, unlocked_confirms, failures, checks; +static unsigned int host_fenced; +static struct work_struct *queued; +static bool queue_rejected, duplicate_during_confirm; +static void mk_queue_halted(int id); +static bool schedule_work(struct work_struct *work) +{ + if (queue_rejected) + return false; + assert(!queued); + queued = work; + return true; +} +#define mutex_lock(m) do { assert(!*(m)); *(m) = true; } while (0) +#define mutex_unlock(m) do { assert(*(m)); *(m) = false; } while (0) +#define lockdep_assert_held(m) assert(*(m)) +static __attribute__((unused)) bool *scoped_lock(bool *m) { mutex_lock(m); return m; } +static __attribute__((unused)) void scoped_unlock(bool **m) { mutex_unlock(*m); } +#define guard(kind) \ + bool *scope_##kind __attribute__((cleanup(scoped_unlock), unused)) = scoped_lock + +static struct mk_instance *mk_instance_find(int id) +{ + if (!lookup_instance || id != lookup_instance->id) + return NULL; + refs++; + lookup_instance->refs++; + return lookup_instance; +} +static void mk_instance_put(struct mk_instance *p) +{ + assert(refs && p->refs); + refs--; + p->refs--; +} +static void mk_instance_set_state(struct mk_instance *p, int state) { p->state = state; } +static int mk_instance_confirm_parked(struct mk_instance *p) +{ + confirms++; + unlocked_confirms += !mk_overlay_mutex; + if (duplicate_during_confirm) { + assert(p->halted_pending); + mk_queue_halted(p->id); + assert(!queued && p->refs == 1); + } + return confirm_error; +} +static struct mk_pending_msg *mk_msg_pending_add(u32 type, u32 op, int id) { return &pending; } +static u64 mk_msg_pending_id(struct mk_pending_msg *p) { return 31; } +static int mk_send_message_id(int id, u32 type, u32 op, u64 msg_id, + void *buf, size_t size) +{ + return send_error; +} +static void mk_msg_pending_cancel(struct mk_pending_msg *p) { } +static int mk_msg_pending_wait(struct mk_pending_msg *p, int ms) { return wait_error; } +static struct mk_cpu_set *mk_cpu_set_alloc(void) { return &targets; } +static void mk_cpu_set_free(struct mk_cpu_set *p) { } +static int mk_force_halt_targets(struct mk_instance *p, struct mk_cpu_set *set) +{ + *set = (struct mk_cpu_set) { 2, { 65537, 0x100010001ULL } }; + return 0; +} +static int mk_arm_force_halt(struct mk_instance *p) { return arm_error; } +static void mk_force_stop_cpu(mk_phys_cpu_t cpu) { stopped++; } +static int mk_confirm_fenced(struct mk_instance *p, struct mk_cpu_set *set) { return fence_error; } +static void mk_pcifront_host_fenced(void) { host_fenced++; } +#define kfree(p) ((void)(p)) +#include "halt-production.h" + +static void expect(const char *name, int got, int want) +{ + checks++; + if (got != want) { + fprintf(stderr, "FAIL %s: %d != %d\n", name, got, want); + failures++; + } +} +static void reset(void) +{ + assert(!refs && !mk_overlay_mutex && !queued); + instance.state = MK_STATE_ACTIVE; + instance.halted_pending = 0; + lookup_instance = &instance; + queue_rejected = duplicate_during_confirm = false; + confirm_error = send_error = wait_error = arm_error = fence_error = 0; + stopped = confirms = host_fenced = 0; + force_supported = true; + host_instance = NULL; +} +static void run_halted(void) +{ + struct work_struct *work = queued; + + assert(work); + queued = NULL; + mk_halted_work_fn(work); +} + +int main(void) +{ + reset(); + confirm_error = -EBUSY; + expect("shutdown ACK is not stop proof", multikernel_halt_by_id(7), -EBUSY); + expect("failed requested confirmation retains active state", instance.state, MK_STATE_ACTIVE); + reset(); + expect("confirmed requested halt", multikernel_halt_by_id(7), 0); + expect("confirmed halt becomes loaded", instance.state, MK_STATE_LOADED); + reset(); + wait_error = -ETIMEDOUT; + expect("lost shutdown ACK", multikernel_halt_by_id(7), -ETIMEDOUT); + expect("lost ACK retains active state", instance.state, MK_STATE_ACTIVE); + reset(); + confirm_error = -EBUSY; + mk_queue_halted(7); + run_halted(); + expect("voluntary HALTED must confirm", confirms, 1); + expect("failed voluntary confirmation retains active state", instance.state, MK_STATE_ACTIVE); + reset(); + mk_queue_halted(7); + run_halted(); + expect("confirmed voluntary halt", instance.state, MK_STATE_LOADED); + expect("completed notice releases restart gate", instance.halted_pending, 0); + reset(); + mk_queue_halted(7); + mk_queue_halted(7); + expect("duplicate notice keeps one reference", refs, 1); + expect("notice excludes restart until retirement", instance.halted_pending, 1); + duplicate_during_confirm = true; + run_halted(); + expect("duplicates do not add confirmations", confirms, 1); + expect("worker releases its reference", refs, 0); + reset(); + mk_queue_halted(7); + lookup_instance = &replacement; + run_halted(); + expect("queued notice retains original identity", instance.state, MK_STATE_LOADED); + expect("ID reuse cannot settle replacement", replacement.state, MK_STATE_ACTIVE); + expect("replacement gets no stale reference", replacement.refs, 0); + reset(); + queue_rejected = true; + mk_queue_halted(7); + expect("queue rejection drops admission reference", refs, 0); + expect("queue rejection releases restart gate", instance.halted_pending, 0); + expect("queue rejection cannot publish reusable state", instance.state, MK_STATE_ACTIVE); + reset(); + lookup_instance = NULL; + mk_queue_halted(7); + expect("removed ID queues nothing", queued == NULL && !refs, 1); + reset(); + instance.state = MK_STATE_LOADED; + mk_queue_halted(7); + run_halted(); + expect("already settled notice does not reconfirm", confirms, 0); + reset(); + confirm_error = -EBUSY; + expect("force halt must confirm", multikernel_force_halt_by_id(7), -EBUSY); + expect("force halt sends both physical targets", stopped, 2); + expect("failed force confirmation retains active state", instance.state, MK_STATE_ACTIVE); + reset(); + arm_error = -ENODEV; + expect("missing force marker fails closed", multikernel_force_halt_by_id(7), -ENODEV); + expect("failed marker does not send unqualified NMIs", stopped, 0); + reset(); + host_instance = &instance; + fence_error = -ETIMEDOUT; + expect("failed host fencing", multikernel_force_halt_by_id(7), -ETIMEDOUT); + expect("failed host fencing retains PCI frontend", host_fenced, 0); + fence_error = 0; + expect("confirmed host fencing", multikernel_force_halt_by_id(7), 0); + expect("confirmed host fencing notifies PCI frontend", host_fenced, 1); + reset(); + force_supported = false; + expect("unsupported remote force stop", multikernel_force_halt_by_id(7), -EOPNOTSUPP); + expect("unsupported force stop sends nothing", stopped, 0); + expect("every settlement is resource serialized", unlocked_confirms, 0); + assert(!refs && !mk_overlay_mutex); + printf("Halt settlement: %u checks, %u failures\n", checks, failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/multikernel/halt_settlement_test.sh b/tools/testing/selftests/multikernel/halt_settlement_test.sh new file mode 100644 index 00000000000000..72f9a489b0ca64 --- /dev/null +++ b/tools/testing/selftests/multikernel/halt_settlement_test.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-halt-settlement.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^static (void|int) mk_instance_settle_halted\(/ { copy = 1 } + /^static void mk_(halted_work_fn|queue_halted)\(/ && !/;$/ { copy = 1 } + /^int multikernel_(force_)?halt_by_id\(/ { copy = 1 } + copy { print } + copy && /^};?/ { copy = 0 }' "$src/kernel/multikernel/core.c" \ + > "$build/halt-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -Wno-unused-parameter \ + -I"$build" "$test_dir/halt_settlement_test.c" -o "$build/halt_settlement_test" +"$build/halt_settlement_test" diff --git a/tools/testing/selftests/multikernel/image_lock_test.c b/tools/testing/selftests/multikernel/image_lock_test.c new file mode 100644 index 00000000000000..19c9ab5f9d450a --- /dev/null +++ b/tools/testing/selftests/multikernel/image_lock_test.c @@ -0,0 +1,596 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Production image operations and overlay rollback, real host mutexes. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define atomic_read(p) atomic_load(p) +#define u64 __u64 +#define CONFIG_MULTIKERNEL 1 +#define CONFIG_CRASH_DUMP 1 +#define __user +static char *tracked_cmdline; +static void test_kfree(void *ptr) +{ + if (ptr == tracked_cmdline) + tracked_cmdline = NULL; + free(ptr); +} +#define kfree test_kfree +#define phys_to_virt(p) ((void *)(p)) +#define WARN_ON_ONCE(x) (x) +#define pr_err(...) do { } while (0) +#define pr_warn(...) do { } while (0) +#define pr_info(...) do { } while (0) +#define pr_debug(...) do { } while (0) +#define kexec_dprintk(...) do { } while (0) +#define SYSCALL_DEFINE5(name, t1, a1, t2, a2, t3, a3, t4, a4, t5, a5) \ + static long name(t1 a1, t2 a2, t3 a3, t4 a4, t5 a5) + +#include "image-types.h" + +static atomic_uint checks, failures, resource_updates; +static _Thread_local unsigned int held; +static _Thread_local bool resource_thread; +static bool is_mk, permitted = true, allocation_fails; +static bool mapping_fails; +static bool pci_lent; +static unsigned int control_page_calls; +static unsigned int image_work_calls; +static const char *probe_at, *fail_at; +static bool probe_started, attempted, finished; +static pthread_t competitor; +static pthread_mutex_t gate = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t changed = PTHREAD_COND_INITIALIZER; +static atomic_bool kexec_held; + +static void expect(const char *name, int got, int want) +{ + atomic_fetch_add(&checks, 1); + if (got != want) { + fprintf(stderr, "FAIL %s: got %d, want %d\n", name, got, want); + atomic_fetch_add(&failures, 1); + } +} + +enum { KEXEC_LOCK = 1, OVERLAY_LOCK = 2, REGISTRY_LOCK = 4 }; +struct mutex { pthread_mutex_t native; unsigned int bit; }; +static struct mutex mk_overlay_mutex = { PTHREAD_MUTEX_INITIALIZER, OVERLAY_LOCK }; +static struct mutex mk_instance_mutex = { PTHREAD_MUTEX_INITIALIZER, REGISTRY_LOCK }; +#define rcu_read_lock() do { assert(!(held & 8)); held |= 8; } while (0) +#define rcu_read_unlock() do { assert(held & 8); held &= ~8U; } while (0) +#define kref_get_unless_zero(p) (references++, true) + +static void mutex_lock(struct mutex *m) +{ + expect("lock order", held & ~(m->bit - 1), 0); + if (resource_thread && m == &mk_overlay_mutex) { + assert(!pthread_mutex_lock(&gate)); + attempted = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + } + assert(!pthread_mutex_lock(&m->native)); + held |= m->bit; +} + +static void mutex_unlock(struct mutex *m) +{ + expect("unlock held mutex", !!(held & m->bit), 1); + expect("reverse unlock order", held & ~(2 * m->bit - 1), 0); + held &= ~m->bit; + assert(!pthread_mutex_unlock(&m->native)); +} + +static struct mutex *scoped_lock(struct mutex *m) +{ + mutex_lock(m); + return m; +} + +static void scoped_unlock(struct mutex **m) +{ + if (*m) { + mutex_unlock(*m); + *m = NULL; + } +} + +#define scoped_guard(kind, m) \ + for (struct mutex *scope_##kind __attribute__((cleanup(scoped_unlock))) = \ + scoped_lock(m); scope_##kind; scoped_unlock(&scope_##kind)) +#define lockdep_assert_held(m) expect("lockdep assertion", !!(held & (m)->bit), 1) + +static bool kexec_trylock(void) +{ + bool unlocked = false; + + if (!atomic_compare_exchange_strong(&kexec_held, &unlocked, true)) + return false; + expect("kexec acquired first", held, 0); + held |= KEXEC_LOCK; + return true; +} + +static void kexec_unlock(void) +{ + expect("resource lock released before kexec", held, KEXEC_LOCK); + held &= ~KEXEC_LOCK; + atomic_store(&kexec_held, false); +} + +struct kimage; +struct mk_instance { + int id; + const char *name; + enum mk_instance_state state; + bool resource_uncertain; + atomic_int halted_pending; + struct mk_cpu_set *cpus; + struct kimage *kimage; + void *ipi_data; + unsigned long ipi_phys; +}; +struct kexec_segment { + void *kbuf, *buf; + size_t bufsz, memsz; + unsigned long mem; +}; +struct kimage { + int type, file_mode, mk_id; + struct mk_instance *mk_instance; + unsigned long nr_segments, start, head, control_page; + struct kexec_segment segment[1]; + void *control_code_page, *swap_page; + char *cmdline_buf; + struct list_head control_pages; +}; +#include "image-cmdline.h" +static struct mk_instance target; +static struct mk_instance self, *mk_self = &self; +static struct kimage *kexec_image, *kexec_crash_image; +static struct { unsigned long start; } crashk_res; +static unsigned int references; +static int mk_instance_idr, kexec_file_dbg_print; +static char image_bytes[16], ring[16], control_page; +static mk_phys_cpu_t cpu_id = 65537; +static struct mk_cpu_set cpus = { .nr = 1, .cap = 1, .ids = &cpu_id }; + +static void *resource_operation(void *unused); + +static void wait_for(bool *condition) +{ + struct timespec deadline; + + assert(!clock_gettime(CLOCK_REALTIME, &deadline)); + deadline.tv_sec += 10; + while (!*condition) + assert(!pthread_cond_timedwait(&changed, &gate, &deadline)); +} + +static int stage(const char *name) +{ + image_work_calls++; + expect("image work holds kexec lock", !!(held & KEXEC_LOCK), 1); + expect("no image work under registry lock", !!(held & REGISTRY_LOCK), 0); + expect("MK work holds resource lock", !!(held & OVERLAY_LOCK), is_mk); + if (probe_at && !probe_started && !strcmp(probe_at, name)) { + probe_started = true; + assert(!pthread_create(&competitor, NULL, resource_operation, NULL)); + assert(!pthread_mutex_lock(&gate)); + wait_for(&attempted); + /* Without exclusion, let the contender finish to prove overlap. */ + if (!(held & OVERLAY_LOCK)) + wait_for(&finished); + expect("overlay cannot mutate during MK image work", + atomic_load(&resource_updates), 0); + assert(!pthread_mutex_unlock(&gate)); + } + return fail_at && !strcmp(fail_at, name) ? -EIO : 0; +} + +static struct mk_instance *idr_find(int *idr, int id) +{ + return id == target.id ? &target : NULL; +} +static void mk_instance_put(struct mk_instance *instance) +{ + if (!instance) + return; + assert(references); + references--; +} +#define mk_instance_track_dump(...) do { } while (0) + +static void mk_kimage_free(struct kimage *image, void *addr, size_t size) +{ + expect("PCI retired before Image memory release", pci_lent, 0); + stage("free-segment"); +} +static void kimage_file_post_load_cleanup(struct kimage *image) +{ + stage("free-buffer"); +} +static int arch_cpu_from_physical_id(mk_phys_cpu_t cpu) +{ + return !mapping_fails && cpu == cpu_id ? 1 : -EINVAL; +} +static int mk_instance_confirm_parked(struct mk_instance *instance) { return stage("confirm"); } +static int kimage_load_segment(struct kimage *image, unsigned int i) { return stage("copy"); } +static int mk_manifest_finalize(struct kimage *image) { return stage("manifest"); } +static unsigned int host_ring_drops; +static __attribute__((unused)) void mk_ipi_ring_drop_pending(void) +{ + host_ring_drops++; + stage("reset-ring"); +} +static void mk_instance_ipi_deactivate(struct mk_instance *instance) +{ + stage("deactivate"); + instance->ipi_data = NULL; +} +static void mk_instance_ipi_publish(struct mk_instance *instance) +{ + stage("publish-ring"); + instance->ipi_data = ring; +} +static void mk_instance_ipi_reset(struct mk_instance *instance) +{ + stage("reset-instance-ring"); +} +static int mk_arch_spawn_instance(struct kimage *image, struct mk_instance *instance, int cpu) +{ + return stage("start"); +} +/* PCI operations are boundary probes, not device emulation. */ +static int mk_pciback_lend(struct mk_instance *instance) +{ + int ret = stage("lend"); + + pci_lent = !ret; + return ret; +} +static void mk_pciback_unlend(struct mk_instance *instance) +{ + stage("unlend"); + pci_lent = false; +} +#include "image-instance.h" +#include "image-core.h" + +static int kimage_free(struct kimage *image) +{ + int ret; + + if (!image) + return 0; + if (image->type == KEXEC_TYPE_MULTIKERNEL) { + ret = kimage_can_free_multikernel(image); + if (ret) + return ret; + } + stage("free-image"); + if (image->type == KEXEC_TYPE_MULTIKERNEL) + kimage_free_multikernel(image); + else + kimage_file_post_load_cleanup(image); + kimage_file_free_cmdline(image); + free(image); + return 0; +} +static struct kimage *do_kimage_alloc_init(void) +{ + struct kimage *image; + + stage("allocate"); + if (allocation_fails) + return NULL; + image = calloc(1, sizeof(*image)); + assert(image); + INIT_LIST_HEAD(&image->control_pages); + return image; +} +static int mk_instance_alloc_boot_pages(struct mk_instance *instance) +{ + return stage("boot-pages"); +} +static int kimage_file_prepare_segments(struct kimage *image, int fd, int initrd, + const char *cmdline, unsigned long len, unsigned int flags) +{ + if (len) { + assert(!tracked_cmdline); + tracked_cmdline = malloc(len); + assert(tracked_cmdline); + memcpy(tracked_cmdline, cmdline, len); + image->cmdline_buf = tracked_cmdline; + } + image->nr_segments = 1; + image->segment[0].kbuf = image_bytes; + image->segment[0].mem = (unsigned long)image_bytes; + image->segment[0].memsz = sizeof(image_bytes); + return stage("read-image"); +} +static int sanity_check_segment_list(struct kimage *image) { return stage("validate"); } +#define KEXEC_CONTROL_PAGE_SIZE 4096 +#define get_order(size) 0 +static void *kimage_alloc_control_pages(struct kimage *image, unsigned int order) +{ + return stage(control_page_calls++ ? "swap-page" : "control-page") ? + NULL : &control_page; +} +static void kimage_free_page_list(struct list_head *pages) { stage("free-pages"); } +static bool kexec_load_permitted(int type) { return permitted; } +static int machine_kexec_prepare(struct kimage *image) { return stage("prepare"); } +static int kimage_crash_copy_vmcoreinfo(struct kimage *image) { return stage("vmcore"); } +static int kexec_calculate_store_digests(struct kimage *image) { return stage("digest"); } +static void kimage_terminate(struct kimage *image) { stage("terminate"); } +static int kexec_post_load(struct kimage *image, unsigned long flags) +{ + return stage("post-load"); +} +static void kimage_remove_from_list(struct kimage *image) { stage("remove-list"); } +static void kimage_add_to_list(struct kimage *image) { stage("publish"); } +static void kimage_update_compat_pointers(struct kimage *image, int type) +{ + if (type == KEXEC_TYPE_CRASH) + kexec_crash_image = image; + else + kexec_image = image; +} +#define xchg(p, value) ({ __typeof__(*(p)) old = *(p); *(p) = (value); old; }) +static void arch_kexec_unprotect_crashkres(void) { stage("unprotect"); } +static void arch_kexec_protect_crashkres(void) { stage("protect"); } +#include "image-file.h" + +struct kernfs_node { void *priv; struct kernfs_node *parent; }; +static struct kernfs_node overlay_root, *mk_overlay_root_kn = &overlay_root; +static struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn) +{ + return kn->parent; +} +static void kernfs_put(struct kernfs_node *kn) { } +static void kernfs_remove_self(struct kernfs_node *kn) { } +static int mk_overlay_parse_and_rollback(struct mk_overlay_tx *tx, void *data) +{ + lockdep_assert_held(&mk_overlay_mutex); + mutex_lock(&mk_instance_mutex); + atomic_fetch_add(&resource_updates, 1); + mutex_unlock(&mk_instance_mutex); + return 0; +} +#include "image-overlay.h" + +static void *resource_operation(void *unused) +{ + struct mk_overlay_tx *tx = calloc(1, sizeof(*tx)); + struct list_head transactions; + struct kernfs_node kn = { .priv = tx, .parent = &overlay_root }; + + assert(tx); + resource_thread = true; + tx->status = MK_OVERLAY_TX_APPLIED; + INIT_LIST_HEAD(&transactions); + list_add(&tx->list, &transactions); + expect("competing overlay succeeds after release", mk_overlay_rmdir(&kn), 0); + expect("resource thread releases all locks", held, 0); + assert(!pthread_mutex_lock(&gate)); + finished = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + return NULL; +} + +static const unsigned long mk_flags = KEXEC_MULTIKERNEL | KEXEC_MK_ID(2); + +static void reset(bool multikernel) +{ + assert(!held && !atomic_load(&kexec_held)); + free(target.kimage); + free(kexec_image); + free(kexec_crash_image); + kexec_image = kexec_crash_image = NULL; + target = (struct mk_instance) { + .id = 2, .name = "spawn", .state = MK_STATE_READY, .cpus = &cpus, + .ipi_phys = (unsigned long)ring, + }; + cpus.nr = 1; + references = 0; + is_mk = multikernel; + permitted = true; + mapping_fails = false; + pci_lent = false; + control_page_calls = 0; + image_work_calls = 0; + allocation_fails = probe_started = attempted = finished = false; + probe_at = fail_at = NULL; + atomic_store(&resource_updates, 0); +} + +static void finish_probe(void) +{ + expect("requested overlap probe reached", probe_started, 1); + assert(!pthread_join(competitor, NULL)); + expect("overlay progresses after image operation", atomic_load(&resource_updates), 1); + expect("operation releases all locks", held, 0); +} + +static void check_cmdline_cleanup(void) +{ + static const char cmdline[] = "fixture"; + const unsigned long flags[] = { 0, KEXEC_FILE_ON_CRASH, mk_flags }; + const char * const stages[] = { "read-image", "validate", "control-page", "swap-page" }; + unsigned int kind, step; + + for (kind = 0; kind < sizeof(flags) / sizeof(*flags); kind++) { + for (step = 0; step < sizeof(stages) / sizeof(*stages); step++) { + /* Crash images do not allocate a swap page. */ + if (flags[kind] == KEXEC_FILE_ON_CRASH && step == 3) + continue; + reset(kind == 2); + fail_at = stages[step]; + expect("cmdline early load failure", kexec_file_load(0, 0, + sizeof(cmdline), cmdline, flags[kind]), step < 2 ? -EIO : -ENOMEM); + expect("early unwind frees cmdline", tracked_cmdline != NULL, 0); + expect("cmdline failure releases instance reference", references, 0); + expect("cmdline failure releases locks", held, 0); + /* Keep the deliberately failing regression run leak-free. */ + free(tracked_cmdline); + tracked_cmdline = NULL; + } + reset(kind == 2); + expect("cmdline successful load", kexec_file_load(0, 0, + sizeof(cmdline), cmdline, flags[kind]), 0); + expect("successful load retains cmdline", tracked_cmdline != NULL, 1); + expect("cmdline unload", kexec_file_load(0, 0, 0, NULL, + flags[kind] | KEXEC_FILE_UNLOAD), 0); + expect("unload frees cmdline", tracked_cmdline != NULL, 0); + } +} + +int main(void) +{ + static const char * const load_stages[] = { "allocate", "copy", "publish" }; + static const char * const fail_stages[] = { + "boot-pages", "read-image", "validate", "prepare", "vmcore", + "digest", "copy", "post-load", "control-page", "swap-page" + }; + static const char * const exec_fail_stages[] = { "confirm", "copy", "manifest", "start" }; + unsigned int i; + + check_cmdline_cleanup(); + for (i = 0; i < sizeof(load_stages) / sizeof(*load_stages); i++) { + reset(true); + probe_at = load_stages[i]; + expect("MK load", kexec_file_load(0, 0, 0, NULL, mk_flags), 0); + finish_probe(); + expect("load publishes complete image", target.kimage != NULL, 1); + expect("published image pins instance", references, 1); + } + for (i = 0; i < sizeof(fail_stages) / sizeof(*fail_stages); i++) { + int error = i >= 8 ? -ENOMEM : -EIO; + + reset(true); + fail_at = fail_stages[i]; + probe_at = "free-buffer"; + expect("failed load", kexec_file_load(0, 0, 0, NULL, mk_flags), error); + finish_probe(); + expect("failed image not published", target.kimage != NULL, 0); + expect("failed load releases reference", references, 0); + } + reset(true); + expect("setup unload", kexec_file_load(0, 0, 0, NULL, mk_flags), 0); + probe_at = "free-segment"; + expect("MK unload", kexec_file_load(0, 0, 0, NULL, mk_flags | KEXEC_FILE_UNLOAD), 0); + finish_probe(); + expect("unload clears image", target.kimage != NULL, 0); + expect("unload releases reference", references, 0); + + reset(true); + expect("setup active unload", kexec_file_load(0, 0, 0, NULL, mk_flags), 0); + target.state = MK_STATE_ACTIVE; + expect("active unload rejected", kexec_file_load(0, 0, 0, NULL, + mk_flags | KEXEC_FILE_UNLOAD), -EBUSY); + expect("active unload retains image", target.kimage != NULL, 1); + expect("active unload retains reference", references, 1); + target.state = MK_STATE_FAILED; + fail_at = "confirm"; + expect("uncertain failed unload rejected", kexec_file_load(0, 0, 0, NULL, + mk_flags | KEXEC_FILE_UNLOAD), -EIO); + expect("failed confirmation retains image", target.kimage != NULL, 1); + expect("failed confirmation retains reference", references, 1); + fail_at = NULL; + expect("confirmed failed start can unload", kexec_file_load(0, 0, 0, NULL, + mk_flags | KEXEC_FILE_UNLOAD), 0); + expect("confirmed unload releases reference", references, 0); + + reset(true); + expect("setup exec", kexec_file_load(0, 0, 0, NULL, mk_flags), 0); + probe_at = "start"; + expect("MK exec", multikernel_kexec_by_id(2), 0); + finish_probe(); + expect("spawn state published", target.state, MK_STATE_ACTIVE); + expect("exec never rewinds or discards shared host receive traffic", host_ring_drops, 0); + /* ACTIVE must be settled by halt, not by an execution request. */ + image_work_calls = 0; + expect("active exec rejected", multikernel_kexec_by_id(2), -EBUSY); + expect("active exec does no confirmation or image work", image_work_calls, 0); + expect("active exec retains state", target.state, MK_STATE_ACTIVE); + expect("active exec retains image", !!target.kimage, 1); + expect("active exec retains reference", references, 1); + expect("active exec releases all locks", held, 0); + + for (i = 0; i < sizeof(exec_fail_stages) / sizeof(*exec_fail_stages); i++) { + reset(true); + expect("setup failed exec", kexec_file_load(0, 0, 0, NULL, mk_flags), 0); + fail_at = probe_at = exec_fail_stages[i]; + expect("failed exec", multikernel_kexec_by_id(2), -EIO); + finish_probe(); + expect("failed exec retains non-reusable state after start", target.state, + !strcmp(exec_fail_stages[i], "start") ? MK_STATE_FAILED : MK_STATE_LOADED); + if (!strcmp(exec_fail_stages[i], "start")) { + expect("uncertain start retains lent PCI functions", pci_lent, 1); + fail_at = NULL; + expect("settled start failure can unload", kexec_file_load(0, 0, 0, NULL, + mk_flags | KEXEC_FILE_UNLOAD), 0); + } + } + reset(true); + expect("setup invalid CPU exec", kexec_file_load(0, 0, 0, NULL, mk_flags), 0); + cpus.nr = 0; + expect("empty CPU assignment", multikernel_kexec_by_id(2), -EINVAL); + cpus.nr = 1; + mapping_fails = true; + expect("CPU mapping failure", multikernel_kexec_by_id(2), -EINVAL); + expect("CPU errors release all locks", held, 0); + + reset(true); + expect("unknown exec ID", multikernel_kexec_by_id(3), -EINVAL); + expect("unknown load ID", kexec_file_load(0, 0, 0, NULL, + KEXEC_MULTIKERNEL | KEXEC_MK_ID(3)), -ENOENT); + expect("zero ID", kexec_file_load(0, 0, 0, NULL, KEXEC_MULTIKERNEL), -EINVAL); + expect("missing unload image", kexec_file_load(0, 0, 0, NULL, + mk_flags | KEXEC_FILE_UNLOAD), -ENOENT); + allocation_fails = true; + expect("allocation failure", kexec_file_load(0, 0, 0, NULL, mk_flags), -ENOMEM); + expect("allocation failure releases lookup reference", references, 0); + expect("error paths release all locks", held, 0); + allocation_fails = false; + atomic_store(&kexec_held, true); + expect("busy file load", kexec_file_load(0, 0, 0, NULL, mk_flags), -EBUSY); + expect("busy exec", multikernel_kexec_by_id(2), -EBUSY); + expect("busy path never takes resource lock", held, 0); + atomic_store(&kexec_held, false); + permitted = false; + expect("unprivileged load", kexec_file_load(0, 0, 0, NULL, mk_flags), -EPERM); + permitted = true; + expect("MK crash combination rejected", kexec_file_load(0, 0, 0, NULL, + mk_flags | KEXEC_FILE_ON_CRASH), -EINVAL); + expect("invalid flags rejected", kexec_file_load(0, 0, 0, NULL, 1UL << 31), -EINVAL); + + reset(false); + expect("normal load", kexec_file_load(0, 0, 0, NULL, 0), 0); + expect("normal unload", kexec_file_load(0, 0, 0, NULL, KEXEC_FILE_UNLOAD), 0); + expect("normal load releases locks", held, 0); + fail_at = "prepare"; + expect("failed normal load", kexec_file_load(0, 0, 0, NULL, 0), -EIO); + expect("failed normal load releases locks", held, 0); + reset(false); + expect("crash load", kexec_file_load(0, 0, 0, NULL, KEXEC_FILE_ON_CRASH), 0); + expect("crash unload", kexec_file_load(0, 0, 0, NULL, + KEXEC_FILE_ON_CRASH | KEXEC_FILE_UNLOAD), 0); + expect("crash load releases locks", held, 0); + fail_at = "prepare"; + expect("failed crash load", kexec_file_load(0, 0, 0, NULL, KEXEC_FILE_ON_CRASH), -EIO); + expect("failed crash load releases locks", held, 0); + reset(false); + printf("image locking: %u checks, %u failures\n", + atomic_load(&checks), atomic_load(&failures)); + return atomic_load(&failures) ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/tools/testing/selftests/multikernel/image_lock_test.sh b/tools/testing/selftests/multikernel/image_lock_test.sh new file mode 100644 index 00000000000000..035685c712cd0f --- /dev/null +++ b/tools/testing/selftests/multikernel/image_lock_test.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-image-lock.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM + +awk '/^typedef u64 mk_phys_cpu_t;/ { copy = 1 } + /^\/\*\*$/ && copy && seen { exit } + copy { print; seen = 1 }' "$src/include/linux/multikernel.h" \ + > "$build/image-types.h" +awk '/^enum mk_instance_state \{/ { copy = 1 } + copy { print } + copy && /^};/ { copy = 0 }' "$src/include/linux/multikernel.h" \ + >> "$build/image-types.h" +awk '/^#define KEXEC_/ { print }' "$src/include/uapi/linux/kexec.h" \ + >> "$build/image-types.h" +awk '/^#define KEXEC_TYPE_/ { print } + /^#define KEXEC_FILE_FLAGS/ { copy = 1 } + copy { print } + copy && !/\\$/ { copy = 0 }' "$src/include/linux/kexec.h" \ + >> "$build/image-types.h" +awk '/^enum mk_overlay_tx_status \{/ { copy = 1 } + /^struct mk_overlay_tx \{/ { copy = 1 } + copy { print } + copy && /^};/ { copy = 0 }' "$src/kernel/multikernel/overlay.c" \ + >> "$build/image-types.h" +awk '/^void mk_instance_set_state\(/ { copy = 1 } + /^int mk_instance_start\(/ { copy = 1 } + /^struct mk_instance \*mk_instance_find\(/ { copy = 1 } + /^int mk_instance_set_kexec_active\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/core.c" \ + > "$build/image-instance.h" +awk '/^void kimage_free_multikernel\(/ { copy = 1 } + /^int kimage_can_free_multikernel\(/ { copy = 1 } + /^struct kimage \*kimage_find_by_id\(/ { copy = 1 } + /^int multikernel_kexec_by_id\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/kexec_core.c" \ + > "$build/image-core.h" +awk '/^static int$/ { candidate = $0; next } + /^kimage_file_alloc_init\(/ { print candidate; copy = 1 } + /^SYSCALL_DEFINE5\(kexec_file_load,/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/kexec_file.c" \ + > "$build/image-file.h" +awk '/^void kimage_file_free_cmdline\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/kexec_file.c" \ + > "$build/image-cmdline.h" +awk '/^int mk_overlay_rmdir\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/overlay.c" \ + > "$build/image-overlay.h" + +${CC:-cc} ${CFLAGS:-} -std=gnu11 -pthread -Wall -Wextra -Werror \ + -Wno-unused-parameter -Wno-unused-but-set-variable -Wno-sign-compare \ + -I"$build" -I"$src/tools/include" "$test_dir/image_lock_test.c" \ + -o "$build/image_lock_test" +timeout 45 "$build/image_lock_test" diff --git a/tools/testing/selftests/multikernel/instance_lookup_test.c b/tools/testing/selftests/multikernel/instance_lookup_test.c new file mode 100644 index 00000000000000..1d1f189b0f9ca8 --- /dev/null +++ b/tools/testing/selftests/multikernel/instance_lookup_test.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Execute the real lookup with hard-IPI context and lock boundaries modeled. */ +#include +#include +#include + +struct mk_instance { unsigned int refcount; }; +static struct mk_instance instance = { .refcount = 1 }; +static bool mk_instance_mutex, mk_instance_lookup_lock; +static int mk_instance_idr; +static unsigned int sleeping_locks; +#define mutex_lock(m) do { assert(!*(m)); *(m) = true; sleeping_locks++; } while (0) +#define mutex_unlock(m) do { assert(*(m)); *(m) = false; } while (0) +#define raw_spin_lock_irqsave(m, flags) \ + do { assert(!*(m)); *(m) = true; (flags) = 0; } while (0) +#define raw_spin_unlock_irqrestore(m, flags) \ + do { assert(*(m)); *(m) = false; (void)(flags); } while (0) + +static struct mk_instance *idr_find(int *idr, int id) +{ + assert(idr == &mk_instance_idr); + assert(mk_instance_mutex || mk_instance_lookup_lock); + return id == 7 ? &instance : NULL; +} + +static bool kref_get_unless_zero(unsigned int *ref) +{ + assert(mk_instance_mutex || mk_instance_lookup_lock); + if (!*ref) + return false; + (*ref)++; + return true; +} +#define rcu_read_lock() do { assert(!mk_instance_lookup_lock); mk_instance_lookup_lock = true; } while (0) +#define rcu_read_unlock() do { assert(mk_instance_lookup_lock); mk_instance_lookup_lock = false; } while (0) + +#include "lookup-production.h" + +int main(void) +{ + assert(mk_instance_find(7) == &instance && instance.refcount == 2); + assert(!mk_instance_find(8) && instance.refcount == 2); + instance.refcount = 0; + assert(!mk_instance_find(7)); + assert(!mk_instance_mutex && !mk_instance_lookup_lock); + if (sleeping_locks) { + fprintf(stderr, "FAIL: IPI instance lookup acquired %u sleeping locks\n", + sleeping_locks); + return 1; + } + puts("Instance lookup: hard-IPI lookup pins without sleeping, PASS"); + return 0; +} diff --git a/tools/testing/selftests/multikernel/instance_lookup_test.sh b/tools/testing/selftests/multikernel/instance_lookup_test.sh new file mode 100644 index 00000000000000..618b9668a3128b --- /dev/null +++ b/tools/testing/selftests/multikernel/instance_lookup_test.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-instance-lookup.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^struct mk_instance \*mk_instance_find\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/core.c" \ + > "$build/lookup-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -I"$build" \ + "$test_dir/instance_lookup_test.c" -o "$build/instance_lookup_test" +"$build/instance_lookup_test" diff --git a/tools/testing/selftests/multikernel/instance_release_test.c b/tools/testing/selftests/multikernel/instance_release_test.c new file mode 100644 index 00000000000000..a125dc801dd309 --- /dev/null +++ b/tools/testing/selftests/multikernel/instance_release_test.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real last-reference callback; scheduler/resource-release boundaries modeled. */ +#include +#include +#include +#include +#include + +#define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) +#define pr_info(...) do { } while (0) +#define pr_err(...) do { } while (0) +#define MK_STATE_READY 1 +struct kref { int unused; }; +struct work_struct { void (*func)(struct work_struct *); }; +struct mk_instance { + struct kref refcount; + struct work_struct release_work; + void *host_tree, *cpus, *name; + int state; + void *kimage; +}; +static unsigned int released, atomic_releases; +static int retire_error; +static bool atomic_context, mk_overlay_mutex; +static struct work_struct *queued; +#define mutex_lock(m) do { assert(!*(m)); *(m) = true; } while (0) +#define mutex_unlock(m) do { assert(*(m)); *(m) = false; } while (0) + +static void release_resource(void *p) +{ + (void)p; + released++; + atomic_releases += atomic_context; +} +#define mk_instance_return_all_cpus(p) release_resource(p) +#define mk_instance_return_pci_devices(p) release_resource(p) +#define mk_instance_return_platform_devices(p) release_resource(p) +#define mk_instance_free_memory(p) release_resource(p) +#define mk_instance_free_boot_pages(p) release_resource(p) +#define mk_instance_track_dump(...) do { } while (0) +#define mk_cpu_set_free(p) release_resource(p) +#define kfree(p) release_resource(p) +#define schedule_work(p) do { assert(!queued); queued = (p); } while (0) +#define INIT_WORK(p, f) ((p)->func = (f)) +#define synchronize_rcu() assert(!atomic_context && !mk_overlay_mutex) +static int mk_instance_prepare_destroy(struct mk_instance *p) +{ + release_resource(p); + return 0; +} +static int mk_instance_retire_resources(struct mk_instance *p) +{ + if (retire_error) + return retire_error; + mk_instance_return_pci_devices(p); + mk_instance_return_platform_devices(p); + mk_instance_free_memory(p); + return 0; +} +#include "release-production.h" + +int main(void) +{ + struct mk_instance instance = { .state = MK_STATE_READY }; + + atomic_context = true; + mk_instance_release(&instance.refcount); + atomic_context = false; + if (atomic_releases || released) { + fprintf(stderr, "FAIL: last IPI reference reclaimed resources in atomic context\n"); + return 1; + } + assert(queued == &instance.release_work); + queued->func(queued); + assert(released == 9 && !mk_overlay_mutex); + released = 0; + retire_error = -EIO; + queued->func(queued); + if (released != 1 || mk_overlay_mutex) { + fprintf(stderr, "FAIL: final release discarded record after failed unmap\n"); + return 1; + } + puts("Instance release: final IPI reference defers reclamation, PASS"); + return 0; +} diff --git a/tools/testing/selftests/multikernel/instance_release_test.sh b/tools/testing/selftests/multikernel/instance_release_test.sh new file mode 100644 index 00000000000000..24a97278c0453b --- /dev/null +++ b/tools/testing/selftests/multikernel/instance_release_test.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-instance-release.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^static void mk_instance_release(_work)?\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/core.c" \ + > "$build/release-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -I"$build" \ + "$test_dir/instance_release_test.c" -o "$build/instance_release_test" +"$build/instance_release_test" diff --git a/tools/testing/selftests/multikernel/ipi_lifetime_test.c b/tools/testing/selftests/multikernel/ipi_lifetime_test.c new file mode 100644 index 00000000000000..86778696f4d151 --- /dev/null +++ b/tools/testing/selftests/multikernel/ipi_lifetime_test.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real registration, removal and dispatch; pthread-controlled RCU boundary. */ +#include +#include +#include +#include +#include +#include +#include + +#define __rcu +#define GFP_KERNEL 0 +#define EXPORT_SYMBOL(x) +#define MK_IPI_RING_SIZE 8 +#define MK_MAX_DATA_SIZE 16 +#define pr_warn_once(...) do { } while (0) +#define kzalloc(n, flags) calloc(1, n) +typedef pthread_mutex_t raw_spinlock_t; +#define __RAW_SPIN_LOCK_UNLOCKED(n) PTHREAD_MUTEX_INITIALIZER +#define raw_spin_lock(m) assert(!pthread_mutex_lock(m)) +#define raw_spin_unlock(m) assert(!pthread_mutex_unlock(m)) +#define raw_spin_lock_irqsave(m, flags) \ + do { (flags) = 0; raw_spin_lock(m); } while (0) +#define raw_spin_unlock_irqrestore(m, flags) \ + do { (void)(flags); raw_spin_unlock(m); } while (0) +#define lockdep_is_held(m) true +#define rcu_dereference(p) __atomic_load_n(&(p), __ATOMIC_ACQUIRE) +#define rcu_dereference_protected(p, c) rcu_dereference(p) +#define rcu_assign_pointer(p, v) __atomic_store_n(&(p), (v), __ATOMIC_RELEASE) +#define RCU_INIT_POINTER(p, v) rcu_assign_pointer(p, v) +#define atomic_read(p) (*(p)) +#define atomic_set(p, v) (*(p) = (v)) +#define atomic_set_release(p, v) __atomic_store_n((p), (v), __ATOMIC_RELEASE) +#define smp_load_acquire(p) (*(p)) + +struct mk_ipi_data { size_t data_size; unsigned long type; }; +typedef void (*mk_ipi_callback_t)(struct mk_ipi_data *, void *); +#include "ipi-handler.h" +struct shared { + struct { + unsigned int head, tail; + struct mk_ipi_data entries[MK_IPI_RING_SIZE]; + } ring; +}; +static struct shared shared; +static struct { struct shared *ipi_data; } self = { &shared }, *mk_self = &self; +static pthread_mutex_t gate = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t changed = PTHREAD_COND_INITIALIZER; +static bool entered, release_callback, unregister_done, grace_entered; +static unsigned int readers, callbacks; +static atomic_bool freed; +static void *retired; + +static void kfree(void *p) +{ + /* Retain the bytes until threads join, recording premature reclamation. */ + if (p) { + retired = p; + atomic_store(&freed, true); + } +} + +static __attribute__((unused)) void rcu_read_lock(void) +{ + assert(!pthread_mutex_lock(&gate)); + readers++; + assert(!pthread_mutex_unlock(&gate)); +} + +static __attribute__((unused)) void rcu_read_unlock(void) +{ + assert(!pthread_mutex_lock(&gate)); + assert(readers); + readers--; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); +} + +static __attribute__((unused)) void synchronize_rcu(void) +{ + assert(!pthread_mutex_lock(&gate)); + grace_entered = true; + assert(!pthread_cond_broadcast(&changed)); + while (readers) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!pthread_mutex_unlock(&gate)); +} + +#include "ipi-production.h" + +static void callback(struct mk_ipi_data *slot, void *ctx) +{ + assert(slot->type == 17 && ctx == &shared); + assert(!pthread_mutex_lock(&gate)); + callbacks++; + entered = true; + assert(!pthread_cond_broadcast(&changed)); + while (!release_callback) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!atomic_load(&freed)); + assert(!pthread_mutex_unlock(&gate)); +} + +static void *dispatch(void *unused) +{ + (void)unused; + mk_ipi_drain_ring(); + return NULL; +} + +static void *unregister(void *handler) +{ + multikernel_unregister_handler(handler); + assert(!pthread_mutex_lock(&gate)); + unregister_done = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + return NULL; +} + +int main(void) +{ + struct mk_ipi_handler *handler; + pthread_t receive, remove; + bool premature; + + assert(!multikernel_register_handler(NULL, NULL, 17)); + multikernel_unregister_handler(NULL); + handler = multikernel_register_handler(callback, &shared, 17); + assert(handler); + shared.ring.head = 1; + shared.ring.entries[0] = (struct mk_ipi_data) { .data_size = 1, .type = 17 }; + assert(!pthread_create(&receive, NULL, dispatch, NULL)); + assert(!pthread_mutex_lock(&gate)); + while (!entered) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_create(&remove, NULL, unregister, handler)); + assert(!pthread_mutex_lock(&gate)); + while (!unregister_done && !grace_entered) + assert(!pthread_cond_wait(&changed, &gate)); + /* Entry into a grace period alone does not prove callback protection. */ + premature = unregister_done || atomic_load(&freed) || readers != 1; + /* Let the baseline finish without dereferencing retired context. */ + if (premature) + atomic_store(&freed, false); + release_callback = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_join(receive, NULL)); + assert(!pthread_join(remove, NULL)); + assert(callbacks == 1 && shared.ring.tail == 1 && !mk_handlers); + free(retired); + shared.ring.head = 2; + shared.ring.entries[1] = (struct mk_ipi_data) { .data_size = 1, .type = 17 }; + mk_ipi_drain_ring(); + assert(callbacks == 1 && shared.ring.tail == 2); + if (premature) { + fprintf(stderr, "FAIL: unregister reclaimed an entered callback\n"); + return 1; + } + puts("IPI lifetime: entered callback drained before unregister, PASS"); + return 0; +} diff --git a/tools/testing/selftests/multikernel/ipi_lifetime_test.sh b/tools/testing/selftests/multikernel/ipi_lifetime_test.sh new file mode 100644 index 00000000000000..81367c675fc295 --- /dev/null +++ b/tools/testing/selftests/multikernel/ipi_lifetime_test.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-ipi-lifetime.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM + +awk '/^struct mk_ipi_handler \{/ { copy = 1 } + copy { print } + copy && /^};/ { copy = 0 }' "$src/include/linux/multikernel.h" \ + > "$build/ipi-handler.h" +awk '/^static struct mk_ipi_handler / { print } + /^static raw_spinlock_t mk_handlers_lock/ { print } + /^static inline unsigned int mk_ring_idx\(/ { copy = 1 } + /^struct mk_ipi_handler \*multikernel_register_handler\(/ { copy = 1 } + /^void multikernel_unregister_handler\(/ { copy = 1 } + /^static (void|int) mk_ipi_drain_ring\(void\)$/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/ipi.c" \ + > "$build/ipi-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -pthread -Wall -Wextra -Werror \ + -I"$build" "$test_dir/ipi_lifetime_test.c" -o "$build/ipi_lifetime_test" +"$build/ipi_lifetime_test" diff --git a/tools/testing/selftests/multikernel/ipi_writer_test.c b/tools/testing/selftests/multikernel/ipi_writer_test.c new file mode 100644 index 00000000000000..e1c772fd95576c --- /dev/null +++ b/tools/testing/selftests/multikernel/ipi_writer_test.c @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real ring producer; allocation, physical IPI and lock boundaries modeled. */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define MK_IPI_RING_SIZE 8 +typedef uint32_t u32; +typedef uint64_t u64; +typedef uint8_t u8; +#include "writer-types.h" +#define MK_PHYS_CPU_INVALID (~0UL) +typedef unsigned long mk_phys_cpu_t; +struct mk_ipi_data { + size_t data_size; + unsigned long sender_cpu, type; + char buffer[MK_MAX_DATA_SIZE]; +}; +struct mk_shared_data { + struct { + unsigned int head, tail; + struct mk_ipi_data entries[MK_IPI_RING_SIZE]; + } ring; +}; +struct mk_instance { + int id; + struct mk_shared_data *ipi_data; + mk_phys_cpu_t ipi_target; + void *cpus; + bool ipi_lock; + unsigned long ipi_phys; +}; +static struct mk_shared_data shared; +static struct mk_instance instance = { .ipi_data = &shared, .ipi_target = 7 }; +static struct mk_instance *mk_self = &instance; +typedef void (*mk_ipi_callback_t)(struct mk_ipi_data *, void *); +struct mk_ipi_handler { + unsigned long ipi_type; + mk_ipi_callback_t callback; + void *context; + struct mk_ipi_handler *next; +}; +static struct mk_ipi_handler *mk_handlers; +static unsigned int acquired_tails, released_tails; +#define rcu_read_lock() do { } while (0) +#define rcu_read_unlock() do { } while (0) +#define rcu_dereference(p) (p) +#define pr_warn_once(...) do { } while (0) +#define smp_load_acquire(p) (*(p)) +#define atomic_set(p, v) (*(p) = (v)) +static __attribute__((unused)) unsigned int atomic_read_acquire(unsigned int *p) +{ + assert(p == &shared.ring.tail); + acquired_tails++; + return *p; +} +static __attribute__((unused)) void atomic_set_release(unsigned int *p, unsigned int value) +{ + assert(p == &shared.ring.tail); + assert(shared.ring.entries[shared.ring.tail].data_size == 0); + *p = value; + released_tails++; +} +static unsigned int refs, ipis, unlocked_writes; +static bool mk_overlay_mutex = true; +#define lockdep_assert_held(m) assert(*(m)) +#define phys_to_virt(p) ((void *)(p)) +#define pr_err(...) do { } while (0) +#define KERN_WARNING "" +#define printk_deferred(...) do { if (0) printf(__VA_ARGS__); } while (0) +#define atomic_read(p) (*(p)) +#define atomic_cmpxchg(p, old, new) \ + ({ \ + unsigned int _old = *(p); \ + if (_old == (old)) \ + *(p) = (new); \ + _old; \ + }) +static _Alignas(struct mk_message) unsigned char expected_wire[MK_MAX_DATA_SIZE]; +static size_t expected_size; +static bool check_wire; +static void publish_size(size_t *p, size_t size) +{ + struct mk_ipi_data *slot = + &shared.ring.entries[(shared.ring.head - 1) & (MK_IPI_RING_SIZE - 1)]; + + assert(instance.ipi_lock && p == &slot->data_size); + if (check_wire) { + assert(size == expected_size); + assert(!memcmp(slot->buffer, expected_wire, size)); + } + *p = size; +} +/* This checks publication order, not concurrent kernel memory ordering. */ +#define smp_store_release(p, v) publish_size(p, v) +#define raw_spin_lock_irqsave(m, flags) \ + do { assert(!*(m)); *(m) = true; (flags) = 0; } while (0) +#define raw_spin_unlock_irqrestore(m, flags) \ + do { assert(*(m)); *(m) = false; (void)(flags); } while (0) +#define mk_cpu_set_first(p) 7UL +#define smp_processor_id() 3 +#define arch_cpu_physical_id(cpu) ((unsigned long)(cpu)) + +static struct mk_instance *mk_instance_find(int id) +{ + if (id != 9) + return NULL; + refs++; + return &instance; +} + +static void mk_instance_put(struct mk_instance *p) +{ + assert(p == &instance && refs); + refs--; +} + +static void *copy_payload(void *dst, const void *src, size_t len) +{ + unlocked_writes += !instance.ipi_lock; + return memcpy(dst, src, len); +} +#define memcpy copy_payload + +static void mk_arch_send_ipi(mk_phys_cpu_t cpu) +{ + assert(cpu == 7 && !instance.ipi_lock); + ipis++; +} + +static void *clear_ring(void *dst, int value, size_t len) +{ + if (dst == &shared) + assert(mk_overlay_mutex && !instance.ipi_data && !instance.ipi_lock); + else + unlocked_writes += !instance.ipi_lock; + return memset(dst, value, len); +} +#define memset clear_ring + +int multikernel_send_ipi_data_to(struct mk_instance *p, void *data, + size_t size, unsigned long type); +#include "writer-production.h" +#undef memcpy +#undef memset + +static void check_messages(void) +{ + unsigned char payload[MK_MAX_DATA_SIZE]; + struct mk_message message = { + .msg_type = 17, .msg_subtype = 23, .msg_id = UINT64_MAX, + }; + const unsigned int sizes[] = { 0, 1, 16, MK_MAX_DATA_SIZE - sizeof(message) }; + unsigned int route, null_payload, i; + unsigned int before; + + memset(payload, 0xa7, sizeof(payload)); + for (route = 0; route < 2; route++) { + for (null_payload = 0; null_payload < 2; null_payload++) { + for (i = 0; i < ARRAY_SIZE(sizes); i++) { + struct mk_message *wire = (struct mk_message *)expected_wire; + const void *data = null_payload ? NULL : payload; + + mk_instance_ipi_reset(&instance); + /* Stale bytes must not leak through padding or NULL payloads. */ + memset(shared.ring.entries[0].buffer, 0x5a, MK_MAX_DATA_SIZE); + message.payload_len = sizes[i]; + expected_size = sizeof(message) + sizes[i]; + memset(expected_wire, 0, sizeof(expected_wire)); + wire->msg_type = message.msg_type; + wire->msg_subtype = message.msg_subtype; + wire->msg_id = message.msg_id; + wire->payload_len = sizes[i]; + if (data) + memcpy(wire->payload, data, sizes[i]); + refs = route; + before = ipis; + check_wire = true; + assert(!mk_send_message_ipi(9, route ? &instance : NULL, + &message, data)); + check_wire = false; + assert(refs == route && ipis == before + 1); + assert(shared.ring.head == 1 && !instance.ipi_lock); + assert(shared.ring.entries[0].type == message.msg_type); + if (route) + mk_instance_put(&instance); + } + } + } + /* All rejection paths retain caller references and leave the ring alone. */ + before = ipis; + message.payload_len++; + assert(mk_send_message_ipi(9, NULL, &message, payload) == -EINVAL); + message.payload_len = UINT32_MAX; + assert(mk_send_message_ipi(9, NULL, &message, payload) == -EINVAL); + message.payload_len = 16; + assert(mk_send_message_ipi(77, NULL, &message, payload) == -EINVAL); + shared.ring.tail = 2; + assert(mk_send_message_ipi(9, NULL, &message, payload) == -ENOSPC); + mk_instance_ipi_deactivate(&instance); + assert(mk_send_message_ipi(9, NULL, &message, payload) == -ENODEV); + refs = 1; + assert(mk_send_message_ipi(9, &instance, &message, payload) == -ENODEV); + assert(refs == 1 && ipis == before && shared.ring.head == 1); + assert(!instance.ipi_lock); + mk_instance_put(&instance); + puts("IPI message writer: 16 wire cases and six rejection paths passed"); +} + +int main(void) +{ + char payload[] = "hello"; + + assert(!multikernel_send_ipi_data(9, payload, sizeof(payload), 17)); + assert(shared.ring.head == 1 && !strcmp(shared.ring.entries[0].buffer, payload)); + assert(shared.ring.entries[0].data_size == sizeof(payload) && ipis == 1); + assert(acquired_tails == 1); + mk_ipi_drain_ring(); + assert(released_tails == 1 && shared.ring.tail == 1); + /* Invalid-size drops must release reuse capacity in the same order. */ + shared.ring.entries[1].data_size = MK_MAX_DATA_SIZE + 1; + shared.ring.head = 2; + mk_ipi_drain_ring(); + assert(released_tails == 2 && shared.ring.tail == 2); + assert(!refs && !instance.ipi_lock); + mk_instance_ipi_deactivate(&instance); + assert(multikernel_send_ipi_data(9, payload, sizeof(payload), 17) == -ENODEV); + assert(!refs && !instance.ipi_lock && ipis == 1); + instance.ipi_phys = (unsigned long)&shared; + mk_instance_ipi_reset(&instance); + assert(instance.ipi_data == &shared && !shared.ring.head); + assert(!shared.ring.entries[0].data_size && !instance.ipi_lock); + assert(!multikernel_send_ipi_data(9, payload, sizeof(payload), 17)); + assert(ipis == 2 && shared.ring.head == 1 && !refs); + /* The reference route must neither acquire nor drop the caller's pin. */ + refs = 1; + assert(!multikernel_send_ipi_data_to(&instance, payload, sizeof(payload), 17)); + assert(refs == 1 && ipis == 3 && shared.ring.head == 2); + shared.ring.tail = 3; + assert(multikernel_send_ipi_data_to(&instance, payload, sizeof(payload), 17) == -ENOSPC); + assert(refs == 1 && ipis == 3 && !instance.ipi_lock); + assert(multikernel_send_ipi_data_to(&instance, payload, MK_MAX_DATA_SIZE + 1, 17) == -EINVAL); + mk_instance_ipi_deactivate(&instance); + assert(multikernel_send_ipi_data_to(&instance, payload, sizeof(payload), 17) == -ENODEV); + assert(refs == 1 && ipis == 3 && !instance.ipi_lock); + mk_instance_put(&instance); + check_messages(); + if (unlocked_writes) { + fprintf(stderr, "FAIL: ring producer bypasses reset/deactivation lock\n"); + return 1; + } + puts("IPI writer: producer excludes reset/deactivation, PASS"); + return 0; +} diff --git a/tools/testing/selftests/multikernel/ipi_writer_test.sh b/tools/testing/selftests/multikernel/ipi_writer_test.sh new file mode 100644 index 00000000000000..297e260384299e --- /dev/null +++ b/tools/testing/selftests/multikernel/ipi_writer_test.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-ipi-writer.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^#define MK_MAX_DATA_SIZE / { print } + /^struct mk_message \{/ { copy = 1 } + copy { print } + copy && /^};/ { copy = 0 }' "$src/include/linux/multikernel.h" \ + > "$build/writer-types.h" +awk '/^static inline unsigned int mk_ring_idx\(/ { copy = 1 } + /^static int mk_send_ipi_data\(/ { copy = 1 } + /^int mk_send_message_ipi\(/ { copy = 1 } + /^int multikernel_send_ipi_data(_to)?\(/ { copy = 1 } + /^void mk_instance_ipi_(reset|publish|deactivate)\(/ { copy = 1 } + /^static (void|int) mk_ipi_drain_ring\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/ipi.c" \ + > "$build/writer-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -I"$build" -I"$src/tools/include" \ + "$test_dir/ipi_writer_test.c" -o "$build/ipi_writer_test" +"$build/ipi_writer_test" diff --git a/tools/testing/selftests/multikernel/kernfs_snapshot_test.c b/tools/testing/selftests/multikernel/kernfs_snapshot_test.c new file mode 100644 index 00000000000000..4aa27a46ce23a2 --- /dev/null +++ b/tools/testing/selftests/multikernel/kernfs_snapshot_test.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Production kernfs callbacks; DT generation and kernfs I/O are boundaries. */ +#include +#include +#include +#include +#include +#include +#include + +struct mk_instance { const char *name; }; +struct kernfs_node { void *priv; }; +struct kernfs_open_file { struct kernfs_node *kn; }; +struct seq_file { char data[32]; size_t count; }; + +static struct mk_instance root = { "root" }, child = { "child" }; +static struct mk_instance *mk_self = &root; +static struct mk_instance *expected_instance; +static pthread_mutex_t mk_overlay_mutex = PTHREAD_MUTEX_INITIALIZER; +static _Thread_local bool snapshot_locked; +static const char snapshot[] = "dtb-snapshot"; +static unsigned int failures, generated, allocated, freed; +static int generation_error; + +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "FAIL line %d: %s\n", __LINE__, #condition); \ + failures++; \ + } \ +} while (0) +#define pr_err(...) do { } while (0) +#define min(a, b) ((a) < (b) ? (a) : (b)) + +static inline int mutex_trylock(pthread_mutex_t *mutex) +{ + int ret = pthread_mutex_trylock(mutex); + + CHECK(ret == 0 || ret == EBUSY); + if (!ret) + snapshot_locked = true; + return !ret; +} + +/* Keep a blocking-lock regression runnable: the runner bounds its lifetime. */ +static inline __attribute__((unused)) void mutex_lock(pthread_mutex_t *mutex) +{ + CHECK(pthread_mutex_lock(mutex) == 0); + snapshot_locked = true; +} + +static inline void mutex_unlock(pthread_mutex_t *mutex) +{ + CHECK(snapshot_locked); + snapshot_locked = false; + CHECK(pthread_mutex_unlock(mutex) == 0); +} + +static int mk_dt_generate_instance_dtb(struct mk_instance *instance, + void **data, size_t *size) +{ + int ret; + + generated++; + CHECK(instance == expected_instance); + /* Assert exclusion without touching mutable kernel resource containers. */ + CHECK(snapshot_locked); + ret = pthread_mutex_trylock(&mk_overlay_mutex); + CHECK(ret == EBUSY); + if (!ret) + pthread_mutex_unlock(&mk_overlay_mutex); + if (generation_error) + return generation_error; + *data = malloc(sizeof(snapshot) - 1); + if (!*data) { + perror("malloc"); + exit(2); + } + allocated++; + memcpy(*data, snapshot, sizeof(snapshot) - 1); + *size = sizeof(snapshot) - 1; + return 0; +} + +static void kfree(void *data) +{ + if (data) + freed++; + free(data); +} + +static void seq_write(struct seq_file *sf, const void *data, size_t size) +{ + CHECK(size <= sizeof(sf->data)); + if (size <= sizeof(sf->data)) { + memcpy(sf->data, data, size); + sf->count = size; + } +} + +#include "kernfs-snapshot-production.h" + +static void check_released(void) +{ + int ret = pthread_mutex_trylock(&mk_overlay_mutex); + + CHECK(!snapshot_locked); + CHECK(ret == 0); + if (!ret) + pthread_mutex_unlock(&mk_overlay_mutex); + CHECK(allocated == freed); +} + +static void check_reads(void) +{ + struct kernfs_node kn = { &child }; + struct kernfs_open_file of = { &kn }; + struct seq_file sf = { 0 }; + char buf[32] = { 0 }; + + expected_instance = &root; + CHECK(root_device_tree_seq_show(&sf, NULL) == 0); + CHECK(sf.count == sizeof(snapshot) - 1); + CHECK(!memcmp(sf.data, snapshot, sizeof(snapshot) - 1)); + check_released(); + expected_instance = &child; + CHECK(instance_device_tree_read(&of, buf, sizeof(buf), 0) == + sizeof(snapshot) - 1); + CHECK(!memcmp(buf, snapshot, sizeof(snapshot) - 1)); + check_released(); + CHECK(instance_device_tree_read(&of, buf, 4, 3) == 4); + CHECK(!memcmp(buf, snapshot + 3, 4)); + check_released(); + CHECK(instance_device_tree_read(&of, buf, sizeof(buf), sizeof(snapshot) - 1) == 0); + check_released(); + CHECK(instance_device_tree_read(&of, buf, sizeof(buf), sizeof(snapshot) + 5) == 0); + check_released(); + CHECK(instance_device_tree_read(&of, buf, 0, 0) == 0); + check_released(); +} + +static void check_errors(void) +{ + struct kernfs_node kn = { &child }; + struct kernfs_open_file of = { &kn }; + struct seq_file sf = { 0 }; + char buf[32] = "unchanged"; + const int errors[] = { -ENOMEM, -EINVAL }; + size_t i; + + for (i = 0; i < sizeof(errors) / sizeof(errors[0]); i++) { + generation_error = errors[i]; + expected_instance = &root; + CHECK(root_device_tree_seq_show(&sf, NULL) == errors[i]); + CHECK(sf.count == 0); + check_released(); + expected_instance = &child; + CHECK(instance_device_tree_read(&of, buf, sizeof(buf), 0) == errors[i]); + CHECK(!strcmp(buf, "unchanged")); + check_released(); + } + generation_error = 0; +} + +static void *check_busy_reads(void *unused) +{ + struct kernfs_node kn = { &child }; + struct kernfs_open_file of = { &kn }; + struct seq_file sf = { 0 }; + char buf[32] = "unchanged"; + unsigned int before = generated; + + expected_instance = &root; + CHECK(root_device_tree_seq_show(&sf, NULL) == -EAGAIN); + CHECK(sf.count == 0); + CHECK(generated == before); + expected_instance = &child; + CHECK(instance_device_tree_read(&of, buf, sizeof(buf), 0) == -EAGAIN); + CHECK(!strcmp(buf, "unchanged")); + CHECK(generated == before); + CHECK(!snapshot_locked); + return NULL; +} + +int main(void) +{ + pthread_t reader; + int ret; + + check_reads(); + check_errors(); + /* Model a controller holding the resource lock while kernfs drains. */ + CHECK(pthread_mutex_lock(&mk_overlay_mutex) == 0); + ret = pthread_create(&reader, NULL, check_busy_reads, NULL); + if (ret) { + fprintf(stderr, "pthread_create: %s\n", strerror(ret)); + return 2; + } + ret = pthread_join(reader, NULL); + if (ret) { + fprintf(stderr, "pthread_join: %s\n", strerror(ret)); + return 2; + } + CHECK(pthread_mutex_unlock(&mk_overlay_mutex) == 0); + check_released(); + /* Retry the same callbacks after the controller lets go. */ + check_reads(); + if (failures) { + fprintf(stderr, "%u kernfs snapshot assertion(s) failed\n", failures); + return 1; + } + puts("PASS: production kernfs snapshot exclusion, contention, retry, errors and reads"); + return 0; +} diff --git a/tools/testing/selftests/multikernel/kernfs_snapshot_test.sh b/tools/testing/selftests/multikernel/kernfs_snapshot_test.sh new file mode 100644 index 00000000000000..aea04814690074 --- /dev/null +++ b/tools/testing/selftests/multikernel/kernfs_snapshot_test.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-kernfs-snapshot.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^static int root_device_tree_seq_show\(/ || + /^static ssize_t instance_device_tree_read\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/kernfs.c" \ + > "$build/kernfs-snapshot-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror \ + -Wno-unused-parameter -Wno-sign-compare -pthread -I"$build" \ + "$test_dir/kernfs_snapshot_test.c" -o "$build/kernfs_snapshot_test" +timeout 15 "$build/kernfs_snapshot_test" diff --git a/tools/testing/selftests/multikernel/mailbox_lifetime_test.c b/tools/testing/selftests/multikernel/mailbox_lifetime_test.c new file mode 100644 index 00000000000000..9c110c6498c41f --- /dev/null +++ b/tools/testing/selftests/multikernel/mailbox_lifetime_test.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real retirement entry points, empty CPU tracking, counted allocator calls. */ +#include +#include +#include +#include +#include +#include "mailbox-states.h" + +#define PAGE_SIZE 4096 +#define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1)) +#define READ_ONCE(x) (x) +static bool warn_on_once(bool condition) { return condition; } +#define WARN_ON_ONCE(x) warn_on_once(x) +#define pr_info(...) ((void)0) +#define pr_err(...) ((void)0) +typedef unsigned long mk_phys_cpu_t; +typedef unsigned long phys_addr_t; +typedef unsigned int u32; + +struct mk_spawn_context { + u32 ready; +}; + +struct mk_cpu_set { + unsigned int nr; + mk_phys_cpu_t *ids; +}; + +struct mk_instance { + int id; + const char *name; + struct mk_cpu_set *cpus_on_slot; + struct { + struct mk_spawn_context *spawn_ctx; + unsigned long spawn_ctx_phys; + void *ident_pgt, *trampoline_va, *park_va; + } arch; +}; + +struct mk_pool { + bool park_lock; + struct { + struct mk_spawn_context *slot; + unsigned long slot_phys, park_phys, cr3; + void *park_va, *pgt; + } arch; +}; + +static struct mk_pool pool, *mk_pool = &pool; +static struct mk_spawn_context ctx, pool_slot; +static unsigned int frees, unmaps, checks; +static int unmap_error; +#define mk_cpu_set_empty(p) (!(p) || !(p)->nr) +#define mk_cpu_set_count(p) ((p) ? (p)->nr : 0) +#define mk_cpu_set_format(buf, len, set) ((void)(buf), (void)(len), (void)(set)) +#define mk_cpu_set_free(p) ((void)(p), frees++) +#define mk_cpu_set_del(p, cpu) ((void)(cpu), (p)->nr = 0) +#define multikernel_free(phys, size) ((void)(phys), (void)(size), frees++) +#define mk_pool_unmap(phys, size) ((void)(phys), (void)(size), unmaps++, unmap_error) +#define mk_free_identity_pgtable(p) ((void)(p), frees++) +#define set_memory_nx(virt, pages) ((void)(virt), (void)(pages), 0) +static bool mk_pool_cpus_returned(void) +{ + return true; +} + +static struct mk_spawn_context *mk_own_context(void) +{ + return &pool_slot; +} + +static int mk_publish_repark(struct mk_spawn_context *p, u32 cpu, + unsigned long park, unsigned long cr3, + unsigned long slot, const char *what) +{ + return -ETIMEDOUT; +} + +static bool *lock(bool *p) +{ + assert(!*p); + *p = true; + return p; +} + +static void unlock(bool **p) +{ + assert(**p); + **p = false; +} + +#define guard(name) bool *held_##name __attribute__((cleanup(unlock))) = lock +int mk_repark_instance_to_host(struct mk_instance *instance); +#include "lifetime-production.h" + +static void reset(struct mk_instance *instance, struct mk_cpu_set *cpus) +{ + assert(!pool.park_lock); + memset(instance, 0, sizeof(*instance)); + memset(&pool, 0, sizeof(pool)); + cpus->nr = 0; + ctx.ready = MK_SLOT_IDLE; + pool_slot.ready = MK_SLOT_IDLE; + instance->cpus_on_slot = cpus; + instance->arch.spawn_ctx = &ctx; + instance->arch.ident_pgt = (void *)1; + pool.arch.slot = &pool_slot; + pool.arch.pgt = (void *)2; + frees = unmaps = 0; +} + +int main(void) +{ + struct mk_instance instance; + struct mk_cpu_set cpus; + unsigned int state; + + for (state = MK_SLOT_PENDING; state <= MK_SLOT_QUARANTINED; state++) { + reset(&instance, &cpus); + ctx.ready = state; + assert(mk_arch_release_instance(&instance) == -EBUSY); + assert(!frees && !unmaps && instance.arch.spawn_ctx == &ctx && + instance.arch.ident_pgt); + checks++; + reset(&instance, &cpus); + pool_slot.ready = state; + assert(mk_pool_park_teardown() == -EBUSY); + assert(!frees && !unmaps && pool.arch.slot == &pool_slot && pool.arch.pgt); + checks++; + } + reset(&instance, &cpus); + assert(!mk_arch_release_instance(&instance) && frees == 2 && + !instance.arch.spawn_ctx); + checks++; + reset(&instance, &cpus); + assert(!mk_pool_park_teardown() && frees == 3 && unmaps == 2 && !pool.arch.slot); + checks++; + reset(&instance, &cpus); + unmap_error = -EIO; + mk_pool_ctrl_free(0x1000, PAGE_SIZE); + assert(!frees && unmaps == 1); + checks++; + printf("Mailbox lifetime: %u checks passed (no live CPU execution)\n", + checks); + return 0; +} diff --git a/tools/testing/selftests/multikernel/mailbox_lifetime_test.sh b/tools/testing/selftests/multikernel/mailbox_lifetime_test.sh new file mode 100644 index 00000000000000..e5ee332493db26 --- /dev/null +++ b/tools/testing/selftests/multikernel/mailbox_lifetime_test.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-mailbox-lifetime.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^#define MK_SLOT_/ { print }' "$src/arch/x86/include/asm/multikernel.h" \ + > "$build/mailbox-states.h" +awk '/^static bool mk_slot_idle\(/ { copy = 1 } + /^static void mk_pool_ctrl_free\(/ { copy = 1 } + /^int mk_(arch_release_instance|repark_instance_to_host|pool_park_teardown)\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/arch/x86/multikernel/spawn.c" > "$build/lifetime-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -Wno-unused-parameter \ + -I"$build" "$test_dir/mailbox_lifetime_test.c" -o "$build/mailbox_lifetime_test" +"$build/mailbox_lifetime_test" diff --git a/tools/testing/selftests/multikernel/mailbox_reader_test.S b/tools/testing/selftests/multikernel/mailbox_reader_test.S new file mode 100644 index 00000000000000..89fa8ecf4460e7 --- /dev/null +++ b/tools/testing/selftests/multikernel/mailbox_reader_test.S @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include "mailbox-offsets.h" +.text +.globl mailbox_read +.type mailbox_read,@function +mailbox_read: + pushq %rbx + pushq %rbp + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + movq %rdi, %r12 + movq %rsi, %r13 +#include "mailbox-reader.h" +.Lpark_wait: +.Lpark_relax: + xorl %eax, %eax + jmp .Ltest_exit +.Lpark_repark: + /* Privileged repark dispatch is covered by kernel build/runtime gates. */ + movl $-1, %eax + jmp .Ltest_exit +.globl mailbox_capture +mailbox_capture: + movq %rdi, observed(%rip) + movq %rsi, observed+8(%rip) + movq %rdx, observed+16(%rip) + movq %rcx, observed+24(%rip) + movq %r8, observed+32(%rip) + movq %r9, observed+40(%rip) + movl $1, %eax +.Ltest_exit: + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbp + popq %rbx + ret +.size mailbox_read,.-mailbox_read +.section .note.GNU-stack,"",@progbits diff --git a/tools/testing/selftests/multikernel/mailbox_reader_test.c b/tools/testing/selftests/multikernel/mailbox_reader_test.c new file mode 100644 index 00000000000000..e6da188ecea205 --- /dev/null +++ b/tools/testing/selftests/multikernel/mailbox_reader_test.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Execute extracted unprivileged reader instructions, not a kernel park loop. */ +#include +#include +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1U << (n)) +#include "mailbox-context.h" + +#ifdef EMIT_OFFSETS +#define EMIT(member) \ + printf("#define MK_CTX_" #member " %zu\n", \ + offsetof(struct mk_spawn_context, member)) +int main(void) +{ + EMIT(ready); + EMIT(target_apic_id); + EMIT(flags); + EMIT(identity_cr3); + EMIT(self_phys); + EMIT(kernel_entry); + EMIT(trampoline_phys); + EMIT(gs_base); + EMIT(stack); + EMIT(secondary_trampoline_phys); + EMIT(spawn_cr3); + printf("#define MK_CTX_bp %zu\n", sizeof(struct mk_spawn_context)); + printf("#define MK_SLOT_PENDING %u\n", MK_SLOT_PENDING); + printf("#define MK_SLOT_BUSY %u\n", MK_SLOT_BUSY); + printf("#define MK_SLOT_ACK %u\n", MK_SLOT_ACK); + printf("#define MK_SPAWN_F_REPARK %u\n", MK_SPAWN_F_REPARK); + printf("#define MK_SPAWN_F_SECONDARY %u\n", MK_SPAWN_F_SECONDARY); + return 0; +} +#else +extern int mailbox_read(struct mk_spawn_context *ctx, unsigned long cpu); +extern void mailbox_capture(void); +unsigned long observed[6]; + +int main(void) +{ + struct mk_spawn_context ctx = { + .ready = MK_SLOT_PENDING, + .target_apic_id = 17, + .identity_cr3 = 0x1000, + .self_phys = 0x2000, + .kernel_entry = 0x3000, + .trampoline_phys = (unsigned long)mailbox_capture, + }; + unsigned int state, checks = 0; + + assert(sizeof(ctx) == 144); + assert(mailbox_read(&ctx, 17) == 1 && ctx.ready == MK_SLOT_ACK); + assert(observed[0] == 0x1000 && observed[1] == 0x2090 && + observed[2] == 0x3000); + assert(observed[3] == (unsigned long)mailbox_capture); + checks++; + ctx.ready = MK_SLOT_PENDING; + ctx.flags = MK_SPAWN_F_SECONDARY; + ctx.gs_base = 0x4000; + ctx.stack = 0x5000; + ctx.secondary_trampoline_phys = (unsigned long)mailbox_capture; + ctx.spawn_cr3 = 0x6000; + assert(mailbox_read(&ctx, 17) == 1 && ctx.ready == MK_SLOT_ACK); + assert(observed[0] == 0x1000 && observed[1] == 0x3000 && + observed[2] == 0x4000); + assert(observed[3] == 0x5000 && + observed[4] == (unsigned long)mailbox_capture); + assert(observed[5] == 0x6000); + checks++; + ctx.ready = MK_SLOT_PENDING; + memset(observed, 0, sizeof(observed)); + assert(!mailbox_read(&ctx, 99) && ctx.ready == MK_SLOT_PENDING && + !observed[0]); + checks++; + for (state = MK_SLOT_IDLE; state <= MK_SLOT_QUARANTINED; state++) { + if (state == MK_SLOT_PENDING) + continue; + ctx.ready = state; + assert(!mailbox_read(&ctx, 17) && ctx.ready == state && + !observed[0]); + checks++; + } + printf("Mailbox reader: %u unprivileged assembly checks passed\n", + checks); + return 0; +} +#endif diff --git a/tools/testing/selftests/multikernel/mailbox_reader_test.sh b/tools/testing/selftests/multikernel/mailbox_reader_test.sh new file mode 100644 index 00000000000000..6220306b51f0b0 --- /dev/null +++ b/tools/testing/selftests/multikernel/mailbox_reader_test.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-mailbox-reader.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +case $(uname -m) in x86_64) ;; *) echo 'SKIP: x86-64 userspace assembly test'; exit 4 ;; esac +awk '/^#define MK_(SLOT_|SPAWN_F_)/ { print } + /^struct mk_spawn_context \{/ { copy = 1 } + copy && /struct boot_params bp;/ { print "};"; exit } + copy { print }' "$src/arch/x86/include/asm/multikernel.h" > "$build/mailbox-context.h" +${CC:-cc} -std=gnu11 -Wall -Wextra -Werror -DEMIT_OFFSETS -I"$build" \ + "$test_dir/mailbox_reader_test.c" -o "$build/offsets" +"$build/offsets" > "$build/mailbox-offsets.h" +# Execute the unprivileged primary/secondary dispatch from the real park loop. +# Omit privileged repark, monitor, CR3 and descriptor-table setup entirely. +awk '/^\.Lpark_pending:/ { copy = 1 } + /^\.Lpark_repark:/ { copy = 0 } + /^\.Lpark_wrong_target:/ { copy = 1 } + /^SYM_CODE_END/ { copy = 0 } + copy && !/ANNOTATE_RETPOLINE_SAFE/ { print }' \ + "$src/arch/x86/multikernel/direct_boot.S" > "$build/mailbox-reader.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -I"$build" \ + "$test_dir/mailbox_reader_test.c" "$test_dir/mailbox_reader_test.S" \ + -o "$build/mailbox_reader_test" +"$build/mailbox_reader_test" diff --git a/tools/testing/selftests/multikernel/mailbox_test.c b/tools/testing/selftests/multikernel/mailbox_test.c new file mode 100644 index 00000000000000..1397a9a09dca13 --- /dev/null +++ b/tools/testing/selftests/multikernel/mailbox_test.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Extracted mailbox writers, with a bounded simulated reader and timeout. */ +#include +#include +#include +#include +#include + +typedef uint32_t u32; +typedef unsigned long phys_addr_t; +#include "mailbox-states.h" +#define USEC_PER_SEC 1000000 +#define virt_to_phys(p) ((unsigned long)(p)) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +/* Baseline extraction only; this single-threaded fixture models no barriers. */ +#define smp_wmb() ((void)0) +/* Baseline extraction only; no hardware memory-order claim. */ +#define smp_store_release(p, v) (*(p) = (v)) +#define pr_err(...) ((void)0) + +struct mk_spawn_context { + unsigned long identity_cr3, kernel_entry, trampoline_virt, + trampoline_phys; + unsigned long park_phys, park_cr3, gs_base, stack; + u32 flags, ready, target_apic_id; +}; + +static struct { + struct { + unsigned long cr3; + struct mk_spawn_context *slot; + } arch; +} pool, *mk_pool = &pool; + +static struct mk_spawn_context *mk_own_context(void) +{ + return pool.arch.slot; +} + +static unsigned int failures, checks; +static unsigned int outcome; +static bool ack_at_deadline; + +static u32 test_cmpxchg(u32 *p, u32 old, u32 value) +{ + u32 seen; + + if (ack_at_deadline && old == MK_SLOT_ACK) { + *p = MK_SLOT_ACK; + ack_at_deadline = false; + } + seen = *p; + if (seen == old) + *p = value; + return seen; +} + +#define cmpxchg(p, old, value) test_cmpxchg(p, old, value) +#define xchg(p, value) __atomic_exchange_n(p, value, __ATOMIC_SEQ_CST) +/* Simulate only the reader state, not kernel execution or elapsed wall time. */ +#define read_poll_timeout(op, val, cond, delay, timeout, sleep, arg) \ + ({ \ + (void)(delay); \ + (void)(timeout); \ + (void)(sleep); \ + (arg) = outcome; \ + (val) = op(arg); \ + (cond) ? 0 : -ETIMEDOUT; \ + }) +#include "mailbox-production.h" + +static void expect(const char *name, bool pass) +{ + checks++; + if (!pass) { + fprintf(stderr, "FAIL: %s\n", name); + failures++; + } +} + +int main(void) +{ + struct mk_spawn_context slot = { .ready = MK_SLOT_BUSY }; + int ret; + + outcome = MK_SLOT_ACK; + ret = mk_slot_wake(&slot, 17, "ACK"); + expect("consume ACK before reuse", !ret && slot.ready == MK_SLOT_IDLE); + slot.ready = MK_SLOT_BUSY; + outcome = MK_SLOT_PENDING; + ret = mk_slot_wake(&slot, 17, "unclaimed timeout"); + expect("pending timeout quarantines instead of revoking", + ret == -ETIMEDOUT && slot.ready == MK_SLOT_QUARANTINED); + slot.ready = MK_SLOT_BUSY; + outcome = MK_SLOT_BUSY; + ret = mk_slot_wake(&slot, 17, "claimed timeout"); + expect("stalled reader cannot authorize reuse", + ret == -ETIMEDOUT && slot.ready == MK_SLOT_QUARANTINED); + slot.ready = MK_SLOT_BUSY; + outcome = MK_SLOT_BUSY; + ack_at_deadline = true; + ret = mk_slot_wake(&slot, 17, "ACK wins timeout race"); + expect("completed ACK wins timeout race", + !ret && slot.ready == MK_SLOT_IDLE); + slot.ready = MK_SLOT_QUARANTINED; + slot.kernel_entry = 0x1234; + mk_set_spawn_context(&slot, 1, 2, 3, 4, 5); + expect("context setup cannot reset quarantine", + slot.ready == MK_SLOT_QUARANTINED && + slot.kernel_entry == 0x1234); + slot.ready = MK_SLOT_BUSY; + mk_set_spawn_context(&slot, 1, 2, 3, 4, 5); + expect("context setup cannot overwrite another owner", + slot.ready == MK_SLOT_BUSY && slot.kernel_entry == 0x1234); + slot.ready = MK_SLOT_IDLE; + mk_set_spawn_context(&slot, 1, 2, 3, 4, 5); + expect("idle setup returns mailbox to idle", + slot.ready == MK_SLOT_IDLE && slot.kernel_entry == 2); + expect("publisher reserves idle", + !mk_slot_begin(&slot) && slot.ready == MK_SLOT_BUSY); + expect("second publisher rejected", mk_slot_begin(&slot) == -EBUSY); + slot.ready = MK_SLOT_QUARANTINED; + expect("quarantine rejects a publisher", mk_slot_begin(&slot) == -EIO); + pool.arch.slot = &slot; + { + struct mk_spawn_context remote = { .ready = MK_SLOT_BUSY }; + + slot.ready = MK_SLOT_IDLE; + outcome = MK_SLOT_PENDING; + ret = mk_slot_wake(&remote, 17, "remote timeout"); + expect("remote uncertainty pins local park resources", + ret == -ETIMEDOUT && + remote.ready == MK_SLOT_QUARANTINED && + slot.ready == MK_SLOT_QUARANTINED); + } + printf("Mailbox writer: %u checks, %u failures\n", checks, failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/multikernel/mailbox_test.sh b/tools/testing/selftests/multikernel/mailbox_test.sh new file mode 100644 index 00000000000000..fe8afccee52325 --- /dev/null +++ b/tools/testing/selftests/multikernel/mailbox_test.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-mailbox.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^#define MK_SLOT_/ { print }' "$src/arch/x86/include/asm/multikernel.h" \ + > "$build/mailbox-states.h" +awk '/^static (int|bool|void) mk_slot_(begin|idle|quarantine)\(/ { copy = 1 } + /^(void|int) mk_set_spawn_context\(/ { copy = 1 } + /^static int mk_slot_wake\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/arch/x86/multikernel/spawn.c" > "$build/mailbox-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror \ + -Wno-unused-parameter -Wno-unused-function -Wno-unused-variable \ + -I"$build" "$test_dir/mailbox_test.c" -o "$build/mailbox_test" +"$build/mailbox_test" diff --git a/tools/testing/selftests/multikernel/memory_teardown_test.c b/tools/testing/selftests/multikernel/memory_teardown_test.c new file mode 100644 index 00000000000000..69ad68d77b516a --- /dev/null +++ b/tools/testing/selftests/multikernel/memory_teardown_test.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real pool/metadata release; architecture release and allocator injected. */ +#include +#include +#include +#include +#include +#include +#include + +#define u64 __u64 +#define MK_STATE_READY 1 +#define MK_STATE_ACTIVE 3 +#define MK_STATE_FAILED 4 +#define MK_CTRL_BLOCK_SIZE 4096 +#define pr_debug(...) do { } while (0) +#define pr_info(...) do { if (0) printf(__VA_ARGS__); } while (0) +#define pr_err(...) do { } while (0) +#define pr_warn(...) do { } while (0) +struct resource { u64 start, end; void *parent; char *name; }; +struct mk_memory_region { struct list_head list; struct resource res; }; +struct mk_pci_device { struct list_head list; }; +struct mk_platform_device { struct list_head list; }; +struct mk_dt_config { int unused; }; +struct mk_instance { + int id, state, region_count; + int resource_uncertain; + const char *name; + void *instance_pool, *ctrl_va, *kimage, *cpus_on_slot; + u64 ctrl_phys; + size_t pool_size, ctrl_used; + struct list_head memory_regions, pci_devices, platform_devices; + int pci_device_count, platform_device_count; + bool pci_devices_valid, platform_devices_valid; +}; +static struct mk_instance *mk_self; +#define mk_cpu_set_empty(p) (!(p)) +static int arch_error, unmap_error; +static unsigned int frees, removals, pools, controls, unmaps, checks, failures; +static unsigned int arch_calls; +#define resource_size(p) ((p)->end - (p)->start + 1) +static int mk_arch_release_instance(struct mk_instance *p) +{ + (void)p; + arch_calls++; + return arch_error; +} +static void remove_resource(struct resource *p) { (void)p; removals++; } +static void kfree(void *p) +{ + if (p) { + frees++; + free(p); + } +} +static void mk_instance_mem_free(struct mk_instance *p, void *ptr, size_t size) +{ + assert(ptr == p->ctrl_va && size == MK_CTRL_BLOCK_SIZE); + assert(unmaps == 1); + controls++; +} +static int mk_pool_unmap(u64 start, size_t size) +{ + assert(start == 0x1000 && size == MK_CTRL_BLOCK_SIZE && !controls); + unmaps++; + return unmap_error; +} +static void multikernel_destroy_instance_pool(void *p) { assert(p); pools++; } +static unsigned int reservations; +static int mk_instance_reserve_memory(struct mk_instance *p, const struct mk_dt_config *c) +{ + (void)p; + (void)c; + reservations++; + return 0; +} +#define mk_instance_reserve_cpus mk_instance_reserve_memory +#define mk_instance_reserve_pci_devices mk_instance_reserve_memory +#define mk_instance_reserve_platform_devices mk_instance_reserve_memory +#include "memory-production.h" + +static void expect(const char *name, int got, int want) +{ + checks++; + if (got != want) { + fprintf(stderr, "FAIL %s: %d != %d\n", name, got, want); + failures++; + } +} + +int main(void) +{ + struct mk_instance p; + struct mk_memory_region *region; + unsigned int i; + int control, ret; + struct mk_dt_config config = { 0 }; + struct mk_pci_device *pci; + + /* The running host has no reserved memory at first baseline admission. */ + for (i = 0; i < 10; i++) { + struct mk_memory_region owned = { 0 }; + + p = (struct mk_instance) { .state = MK_STATE_READY }; + INIT_LIST_HEAD(&p.memory_regions); + INIT_LIST_HEAD(&p.pci_devices); + INIT_LIST_HEAD(&p.platform_devices); + mk_self = i == 9 ? NULL : &p; + if (i == 1) + p.state = MK_STATE_ACTIVE; + if (i == 2) + p.state = MK_STATE_FAILED; + if (i == 3) + p.kimage = &p; + if (i == 4) + p.resource_uncertain = 1; + if (i == 5) + p.instance_pool = &p; + if (i == 6) + p.ctrl_va = &control; + if (i == 7) + list_add(&owned.list, &p.memory_regions); + if (i == 8) + p.cpus_on_slot = &p; + arch_error = -EBUSY; + arch_calls = frees = removals = pools = controls = unmaps = 0; + expect("baseline admission", mk_baseline_clear_resources(&p), + i == 0 ? 0 : -EBUSY); + expect("only owned or child memory needs stop proof", arch_calls, i >= 5); + expect("running host loses no allocation", frees + removals + pools + controls, 0); + } + mk_self = NULL; + + for (i = 0; i < 7; i++) { + p = (struct mk_instance) { .id = 7, .name = "spawn", .state = MK_STATE_READY, + .instance_pool = &p, .pool_size = 4096, .ctrl_va = &control, + .ctrl_phys = 0x1000, .ctrl_used = 64, .region_count = 1 }; + INIT_LIST_HEAD(&p.memory_regions); + region = calloc(1, sizeof(*region)); + assert(region); + region->res = (struct resource) { .start = 0x1000, .end = 0x1fff, + .parent = &p, .name = strdup("memory") }; + assert(region->res.name); + list_add(®ion->list, &p.memory_regions); + frees = removals = pools = controls = unmaps = 0; + arch_error = i == 0 ? -EBUSY : 0; + unmap_error = i == 6 ? -EIO : 0; + if (i == 1) + p.state = MK_STATE_ACTIVE; + if (i == 2) + p.state = MK_STATE_FAILED; + if (i == 3) + p.kimage = &p; + if (i == 5) + p.resource_uncertain = 1; + ret = mk_instance_free_memory(&p); + expect("release status", ret, i == 4 ? 0 : i == 6 ? -EIO : -EBUSY); + if (i != 4) { + expect("unsafe release retains allocation", p.instance_pool != NULL, 1); + expect("unsafe release retains region metadata", p.region_count, 1); + expect("unsafe release retains control address", p.ctrl_va == &control, 1); + expect("unsafe release retains control accounting", p.ctrl_used, 64); + expect("unsafe release detaches nothing", frees + removals + pools + controls, 0); + expect("only stopped memory reaches unmap", unmaps, i == 6); + /* Dispose only fixture objects, including the baseline's broken case. */ + if (!frees) { + free(region->res.name); + free(region); + } + } else { + expect("confirmed release returns pool", p.instance_pool == NULL, 1); + expect("confirmed release clears metadata", list_empty(&p.memory_regions), 1); + expect("confirmed release frees metadata", frees, 2); + expect("confirmed release returns control allocation", controls, 1); + expect("confirmed release returns pool once", pools, 1); + } + } + /* Re-reservation and baseline replacement must not bypass unmap refusal. */ + p = (struct mk_instance) { .state = MK_STATE_READY, .ctrl_va = &control, + .ctrl_phys = 0x1000, .instance_pool = &p, .pci_device_count = 1, + .pci_devices_valid = true }; + INIT_LIST_HEAD(&p.memory_regions); + INIT_LIST_HEAD(&p.pci_devices); + INIT_LIST_HEAD(&p.platform_devices); + pci = calloc(1, sizeof(*pci)); + assert(pci); + list_add(&pci->list, &p.pci_devices); + arch_error = 0; + unmap_error = -EIO; + frees = controls = 0; + expect("re-reservation propagates failure", mk_instance_reserve_resources(&p, &config), -EIO); + expect("no replacement reservation", reservations, 0); + expect("baseline propagates failure", mk_baseline_clear_resources(&p), -EIO); + expect("baseline device metadata retained", !list_empty(&p.pci_devices), 1); + expect("baseline count retained", p.pci_device_count, 1); + expect("baseline no free on error", frees, 0); + free(pci); + printf("Memory teardown: %u checks, %u failures\n", checks, failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/multikernel/memory_teardown_test.sh b/tools/testing/selftests/multikernel/memory_teardown_test.sh new file mode 100644 index 00000000000000..33f45f6f30c94c --- /dev/null +++ b/tools/testing/selftests/multikernel/memory_teardown_test.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-memory-teardown.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +if [ -n "${MK_CORE_REV:-}" ]; then + revision=$(git -C "$src" rev-parse --verify "$MK_CORE_REV^{commit}") + git -C "$src" show "$revision:kernel/multikernel/core.c" > "$build/core-input.c" + input="$build/core-input.c" +else + input="$src/kernel/multikernel/core.c" +fi +awk '/^(void|int) mk_instance_(free_memory|reserve_resources)\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$input" > "$build/memory-production.h" +awk '/^static (void|int) mk_baseline_clear_resources\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/baseline.c" >> "$build/memory-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -I"$build" \ + -I"$src/tools/include" "$test_dir/memory_teardown_test.c" -o "$build/memory_teardown_test" +"$build/memory_teardown_test" diff --git a/tools/testing/selftests/multikernel/messaging_registry_test.c b/tools/testing/selftests/multikernel/messaging_registry_test.c new file mode 100644 index 00000000000000..58d68c2405b671 --- /dev/null +++ b/tools/testing/selftests/multikernel/messaging_registry_test.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Extracted messaging code; pthread-controlled, non-native IRQ/RCU boundary. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef uint32_t u32; +typedef uint64_t u64; +#define __init +#define GFP_KERNEL 0 +#define EXPORT_SYMBOL(x) +#define pr_warn(...) do { } while (0) +#define pr_err(...) do { } while (0) +#define pr_info(...) do { } while (0) +#define pr_debug(...) do { } while (0) +#define WARN_ON_ONCE(x) (!!(x)) +static _Thread_local bool interrupt_context; +#define in_interrupt() interrupt_context +static _Thread_local unsigned int role; +static pthread_mutex_t gate = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t changed = PTHREAD_COND_INITIALIZER; +static bool blocked, backend_first, backend_second, hold_register, release_register; +static bool callback_entered, release_callback, grace_entered; +static bool retirement_done, register_done, early_free; +static unsigned int live_handlers, readers; +static int allocation_fail_after = -1; +static void *entered_context; + +typedef pthread_mutex_t raw_spinlock_t; +#define __RAW_SPIN_LOCK_UNLOCKED(n) PTHREAD_MUTEX_INITIALIZER +#define DEFINE_MUTEX(n) pthread_mutex_t n = PTHREAD_MUTEX_INITIALIZER +static void writer_lock(pthread_mutex_t *lock) +{ + int ret = pthread_mutex_trylock(lock); + + if (ret == EBUSY) { + assert(!pthread_mutex_lock(&gate)); + if (role == 2) { + blocked = true; + assert(!pthread_cond_broadcast(&changed)); + } + assert(!pthread_mutex_unlock(&gate)); + ret = pthread_mutex_lock(lock); + } + assert(!ret); +} +#define mutex_lock(m) writer_lock(m) +#define mutex_unlock(m) assert(!pthread_mutex_unlock(m)) +#define raw_spin_lock_irqsave(m, flags) \ + do { (flags) = 0; writer_lock(m); } while (0) +#define raw_spin_unlock_irqrestore(m, flags) \ + do { (void)(flags); mutex_unlock(m); } while (0) + +static void *kzalloc(size_t size, int flags) +{ + (void)flags; + if (allocation_fail_after == 0) + return NULL; + if (allocation_fail_after > 0) + allocation_fail_after--; + return calloc(1, size); +} + +static void kfree(void *ptr) +{ + assert(!pthread_mutex_lock(&gate)); + if (ptr == entered_context && readers) + early_free = true; + assert(!pthread_mutex_unlock(&gate)); + free(ptr); +} + +struct mk_message { u32 msg_type, msg_subtype; u64 msg_id; u32 payload_len; char payload[]; }; +struct mk_ipi_data { u32 type; size_t data_size; void *buffer; u64 sender_cpu; }; +typedef void (*mk_msg_handler_t)(u32, u32, u64, void *, u32, void *); +struct mk_ipi_handler { + void (*callback)(struct mk_ipi_data *, void *); + void *context; +}; + +static struct mk_ipi_handler *multikernel_register_handler( + void (*callback)(struct mk_ipi_data *, void *), void *ctx, unsigned int type) +{ + struct mk_ipi_handler *handler = kzalloc(sizeof(*handler), GFP_KERNEL); + + (void)type; + if (!handler) + return NULL; + handler->callback = callback; + handler->context = ctx; + assert(!pthread_mutex_lock(&gate)); + if (role == 1) { + backend_first = true; + assert(!pthread_cond_broadcast(&changed)); + } + if (role == 2) { + backend_second = true; + assert(!pthread_cond_broadcast(&changed)); + } + while (role == 1 && hold_register && !release_register) + assert(!pthread_cond_wait(&changed, &gate)); + live_handlers++; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + return handler; +} + +static void multikernel_unregister_handler(struct mk_ipi_handler *handler) +{ + assert(!interrupt_context); + assert(!pthread_mutex_lock(&gate)); + grace_entered = true; + assert(!pthread_cond_broadcast(&changed)); + /* Modeled unlink/grace period, not a native RCU implementation. */ + while (readers) + assert(!pthread_cond_wait(&changed, &gate)); + assert(live_handlers); + live_handlers--; + assert(!pthread_mutex_unlock(&gate)); + kfree(handler); +} + +#include "registry-production.h" + +static int failures; +#define CHECK(expr, text) do { \ + if (!(expr)) { \ + fprintf(stderr, "FAIL: %s\n", text); \ + failures++; \ + } \ +} while (0) + +static void callback(u32 type, u32 subtype, u64 id, void *payload, u32 len, void *ctx) +{ + assert(type == 17 && subtype == 2 && id == 3 && !payload && !len && ctx == &readers); + assert(!pthread_mutex_lock(&gate)); + callback_entered = true; + assert(!pthread_cond_broadcast(&changed)); + while (!release_callback) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!pthread_mutex_unlock(&gate)); +} + +struct operation { int result; bool cleanup; }; +static void *register_thread(void *arg) +{ + struct operation *op = arg; + + role = op->cleanup ? 1 : 2; + op->result = mk_register_msg_handler(17, callback, &readers); + assert(!pthread_mutex_lock(&gate)); + register_done = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + return NULL; +} + +static void *retire_thread(void *arg) +{ + struct operation *op = arg; + + if (op->cleanup) + mk_messaging_cleanup(); + else + op->result = mk_unregister_msg_handler(17, callback); + assert(!pthread_mutex_lock(&gate)); + retirement_done = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + return NULL; +} + +static void *dispatch_thread(void *arg) +{ + struct mk_ipi_handler *handler = arg; + struct mk_message msg = { .msg_type = 17, .msg_subtype = 2, .msg_id = 3 }; + struct mk_ipi_data data = { .type = 17, .data_size = sizeof(msg), .buffer = &msg }; + + assert(!pthread_mutex_lock(&gate)); + readers++; + entered_context = handler->context; + assert(!pthread_mutex_unlock(&gate)); + interrupt_context = true; + handler->callback(&data, handler->context); + interrupt_context = false; + assert(!pthread_mutex_lock(&gate)); + readers--; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + return NULL; +} + +static void duplicate_case(void) +{ + pthread_t first, second; + struct operation a = { .cleanup = true }, b = { 0 }; + + assert(!mk_messaging_init()); + hold_register = true; + assert(!pthread_create(&first, NULL, register_thread, &a)); + /* First writer reached backend registration, before production insertion. */ + assert(!pthread_mutex_lock(&gate)); + while (!backend_first) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_create(&second, NULL, register_thread, &b)); + assert(!pthread_mutex_lock(&gate)); + while (!blocked && !backend_second) + assert(!pthread_cond_wait(&changed, &gate)); + release_register = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_join(first, NULL)); + assert(!pthread_join(second, NULL)); + CHECK(a.result == 0 && b.result == -EEXIST && live_handlers == 1, + "same-type concurrent registration must have exactly one success"); + mk_messaging_cleanup(); +} + +static void retirement_case(bool cleanup) +{ + pthread_t receive, retire, insert; + struct operation removal = { .cleanup = cleanup }, registration = { 0 }; + bool overlapped; + + assert(!mk_messaging_init()); + assert(!mk_register_msg_handler(17, callback, &readers)); + assert(!pthread_create(&receive, NULL, dispatch_thread, mk_msg_type_handlers->ipi_handler)); + assert(!pthread_mutex_lock(&gate)); + while (!callback_entered) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_create(&retire, NULL, retire_thread, &removal)); + assert(!pthread_mutex_lock(&gate)); + while (!grace_entered) + assert(!pthread_cond_wait(&changed, &gate)); + CHECK(!retirement_done && readers == 1 && !early_free, + "entered callback context must survive retirement"); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_create(&insert, NULL, register_thread, ®istration)); + assert(!pthread_mutex_lock(&gate)); + while (!blocked && !backend_second && !register_done) + assert(!pthread_cond_wait(&changed, &gate)); + overlapped = backend_second; + release_callback = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_join(receive, NULL)); + assert(!pthread_join(retire, NULL)); + assert(!pthread_join(insert, NULL)); + CHECK(!overlapped && !early_free, "new same-type registration must wait for old callback retirement"); + if (cleanup) { + CHECK(registration.result == -ESHUTDOWN && !mk_msg_type_handlers && !live_handlers, + "cleanup must close admission and leave no surviving registration"); + CHECK(mk_register_msg_handler(18, callback, NULL) == -ESHUTDOWN, + "registration after completed cleanup must fail"); + } else { + CHECK(removal.result == 0 && registration.result == 0 && live_handlers == 1, + "registration may succeed after unregister has drained the callback"); + } + mk_messaging_cleanup(); +} + +static void cleanup_case(void) { retirement_case(true); } +static void unregister_case(void) { retirement_case(false); } + +static void *second_cleanup_thread(void *arg) +{ + role = 2; + return retire_thread(arg); +} + +static void concurrent_cleanup_case(void) +{ + pthread_t receive, first, second; + struct operation a = { .cleanup = true }, b = { .cleanup = true }; + + assert(!mk_messaging_init()); + assert(!mk_register_msg_handler(17, callback, &readers)); + assert(!pthread_create(&receive, NULL, dispatch_thread, mk_msg_type_handlers->ipi_handler)); + assert(!pthread_mutex_lock(&gate)); + while (!callback_entered) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_create(&first, NULL, retire_thread, &a)); + assert(!pthread_mutex_lock(&gate)); + while (!grace_entered) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_create(&second, NULL, second_cleanup_thread, &b)); + assert(!pthread_mutex_lock(&gate)); + while (!blocked && !retirement_done) + assert(!pthread_cond_wait(&changed, &gate)); + CHECK(!retirement_done, "every concurrent cleanup must wait for callback retirement"); + release_callback = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_join(receive, NULL)); + assert(!pthread_join(first, NULL)); + assert(!pthread_join(second, NULL)); + CHECK(!live_handlers && !mk_msg_type_handlers && !early_free, + "concurrent cleanup retires each handler exactly once"); +} + +static void admission_case(void) +{ + CHECK(!mk_register_msg_handler(17, callback, NULL), + "early PCI handler registration must succeed"); + mk_messaging_cleanup(); + CHECK(mk_messaging_init() < 0, "cleanup must prevent a later init reopening admission"); + CHECK(mk_register_msg_handler(18, callback, NULL) == -ESHUTDOWN, + "initialization cannot reopen cleaned-up messaging"); + mk_messaging_cleanup(); +} + +static void early_init_case(void) +{ + struct mk_msg_type_handler *early; + + CHECK(!mk_register_msg_handler(17, callback, &readers), + "arch initcall can register before messaging subsys init"); + early = mk_msg_type_handlers; + CHECK(!mk_messaging_init(), "normal messaging initialization succeeds"); + CHECK(early && mk_msg_type_handlers == early && live_handlers == 1, + "initialization preserves the early handler and its context"); + CHECK(mk_register_msg_handler(17, callback, NULL) == -EEXIST, + "early handler still excludes duplicate registration after init"); + mk_messaging_cleanup(); + CHECK(!mk_msg_type_handlers && !live_handlers, "cleanup retires early handlers"); +} + +static void errors_case(void) +{ + assert(!mk_messaging_init()); + CHECK(mk_messaging_init() < 0, "initialization must open admission only once"); + CHECK(mk_register_msg_handler(17, NULL, NULL) == -EINVAL, "null registration handler"); + CHECK(mk_unregister_msg_handler(17, NULL) == -EINVAL, "null removal handler"); + CHECK(mk_unregister_msg_handler(17, callback) == -ENOENT, "missing handler"); + allocation_fail_after = 0; + CHECK(mk_register_msg_handler(17, callback, NULL) == -ENOMEM, "entry allocation failure"); + allocation_fail_after = 1; + CHECK(mk_register_msg_handler(17, callback, NULL) == -ENOMEM, "IPI allocation failure"); + allocation_fail_after = -1; + CHECK(!mk_register_msg_handler(17, callback, &readers), "retry after allocation failure"); + CHECK(mk_register_msg_handler(17, callback, NULL) == -EEXIST, "duplicate handler"); + CHECK(!mk_unregister_msg_handler(17, callback) && !live_handlers, "successful retirement"); + mk_messaging_cleanup(); +} + +static void context_case(void) +{ + assert(!mk_messaging_init()); + assert(!mk_register_msg_handler(17, callback, &readers)); + interrupt_context = true; + CHECK(mk_register_msg_handler(18, callback, NULL) == -EWOULDBLOCK, + "interrupt-context registration must be rejected"); + CHECK(mk_unregister_msg_handler(17, callback) == -EWOULDBLOCK, + "interrupt-context unregister must be rejected"); + mk_messaging_cleanup(); + CHECK(live_handlers == 1 && mk_msg_type_handlers, + "interrupt-context cleanup must leave the live registry untouched"); + interrupt_context = false; + assert(!mk_unregister_msg_handler(17, callback)); + mk_messaging_cleanup(); +} + +int main(void) +{ + void (*cases[])(void) = { duplicate_case, cleanup_case, unregister_case, + concurrent_cleanup_case, admission_case, early_init_case, errors_case, context_case }; + unsigned int i, failed = 0; + + for (i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + int status; + pid_t child = fork(); + + assert(child >= 0); + if (!child) { + alarm(10); + cases[i](); + exit(failures ? 1 : 0); + } + assert(waitpid(child, &status, 0) == child); + if (!WIFEXITED(status) || WEXITSTATUS(status)) + failed++; + } + if (failed) { + fprintf(stderr, "Messaging registry: %u/8 scenarios failed\n", failed); + return 1; + } + puts("Messaging registry: 8 scenarios PASS (non-native IRQ/RCU boundaries)"); + return 0; +} diff --git a/tools/testing/selftests/multikernel/messaging_registry_test.sh b/tools/testing/selftests/multikernel/messaging_registry_test.sh new file mode 100755 index 00000000000000..e7320e39808e9a --- /dev/null +++ b/tools/testing/selftests/multikernel/messaging_registry_test.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-messaging-registry.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^struct mk_msg_type_handler \{/ { copy = 1 } + /^static struct mk_msg_type_handler \*mk_msg_type_handlers;/ { print } + /^static raw_spinlock_t mk_msg_type_handlers_lock/ { print } + /^static DEFINE_MUTEX\(mk_msg_type_handlers_lock\);/ { print } + /^static bool mk_messaging_/ { print } + /^static void mk_message_type_ipi_callback\(/ { copy = 1 } + /^int mk_(un)?register_msg_handler\(/ { copy = 1 } + /^int __init mk_messaging_init\(/ { copy = 1 } + /^void mk_messaging_cleanup\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/messaging.c" \ + > "$build/registry-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -pthread -Wall -Wextra -Werror \ + -I"$build" "$test_dir/messaging_registry_test.c" -o "$build/messaging_registry_test" +"$build/messaging_registry_test" diff --git a/tools/testing/selftests/multikernel/mktty_init_test.c b/tools/testing/selftests/multikernel/mktty_init_test.c new file mode 100644 index 00000000000000..af2dd78dfdaabd --- /dev/null +++ b/tools/testing/selftests/multikernel/mktty_init_test.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real initialization/unwind/cleanup; substitute registration APIs only. */ +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint64_t u64; +typedef int raw_spinlock_t; +#define pr_info(...) \ + do { \ + } while (0) +#define pr_warn(...) \ + do { \ + } while (0) +#define TTY_DRIVER_RESET_TERMIOS 1 +#define TTY_DRIVER_REAL_RAW 2 +#define TTY_DRIVER_TYPE_CONSOLE 3 +#define B115200 1 +#define CS8 2 +#define CREAD 4 +#define HUPCL 8 +#define CLOCAL 16 +#define IS_ERR(p) ((intptr_t)(p) < 0) +#define PTR_ERR(p) ((intptr_t)(p)) +#define raw_spin_lock_init(p) (*(p) = 0) +#define raw_spin_lock_irqsave(p, flags) \ + do { \ + assert(!*(p)); \ + *(p) = 1; \ + (flags) = 0; \ + } while (0) +#define raw_spin_unlock_irqrestore(p, flags) \ + do { \ + assert(*(p)); \ + *(p) = 0; \ + (void)(flags); \ + } while (0) +struct tty_port { + void *ops; +}; +struct termios { + unsigned int c_cflag; +} tty_std_termios; +struct tty_driver { + const char *driver_name, *name; + int type; + struct termios init_termios; +} driver; +struct mk_ipi_handler { + int unused; +} handler; +struct mk_ipi_data; +struct { + int id; +} self, *mk_self = &self; +static int mktty_host_miscdev, mktty_spawn_console, mktty_spawn_ops, + mktty_spawn_port_ops; +static bool fail_handler, fail_register, fail_alloc, misc_live, handler_live; +static bool allocated, registered, port_live, console_live, drain_expected, + drained; +static int misc_register(void *p) +{ + misc_live = true; + return 0; +} +static void misc_deregister(void *p) +{ + assert(misc_live); + misc_live = false; +} +static void mktty_host_ipi_handler(struct mk_ipi_data *p, void *ctx) +{ +} +static void mktty_spawn_ipi_handler(struct mk_ipi_data *p, void *ctx) +{ +} +static struct mk_ipi_handler * +multikernel_register_handler(void (*fn)(struct mk_ipi_data *, void *), + void *ctx, unsigned int type) +{ + if (fail_handler) + return NULL; + handler_live = true; + return &handler; +} +static void multikernel_unregister_handler(struct mk_ipi_handler *h) +{ + assert(h == &handler && handler_live); + handler_live = false; +} +static struct tty_driver *tty_alloc_driver(int count, int flags) +{ + if (fail_alloc) + return (void *)(intptr_t)-ENOMEM; + allocated = true; + return &driver; +} +static void tty_port_init(struct tty_port *p) +{ + port_live = true; +} +static void tty_port_destroy(struct tty_port *p) +{ + assert(port_live && !handler_live && (!drain_expected || drained)); + port_live = false; +} +static void tty_set_operations(struct tty_driver *d, void *ops) +{ +} +static void tty_port_link_device(struct tty_port *p, struct tty_driver *d, + int n) +{ +} +static int tty_register_driver(struct tty_driver *d) +{ + if (fail_register) + return -EIO; + registered = true; + return 0; +} +static void tty_unregister_driver(struct tty_driver *d) +{ + assert(registered); + registered = false; +} +static void tty_driver_kref_put(struct tty_driver *d) +{ + assert(allocated && !registered); + allocated = false; +} +static void register_console(void *c) +{ + assert(!console_live); + console_live = true; +} +static void unregister_console(void *c) +{ + assert(console_live); + console_live = false; +} +static void mktty_spawn_shutdown(struct tty_port *p) +{ + assert(!handler_live); + drained = true; +} +#include "init-production.h" +static int failures; +#define CHECK(c, msg) \ + do { \ + if (!(c)) { \ + fprintf(stderr, "FAIL: %s\n", msg); \ + failures++; \ + } \ + } while (0) +static void host_failure(void) +{ + fail_handler = true; + CHECK(mktty_host_init() == -ENOMEM, + "host handler failure must fail init"); + CHECK(!misc_live && !handler_live, + "host init failure must undo misc registration"); +} +static void spawn_failure(void) +{ + self.id = 1; + fail_handler = true; + console_live = mktty_console_registered = + true; /* independent early console */ + CHECK(mktty_spawn_init() == -ENOMEM, + "spawn handler failure must fail init"); + CHECK(!allocated && !registered && !port_live && !mktty_spawn_driver, + "spawn handler failure must undo driver and port publication"); + CHECK(console_live && mktty_console_registered, + "failed late init must preserve existing early console"); +} +static void earlier_failure(void) +{ + self.id = 1; + fail_alloc = true; + CHECK(mktty_spawn_init() == -ENOMEM && !allocated && !port_live, + "allocation failure must not publish port"); + fail_alloc = false; + fail_register = true; + CHECK(mktty_spawn_init() == -EIO && !allocated && !registered && + !port_live, + "driver registration failure must unwind allocation and port"); +} +static void cleanup_case(void) +{ + CHECK(!mktty_host_init(), "host init success"); + mktty_host_cleanup(); + CHECK(!misc_live && !handler_live, + "host cleanup must unregister both interfaces"); + self.id = 1; + CHECK(!mktty_spawn_init(), "spawn init success"); + drain_expected = true; + mktty_spawn.rx_enabled = true; + mktty_spawn_cleanup(); + CHECK(mktty_spawn.stopping && !mktty_spawn.rx_enabled && drained && + !port_live && !registered && !allocated && !console_live, + "spawn cleanup must close admission and drain before destroying port"); +} +int main(void) +{ + void (*cases[])(void) = { host_failure, spawn_failure, earlier_failure, + cleanup_case }; + unsigned int i, bad = 0; + for (i = 0; i < 4; i++) { + int status; + pid_t child = fork(); + assert(child >= 0); + if (!child) { + cases[i](); + exit(failures != 0); + } + assert(waitpid(child, &status, 0) == child); + bad += !WIFEXITED(status) || WEXITSTATUS(status); + } + printf("mktty init: %u/4 scenarios %s (substituted registration APIs)\n", + bad ? bad : 4, bad ? "FAIL" : "PASS"); + return bad != 0; +} diff --git a/tools/testing/selftests/multikernel/mktty_init_test.sh b/tools/testing/selftests/multikernel/mktty_init_test.sh new file mode 100644 index 00000000000000..4942fcb32a11a8 --- /dev/null +++ b/tools/testing/selftests/multikernel/mktty_init_test.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +ulimit -c 0 +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-mktty-init.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^#define MKTTY_/ { print } + /^struct mktty_spawn_state \{/ { copy = 1 } + /^static (struct mk_ipi_handler \*mktty_(host|spawn)_handler;|struct tty_driver \*mktty_spawn_driver;|struct mktty_spawn_state mktty_spawn;|bool mktty_console_registered;)/ { print } + /^static (int|void) mktty_(host|spawn)_(init|cleanup)\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/drivers/tty/mktty.c" > "$build/init-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -Wno-unused-parameter \ + -I"$build" "$test_dir/mktty_init_test.c" -o "$build/mktty_init_test" +"$build/mktty_init_test" diff --git a/tools/testing/selftests/multikernel/mktty_lifetime_test.c b/tools/testing/selftests/multikernel/mktty_lifetime_test.c new file mode 100644 index 00000000000000..85e5091f220731 --- /dev/null +++ b/tools/testing/selftests/multikernel/mktty_lifetime_test.c @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Extracted host mktty paths; controlled, non-native IRQ/scheduler boundaries. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef uint32_t u32; +typedef uint64_t u64; +typedef unsigned int __poll_t; +typedef int poll_table; +#define __user +#define MK_MAX_DATA_SIZE 32 +#define GFP_KERNEL 0 +#define pr_info(...) \ + do { \ + } while (0) +#define pr_warn_ratelimited(...) \ + do { \ + } while (0) +#define min(a, b) ((a) < (b) ? (a) : (b)) +#define kzalloc(n, flags) calloc(1, n) +#define kmalloc(n, flags) malloc(n) +#define copy_from_user(dst, src, n) (memcpy(dst, src, n), 0) +#define put_user(c, p) (*(p) = (c), 0) +static atomic_uint snapshots; +#define READ_ONCE(p) \ + (atomic_fetch_add(&snapshots, 1), \ + __atomic_load_n(&(p), __ATOMIC_RELAXED)) +#define WRITE_ONCE(p, v) \ + do { \ + assert(holds_connections && irq_disabled); \ + __atomic_store_n(&(p), (v), __ATOMIC_RELAXED); \ + } while (0) + +struct list_head { + struct list_head *next, *prev; +}; +#define LIST_HEAD(n) struct list_head n = { &n, &n } +#define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) +#define list_for_each_entry(p, h, m) \ + for (struct list_head *cursor = (h)->next; \ + cursor != (h) && \ + ((p) = container_of(cursor, __typeof__(*(p)), m), 1); \ + cursor = cursor->next) +static void list_add(struct list_head *node, struct list_head *head) +{ + node->next = head->next; + node->prev = head; + head->next->prev = node; + head->next = node; +} +static void list_del(struct list_head *node) +{ + node->prev->next = node->next; + node->next->prev = node->prev; +} + +typedef pthread_mutex_t spinlock_t; +#define DEFINE_SPINLOCK(n) spinlock_t n = PTHREAD_MUTEX_INITIALIZER +static spinlock_t *connections_lock; +static _Thread_local bool irq_disabled, in_callback, holds_connections; +static bool inject_irq; +static unsigned int irq_hazards; +static pthread_mutex_t gate = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t changed = PTHREAD_COND_INITIALIZER; +static bool callback_entered, release_callback, removal_blocked; +static bool pause_callback, race_selection; +static unsigned int selecting; +static atomic_uint freed; +static unsigned int unsafe_locks, unsafe_wakes, wakes; +static bool pause_worker, worker_entered, release_worker, flush_entered; + +static void test_lock(spinlock_t *lock) +{ + int ret = pthread_mutex_trylock(lock); + + if (ret == EBUSY) { + assert(!pthread_mutex_lock(&gate)); + if (lock == connections_lock && !in_callback) { + removal_blocked = true; + assert(!pthread_cond_broadcast(&changed)); + } + assert(!pthread_mutex_unlock(&gate)); + ret = pthread_mutex_lock(lock); + } + assert(!ret); + if (lock == connections_lock) { + holds_connections = true; + /* A same-CPU hardIRQ at this checkpoint would reacquire this lock. */ + if (inject_irq && !irq_disabled && !in_callback) + irq_hazards++; + } +} +static void __attribute__((unused)) spin_lock(spinlock_t *lock) +{ + if (in_callback) + unsafe_locks++; + test_lock(lock); +} +static void spin_unlock(spinlock_t *lock) +{ + if (lock == connections_lock) + holds_connections = false; + assert(!pthread_mutex_unlock(lock)); +} +#define spin_lock_init(p) assert(!pthread_mutex_init(p, NULL)) +#define spin_lock_irqsave(p, flags) \ + do { \ + (flags) = irq_disabled; \ + irq_disabled = true; \ + spin_lock(p); \ + } while (0) +#define spin_unlock_irqrestore(p, flags) \ + do { \ + spin_unlock(p); \ + irq_disabled = (flags); \ + } while (0) +#define lockdep_assert_held(p) \ + assert(holds_connections && (p) == connections_lock) +typedef spinlock_t raw_spinlock_t; +#define DEFINE_RAW_SPINLOCK(n) DEFINE_SPINLOCK(n) +#define raw_spin_lock_init(p) spin_lock_init(p) +#define raw_spin_lock(p) test_lock(p) +#define raw_spin_unlock(p) spin_unlock(p) +#define raw_spin_lock_irqsave(p, flags) \ + do { \ + (flags) = irq_disabled; \ + irq_disabled = true; \ + test_lock(p); \ + } while (0) +#define raw_spin_unlock_irqrestore(p, flags) spin_unlock_irqrestore(p, flags) + +struct work_struct { + void (*fn)(struct work_struct *); + bool pending, running; +}; +static struct work_struct *queued_work; +#define INIT_WORK(w, f) \ + do { \ + (w)->fn = (f); \ + (w)->pending = (w)->running = false; \ + } while (0) + +static void callback_pause(void) +{ + if (!pause_callback) + return; + assert(!pthread_mutex_lock(&gate)); + callback_entered = true; + assert(!pthread_cond_broadcast(&changed)); + while (!release_callback) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!atomic_load(&freed)); + assert(!pthread_mutex_unlock(&gate)); +} + +static bool __attribute__((unused)) schedule_work(struct work_struct *work) +{ + bool queued; + + if (in_callback) + callback_pause(); + assert(!pthread_mutex_lock(&gate)); + queued = !work->pending; + work->pending = true; + queued_work = work; + assert(!pthread_mutex_unlock(&gate)); + return queued; +} + +static void run_work(struct work_struct *work) +{ + assert(!pthread_mutex_lock(&gate)); + assert(work->pending && !work->running); + work->pending = false; + work->running = true; + assert(!pthread_mutex_unlock(&gate)); + work->fn(work); + assert(!pthread_mutex_lock(&gate)); + work->running = false; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); +} + +static void __attribute__((unused)) flush_work(struct work_struct *work) +{ + assert(!holds_connections && !irq_disabled && !in_callback); + assert(!pthread_mutex_lock(&gate)); + flush_entered = true; + assert(!pthread_cond_broadcast(&changed)); + while (work->running) + assert(!pthread_cond_wait(&changed, &gate)); + while (work->pending) { + assert(!pthread_mutex_unlock(&gate)); + run_work(work); + assert(!pthread_mutex_lock(&gate)); + } + queued_work = NULL; + assert(!pthread_mutex_unlock(&gate)); +} + +typedef struct { + int initialized; +} wait_queue_head_t; +#define init_waitqueue_head(p) ((p)->initialized = 1) +#define wait_event_interruptible(w, condition) ((condition) ? 0 : -EINTR) +static void wake_up_interruptible(wait_queue_head_t *wait) +{ + assert(wait->initialized); + wakes++; + if (in_callback || holds_connections || irq_disabled) + unsafe_wakes++; + if (in_callback) + callback_pause(); + if (!pause_worker) + return; + assert(!pthread_mutex_lock(&gate)); + worker_entered = true; + assert(!pthread_cond_broadcast(&changed)); + while (!release_worker) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!atomic_load(&freed)); + assert(!pthread_mutex_unlock(&gate)); +} + +struct inode { + int unused; +}; +struct file { + void *private_data; + int f_flags; +}; +static void poll_wait(struct file *file, wait_queue_head_t *wait, + poll_table *table) +{ + assert(!holds_connections && wait->initialized); +} + +static void kfree(void *ptr) +{ + assert(!holds_connections); + if (ptr) + atomic_fetch_add(&freed, 1); + free(ptr); +} + +static int kstrtoint(const char *str, int base, int *value) +{ + char *end; + long parsed = strtol(str, &end, base); + + assert(!holds_connections && !irq_disabled); + if (end == str || (*end && strcmp(end, "\n"))) + return -EINVAL; + *value = parsed; + return 0; +} + +struct mk_instance { + int id; +}; +static struct mk_instance instances[] = { { 1 }, { 2 } }; +static struct mk_instance *mk_instance_find(int id) +{ + assert(!holds_connections && !irq_disabled); + if (id < 1 || id > 2) + return NULL; + if (race_selection) { + assert(!pthread_mutex_lock(&gate)); + selecting++; + assert(!pthread_cond_broadcast(&changed)); + while (selecting < 2) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!pthread_mutex_unlock(&gate)); + } + return &instances[id - 1]; +} +static void mk_instance_put(struct mk_instance *instance) +{ + assert(instance && !holds_connections); +} + +struct mk_ipi_data { + void *buffer; + size_t data_size; +}; +static unsigned int sends; +static int expected_target; +static int multikernel_send_ipi_data(int id, void *buf, size_t size, u32 type) +{ + assert(id == expected_target && !holds_connections && !irq_disabled); + assert(buf && size >= 16 && type); + sends++; + return 0; +} + +#include "mktty-production.h" + +static int failures; +#define CHECK(expr, text) \ + do { \ + if (!(expr)) { \ + fprintf(stderr, "FAIL: %s\n", text); \ + failures++; \ + } \ + } while (0) + +static void deliver(int id) +{ + struct mktty_message msg = { .type = MKTTY_MSG_OUTPUT, + .console_id = id, + .len = 1, + .data = { 'x' } }; + struct mk_ipi_data data = { .buffer = &msg, .data_size = 17 }; + + in_callback = true; + irq_disabled = true; + mktty_host_ipi_handler(&data, NULL); + irq_disabled = false; + in_callback = false; +} + +static void irq_case(void) +{ + struct file file = { 0 }; + + inject_irq = true; + assert(!mktty_host_open(NULL, &file)); + assert(!mktty_host_release(NULL, &file)); + CHECK(!irq_hazards, + "open/release list locks must exclude same-CPU hardIRQ reentry"); + CHECK(atomic_load(&freed) == 2 && + mktty_host_conns.next == &mktty_host_conns, + "release must unlink before freeing connection and RX buffer"); +} + +static void *receive_thread(void *unused) +{ + deliver(1); + return NULL; +} +static void *release_thread(void *file) +{ + mktty_host_release(NULL, file); + return NULL; +} + +static void lifetime_case(void) +{ + struct file file = { 0 }; + pthread_t receive, remove; + + assert(!mktty_host_open(NULL, &file)); + assert(mktty_host_write(&file, "1\n", 2, NULL) == 2); + pause_callback = true; + assert(!pthread_create(&receive, NULL, receive_thread, NULL)); + assert(!pthread_mutex_lock(&gate)); + while (!callback_entered) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_create(&remove, NULL, release_thread, &file)); + assert(!pthread_mutex_lock(&gate)); + while (!removal_blocked) + assert(!pthread_cond_wait(&changed, &gate)); + CHECK(!atomic_load(&freed), + "release must wait for callback's connection use"); + release_callback = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_join(receive, NULL)); + assert(!pthread_join(remove, NULL)); + CHECK(atomic_load(&freed) == 2, + "release frees only after callback unlocks"); + pause_callback = false; + deliver(1); + CHECK(atomic_load(&freed) == 2, + "callback after release must find no connection"); +} + +struct selection { + struct file *file; + const char *id; + ssize_t result; +}; +static void *select_thread(void *arg) +{ + struct selection *selection = arg; + + selection->result = + mktty_host_write(selection->file, selection->id, 2, NULL); + return NULL; +} + +static void target_case(void) +{ + struct file file = { 0 }; + struct selection a = { .file = &file, .id = "1\n" }; + struct selection b = { .file = &file, .id = "2\n" }; + pthread_t first, second; + struct mktty_host_conn *conn; + char input[MKTTY_MAX_DATA + 1] = { 'a' }, output; + unsigned int reads_before; + + assert(!mktty_host_open(NULL, &file)); + race_selection = true; + assert(!pthread_create(&first, NULL, select_thread, &a)); + assert(!pthread_create(&second, NULL, select_thread, &b)); + assert(!pthread_join(first, NULL)); + assert(!pthread_join(second, NULL)); + race_selection = false; + conn = file.private_data; + CHECK((a.result == 2 && b.result == -EISCONN && + conn->instance_id == 1) || + (b.result == 2 && a.result == -EISCONN && + conn->instance_id == 2), + "competing first writes must publish one immutable connection target"); + expected_target = conn->instance_id; + reads_before = snapshots; + CHECK(mktty_host_write(&file, input, sizeof(input), NULL) == + sizeof(input) && + sends == 2, + "all data-write chunks use the selected target"); + CHECK(snapshots == reads_before + 1, + "data write must snapshot target once"); + deliver(expected_target); + reads_before = snapshots; + CHECK(mktty_host_poll(&file, NULL) & EPOLLIN, + "poll observes received byte"); + CHECK(mktty_host_read(&file, &output, 1, NULL) == 1 && output == 'x', + "read receives callback byte"); + CHECK(snapshots == reads_before + 2, + "read/poll must use marked target reads"); + assert(!mktty_host_release(NULL, &file)); +} + +static void validation_case(void) +{ + struct file file = { .f_flags = O_NONBLOCK }; + char output; + + assert(!mktty_host_open(NULL, &file)); + CHECK(mktty_host_read(&file, &output, 1, NULL) == -ENOTCONN, + "unconnected read"); + CHECK(mktty_host_poll(&file, NULL) == (EPOLLOUT | EPOLLWRNORM), + "unconnected poll"); + CHECK(mktty_host_write(&file, "0\n", 2, NULL) == -EINVAL, + "invalid target"); + CHECK(mktty_host_write(&file, "3\n", 2, NULL) == -ENODEV, + "missing target"); + CHECK(((struct mktty_host_conn *)file.private_data)->instance_id == -1, + "failed selection leaves connection unpublished"); + CHECK(mktty_host_write(&file, "1\n", 2, NULL) == 2, + "selection after failure"); + CHECK(mktty_host_read(&file, &output, 1, NULL) == -EAGAIN, + "connected nonblocking read"); + assert(!mktty_host_release(NULL, &file)); +} + +static void deferred_case(void) +{ + struct file file = { 0 }; + + assert(!mktty_host_open(NULL, &file)); + assert(mktty_host_write(&file, "1\n", 2, NULL) == 2); + deliver(1); + deliver(1); + CHECK(!unsafe_locks, "hard IPI must use raw locks only"); + CHECK(!wakes, "IPI must queue wakeup, not execute it"); + assert(!mktty_host_release(NULL, &file)); + CHECK(wakes == 1 && !unsafe_wakes, + "coalesced wakeup must run outside IRQ/raw locks before free"); +} + +static void truncated_case(void) +{ + struct file file = { .f_flags = O_NONBLOCK }; + struct mktty_message msg = { .type = MKTTY_MSG_OUTPUT, + .console_id = 1, + .len = 2, + .data = { 'a', 'b' } }; + struct mk_ipi_data data = { .buffer = &msg, .data_size = 17 }; + char output; + + assert(!mktty_host_open(NULL, &file)); + assert(mktty_host_write(&file, "1\n", 2, NULL) == 2); + mktty_host_ipi_handler(&data, NULL); + CHECK(mktty_host_read(&file, &output, 1, NULL) == -EAGAIN, + "declared payload beyond received bytes must not enter RX buffer"); + assert(!mktty_host_release(NULL, &file)); +} + +static void consumed_case(void) +{ + struct file file = { 0 }; + char output; + + assert(!mktty_host_open(NULL, &file)); + assert(mktty_host_write(&file, "1\n", 2, NULL) == 2); + deliver(1); + assert(mktty_host_read(&file, &output, 1, NULL) == 1 && output == 'x'); + assert(!mktty_host_release(NULL, &file)); + CHECK(!wakes, + "worker must check RX state under its lock before waking readers"); +} + +static void *worker_thread(void *work) +{ + run_work(work); + return NULL; +} + +static void worker_close_case(void) +{ + struct file file = { 0 }; + pthread_t worker, closer; + + assert(!mktty_host_open(NULL, &file)); + assert(mktty_host_write(&file, "1\n", 2, NULL) == 2); + deliver(1); + CHECK(queued_work, "receive must publish deferred work"); + if (!queued_work) { + mktty_host_release(NULL, &file); + return; + } + pause_worker = true; + assert(!pthread_create(&worker, NULL, worker_thread, queued_work)); + assert(!pthread_mutex_lock(&gate)); + while (!worker_entered) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!pthread_mutex_unlock(&gate)); + /* A second receive while running needs another wake, not lost work. */ + deliver(1); + assert(!pthread_create(&closer, NULL, release_thread, &file)); + assert(!pthread_mutex_lock(&gate)); + while (!flush_entered) + assert(!pthread_cond_wait(&changed, &gate)); + CHECK(!atomic_load(&freed), + "close must retain object while wake worker runs"); + CHECK(mktty_host_conns.next == &mktty_host_conns, + "unlink must precede worker drain"); + release_worker = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_join(worker, NULL)); + assert(!pthread_join(closer, NULL)); + CHECK(wakes == 2 && !unsafe_wakes && atomic_load(&freed) == 2, + "close must drain running and requeued wake before freeing"); +} + +int main(void) +{ + void (*cases[])(void) = { irq_case, lifetime_case, + target_case, validation_case, + deferred_case, truncated_case, + worker_close_case, consumed_case }; + unsigned int i, failed = 0; + + connections_lock = &mktty_host_conns_lock; + for (i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + int status; + pid_t child = fork(); + + assert(child >= 0); + if (!child) { + alarm(10); + cases[i](); + exit(failures ? 1 : 0); + } + assert(waitpid(child, &status, 0) == child); + if (!WIFEXITED(status) || WEXITSTATUS(status)) + failed++; + } + if (failed) { + fprintf(stderr, "mktty lifetime: %u/8 scenarios failed\n", + failed); + return 1; + } + puts("mktty lifetime: 8 scenarios PASS (non-native IRQ/scheduler boundaries)"); + return 0; +} diff --git a/tools/testing/selftests/multikernel/mktty_lifetime_test.sh b/tools/testing/selftests/multikernel/mktty_lifetime_test.sh new file mode 100755 index 00000000000000..637d7d12227a4b --- /dev/null +++ b/tools/testing/selftests/multikernel/mktty_lifetime_test.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-mktty-lifetime.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^#define MKTTY_/ { print } + /^struct mktty_(message|host_conn) \{/ { copy = 1 } + /^static LIST_HEAD\(mktty_host_conns\);/ { print } + /^static DEFINE_(RAW_)?SPINLOCK\(mktty_host_conns_lock\);/ { print } + /^static struct mktty_host_conn \*mktty_find_conn\(/ { copy = 1 } + /^static (int|ssize_t|__poll_t|void) mktty_host_(open|release|read|write|poll|ipi_handler|wake_work)\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/drivers/tty/mktty.c" \ + > "$build/mktty-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -pthread -Wall -Wextra -Werror \ + -Wno-unused-parameter -Wno-sign-compare \ + -I"$build" "$test_dir/mktty_lifetime_test.c" \ + -o "$build/mktty_lifetime_test" +"$build/mktty_lifetime_test" diff --git a/tools/testing/selftests/multikernel/mktty_spawn_test.c b/tools/testing/selftests/multikernel/mktty_spawn_test.c new file mode 100644 index 00000000000000..05cf43fdfa34f4 --- /dev/null +++ b/tools/testing/selftests/multikernel/mktty_spawn_test.c @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual MKTTY RX/worker functions; deterministic kernel API boundaries. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef uint32_t u32; +typedef uint64_t u64; +typedef unsigned char u8; +typedef int spinlock_t; +typedef int raw_spinlock_t; +#define MK_MAX_DATA_SIZE 4096 +#define min_t(t, a, b) ((t)(a) < (t)(b) ? (t)(a) : (t)(b)) +static bool in_irq, irq_disabled; +static unsigned int raw_depth, bad_context, calls, refs; +static unsigned long long reported_drops; +static unsigned char delivered[65536]; +static size_t delivered_size, insert_limit = 65536; +static size_t flip_pending; +static void (*insert_hook)(void); +#define raw_spin_lock_init(p) (*(p) = 0) +#define raw_spin_lock_irqsave(p, flags) \ + do { \ + assert(!*(p)); \ + *(p) = 1; \ + raw_depth++; \ + (flags) = irq_disabled; \ + irq_disabled = true; \ + } while (0) +#define raw_spin_unlock_irqrestore(p, flags) \ + do { \ + assert(*(p)); \ + *(p) = 0; \ + raw_depth--; \ + irq_disabled = (flags); \ + } while (0) +#define pr_warn_ratelimited(fmt, count) \ + do { \ + assert(!in_irq && !raw_depth); \ + reported_drops += (count); \ + } while (0) + +/* kfifo is a boundary, not an alternative implementation of the driver. */ +struct fifo { + unsigned char data[16384]; + unsigned int in, out; +}; +#define DEFINE_KFIFO(name, type, size) struct fifo name +#define kfifo_len(f) ((f)->in - (f)->out) +#define kfifo_is_empty(f) (!kfifo_len(f)) +#define kfifo_reset(f) \ + do { \ + assert(raw_depth); \ + (f)->in = (f)->out = 0; \ + } while (0) +static unsigned int + __attribute__((unused)) kfifo_in(struct fifo *f, const void *data, + unsigned int n) +{ + unsigned int i; + assert(raw_depth); + n = min_t(unsigned int, n, sizeof(f->data) - kfifo_len(f)); + for (i = 0; i < n; i++) + f->data[f->in++ % sizeof(f->data)] = ((const u8 *)data)[i]; + return n; +} +static unsigned int __attribute__((unused)) +kfifo_out(struct fifo *f, void *data, unsigned int n) +{ + unsigned int i; + assert(raw_depth); + n = min_t(unsigned int, n, kfifo_len(f)); + for (i = 0; i < n; i++) + ((u8 *)data)[i] = f->data[f->out++ % sizeof(f->data)]; + return n; +} + +struct work_struct { + void (*fn)(struct work_struct *); + bool pending, running; +}; +#define DECLARE_WORK(name, func) struct work_struct name = { .fn = func } +static struct work_struct *queued; +static unsigned int runs; +static bool __attribute__((unused)) schedule_work(struct work_struct *work) +{ + bool fresh = !work->pending; + assert(raw_depth); /* enqueued before releasing admission/lifetime protection */ + work->pending = true; + queued = work; + return fresh; +} +static void __attribute__((unused)) cancel_work_sync(struct work_struct *work) +{ + assert(!in_irq && !raw_depth && !work->running); + work->pending = false; +} +static void run_one(void) +{ + struct work_struct *work = queued; + assert(work && work->pending && !work->running); + work->pending = false; + work->running = true; + runs++; + work->fn(work); + work->running = false; +} +static void drain(void) +{ + unsigned int limit = 32; + while (queued && queued->pending) { + assert(limit--); + run_one(); + } +} + +struct tty_struct { + int index; +} tty; +struct tty_port { + bool open; +}; +static void mktty_spawn_shutdown(struct tty_port *port); +static void tty_port_hangup(struct tty_port *port) +{ + port->open = false; + mktty_spawn_shutdown(port); +} +static void __attribute__((unused)) tty_ldisc_flush(struct tty_struct *t) +{ + assert(t == &tty && !refs && !in_irq && !raw_depth); + flip_pending = 0; +} +static struct tty_struct *tty_port_tty_get(struct tty_port *port) +{ + calls++; + if (in_irq || raw_depth || irq_disabled) + bad_context++; + if (!port->open) + return NULL; + refs++; + return &tty; +} +static int tty_insert_flip_string(struct tty_port *port, const void *data, + size_t size) +{ + if (in_irq || raw_depth || irq_disabled) + bad_context++; + assert(refs && port->open); + size = min_t(size_t, size, insert_limit); + assert(delivered_size + size <= sizeof(delivered)); + memcpy(delivered + delivered_size, data, size); + delivered_size += size; + flip_pending += size; + if (insert_hook) { + void (*hook)(void) = insert_hook; + insert_hook = NULL; + hook(); + } + return size; +} +static void tty_flip_buffer_push(struct tty_port *port) +{ + assert(refs); +} +static void tty_kref_put(struct tty_struct *t) +{ + assert(t == &tty && refs); + refs--; +} +struct mk_ipi_data { + void *buffer; + size_t data_size; +}; +#include "spawn-production.h" + +static int failures; +#define CHECK(c, msg) \ + do { \ + if (!(c)) { \ + fprintf(stderr, "FAIL: %s\n", msg); \ + failures++; \ + } \ + } while (0) +static void open_port(void) +{ + raw_spin_lock_init(&mktty_spawn.lock); + mktty_spawn.port.open = true; + assert(!mktty_spawn_activate(&mktty_spawn.port, &tty)); +} +static void send_bytes(unsigned int n, char value, unsigned int received) +{ + struct mktty_message msg = { .type = MKTTY_MSG_INPUT, .len = n }; + struct mk_ipi_data data = { .buffer = &msg, + .data_size = received + 16 }; + assert(n <= sizeof(msg.data)); + memset(msg.data, value, n); + in_irq = irq_disabled = true; + mktty_spawn_ipi_handler(&data, NULL); + in_irq = irq_disabled = false; + memset(msg.data, '!', n); /* transport immediately reuses its payload */ +} +static void context_case(void) +{ + open_port(); + send_bytes(3, 'a', 3); + send_bytes(2, 'b', 2); + CHECK(!calls && !delivered_size, + "hard IPI must only copy, not call TTY"); + drain(); + CHECK(!bad_context && !refs && delivered_size == 5 && + !memcmp(delivered, "aaabb", 5), + "worker must deliver owned bytes in order in process context"); + CHECK(runs == 1, "pending receives should coalesce into one worker"); +} +static void truncated_case(void) +{ + open_port(); + send_bytes(2, 'a', 1); + drain(); + CHECK(!calls && !delivered_size, + "truncated payload must not be queued"); +} +static void shutdown_case(void) +{ + open_port(); + send_bytes(3, 'x', 3); + mktty_spawn_shutdown(&mktty_spawn.port); + drain(); + CHECK(!delivered_size, + "shutdown must cancel queued old-session delivery"); + send_bytes(2, 'y', 2); + drain(); + CHECK(!delivered_size, "inactive session must not admit receive work"); + assert(!mktty_spawn_activate(&mktty_spawn.port, &tty)); + send_bytes(1, 'z', 1); + drain(); + CHECK(delivered_size == 1 && delivered[0] == 'z', + "reopen must not inherit old input"); +} +static void overflow_case(void) +{ + unsigned int i; + open_port(); + for (i = 0; i < 5; i++) + send_bytes(4000, 'q', 4000); + drain(); + CHECK(delivered_size == 16384 && reported_drops == 3616, + "FIFO must bound and account overflow"); + send_bytes(7, 'r', 7); + drain(); + CHECK(delivered_size == 16391 && + !memcmp(delivered + 16384, "rrrrrrr", 7), + "FIFO wrap must preserve later input"); +} +static void inject_receive(void) +{ + send_bytes(2, 'b', 2); +} +static void running_case(void) +{ + open_port(); + insert_hook = inject_receive; + send_bytes(3, 'a', 3); + drain(); + CHECK(delivered_size == 5 && !memcmp(delivered, "aaabb", 5) && !refs, + "receive during delivery must not lose work or references"); + insert_limit = 1; + send_bytes(4, 'c', 4); + drain(); + CHECK(delivered_size == 6 && reported_drops == 3, + "short TTY insert must account undelivered bytes"); +} +static void hangup_case(void) +{ + open_port(); + send_bytes(3, 'x', 3); + drain(); + assert(flip_pending == 3); + mktty_spawn_hangup(&tty); + CHECK(!flip_pending, + "hangup must flush bytes inserted before RX worker drainage"); + open_port(); + send_bytes(1, 'z', 1); + drain(); + CHECK(flip_pending == 1, + "new session must not inherit old flip-buffer bytes"); +} +int main(void) +{ + void (*cases[])(void) = { context_case, truncated_case, shutdown_case, + overflow_case, running_case, hangup_case }; + unsigned int i, bad = 0; + for (i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + int status; + pid_t child = fork(); + assert(child >= 0); + if (!child) { + alarm(5); + cases[i](); + exit(failures != 0); + } + assert(waitpid(child, &status, 0) == child); + bad += !WIFEXITED(status) || WEXITSTATUS(status); + } + printf("mktty spawn: %u/6 scenarios %s (substituted kernel boundaries)\n", + bad ? bad : 6, bad ? "FAIL" : "PASS"); + return bad != 0; +} diff --git a/tools/testing/selftests/multikernel/mktty_spawn_test.sh b/tools/testing/selftests/multikernel/mktty_spawn_test.sh new file mode 100644 index 00000000000000..2fa31bccb2d79b --- /dev/null +++ b/tools/testing/selftests/multikernel/mktty_spawn_test.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-mktty-spawn.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^#define MKTTY_/ { print } + /^struct mktty_(message|spawn_state) \{/ { copy = 1 } + /^static struct mktty_spawn_state mktty_spawn;/ { print } + /^static (DECLARE_WORK|DEFINE_KFIFO)\(mktty_spawn_rx/ { print } + /^static void mktty_spawn_receive_work\(struct work_struct \*work\);/ { print; next } + /^static (int|void) mktty_spawn_(activate|shutdown|hangup|ipi_handler|receive_work|report_drops)\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/drivers/tty/mktty.c" > "$build/spawn-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -Wno-unused-parameter \ + -I"$build" "$test_dir/mktty_spawn_test.c" -o "$build/mktty_spawn_test" +"$build/mktty_spawn_test" diff --git a/tools/testing/selftests/multikernel/overlay_retention_test.c b/tools/testing/selftests/multikernel/overlay_retention_test.c new file mode 100644 index 00000000000000..1a20d01b75f940 --- /dev/null +++ b/tools/testing/selftests/multikernel/overlay_retention_test.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual transaction lifetime functions; resource effects and kernfs modeled. */ +#include +#include +#include +#include +#include +#include +#include +#include + +struct kernfs_node { + void *priv; + struct kernfs_node *parent; + bool active, drained; +}; + +#include "overlay-types.h" +static struct kernfs_node root, node, *mk_overlay_root_kn = &root; +static LIST_HEAD(mk_overlay_tx_list); +static int mk_overlay_next_id; +static bool mk_overlay_mutex; +static int directory_error, files_error, apply_error, rollback_error; +static unsigned int effects, inverse_calls, frees, checks, failures; +#define pr_info(...) do { } while (0) +#define pr_err(...) do { } while (0) +#define GFP_KERNEL 0 +#define kzalloc(n, flags) ((void)(flags), calloc(1, n)) +#define atomic_inc_return(p) (++*(p)) +static void expect(const char *why, int got, int want) +{ + checks++; + if (got != want) { + fprintf(stderr, "FAIL %s: %d != %d\n", why, got, want); + failures++; + } +} + +static bool *lock(bool *p) +{ + assert(!*p); + *p = true; + return p; +} + +static void unlock(bool **p) +{ + if (*p) { + assert(**p); + **p = false; + *p = NULL; + } +} + +#define guard(kind) bool *held_##kind __attribute__((cleanup(unlock))) = lock +#define scoped_guard(kind, p) \ + for (bool *held_##kind __attribute__((cleanup(unlock))) = lock(p); \ + held_##kind; unlock(&held_##kind)) +static void *kmemdup(const void *p, size_t size, int flags) +{ + void *copy = malloc(size); + + assert(copy); + memcpy(copy, p, size); + return copy; +} + +static void kfree(void *p) +{ + if (p == node.priv) { + expect("drain readers before freeing transaction", + !node.active || node.drained, true); + node.priv = NULL; + } + frees++; + free(p); +} + +static struct kernfs_node *kernfs_create_dir(struct kernfs_node *parent, + const char *name, int mode, void *priv) +{ + assert(mk_overlay_mutex); + if (directory_error) + return ERR_PTR(directory_error); + node = (struct kernfs_node) { .priv = priv, .parent = parent }; + return &node; +} + +static void kernfs_remove(struct kernfs_node *kn) +{ + assert(!kn->active); + kn->drained = true; +} + +static void kernfs_remove_self(struct kernfs_node *kn) +{ + assert(!mk_overlay_mutex && kn->priv); + assert(((struct mk_overlay_tx *)kn->priv)->status == MK_OVERLAY_TX_REMOVED); + kn->drained = true; +} + +static void kernfs_activate(struct kernfs_node *kn) { kn->active = true; } +static struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn) { return kn->parent; } +static void kernfs_put(struct kernfs_node *kn) { } +static void mk_overlay_parse_metadata(struct mk_overlay_tx *tx, const void *blob) { } +static int mk_overlay_create_tx_files(struct mk_overlay_tx *tx) { return files_error; } +static int mk_overlay_parse_and_apply(struct mk_overlay_tx *tx, const void *blob) +{ + assert(mk_overlay_mutex); + expect("create inactive record before resource effects", + node.priv == tx && !node.active, true); + /* One resource succeeds, then the second can fail or time out. */ + effects++; + return apply_error; +} + +static int mk_overlay_parse_and_rollback(struct mk_overlay_tx *tx, const void *blob) +{ + assert(mk_overlay_mutex); + inverse_calls++; + return rollback_error; +} + +#include "overlay-production.h" + +static void reset(void) +{ + struct mk_overlay_tx *tx, *next; + + assert(!mk_overlay_mutex); + /* Dispose fixture objects only; production must retain these transactions. */ + list_for_each_entry_safe(tx, next, &mk_overlay_tx_list, list) { + list_del(&tx->list); + free(tx->dtbo_data); + free(tx); + } + node = (struct kernfs_node) { 0 }; + directory_error = 0; + files_error = 0; + apply_error = 0; + rollback_error = 0; + effects = 0; + inverse_calls = 0; + frees = 0; +} + +int main(void) +{ + struct mk_overlay_tx *tx; + int i; + + for (i = 0; i < 2; i++) { + reset(); + if (i) + files_error = -ENOMEM; + else + directory_error = -ENOMEM; + expect("metadata allocation error returned", mk_overlay_apply("test", 5), -ENOMEM); + expect("metadata allocation failure has no effects", effects, 0); + expect("unapplied record freed", frees, 2); + } + reset(); + apply_error = -ETIMEDOUT; + expect("failed transaction has inspectable ID", mk_overlay_apply("test", 5) > 0, true); + tx = node.priv; + assert(tx); + expect("failed apply remains visible", tx->status, MK_OVERLAY_TX_FAILED); + expect("failed apply cannot inverse unexecuted operations", + mk_overlay_rmdir(&node), -EBUSY); + if (node.priv) { + expect("failed transaction still retained", !list_empty(&mk_overlay_tx_list), true); + expect("repeat removal still refused", mk_overlay_rmdir(&node), -EBUSY); + } + expect("failed apply never runs inverse", inverse_calls, 0); + expect("failed apply never freed", frees, 0); + reset(); + assert(mk_overlay_apply("test", 5) > 0); + rollback_error = -EIO; + expect("partial rollback failure returned", mk_overlay_rmdir(&node), -EIO); + assert(node.priv); + expect("partial rollback retained", ((struct mk_overlay_tx *)node.priv)->status, + MK_OVERLAY_TX_FAILED); + rollback_error = 0; + expect("partial inverse cannot run again", mk_overlay_rmdir(&node), -EBUSY); + expect("partial inverse only attempted once", inverse_calls, 1); + expect("failed rollback never freed", frees, 0); + reset(); + assert(mk_overlay_apply("test", 5) > 0); + expect("normal removal succeeds", mk_overlay_rmdir(&node), 0); + expect("normal removal inverses once", inverse_calls, 1); + expect("normal removal drains and frees", frees, 2); + expect("normal removal unlinks transaction", list_empty(&mk_overlay_tx_list), true); + reset(); + printf("Overlay retention: %u checks, %u failures (resource effects/kernfs modeled)\n", + checks, failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/multikernel/overlay_retention_test.sh b/tools/testing/selftests/multikernel/overlay_retention_test.sh new file mode 100644 index 00000000000000..80f2f74122face --- /dev/null +++ b/tools/testing/selftests/multikernel/overlay_retention_test.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-overlay-retention.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^enum mk_overlay_tx_status \{/ { copy = 1 } + /^struct mk_overlay_tx \{/ { copy = 1 } + copy { print } + copy && /^};/ { copy = 0 }' "$src/kernel/multikernel/overlay.c" > "$build/overlay-types.h" +awk '/^static int mk_overlay_apply\(/ { copy = 1 } + /^int mk_overlay_rmdir\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/overlay.c" > "$build/overlay-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror \ + -Wno-unused-parameter -I"$build" -I"$src/tools/include" \ + "$test_dir/overlay_retention_test.c" -o "$build/overlay_retention_test" +"$build/overlay_retention_test" diff --git a/tools/testing/selftests/multikernel/pciback_lifetime_test.c b/tools/testing/selftests/multikernel/pciback_lifetime_test.c new file mode 100644 index 00000000000000..ece9c76176619c --- /dev/null +++ b/tools/testing/selftests/multikernel/pciback_lifetime_test.c @@ -0,0 +1,587 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual request/retirement functions; PCI, workqueue and RCU boundaries modeled. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef unsigned int u32; +#define __init +typedef unsigned short u16; +typedef unsigned char u8; +#include "pciback-types.h" +#define GFP_ATOMIC 0 +#define PCIBIOS_DEVICE_NOT_FOUND 0x86 +static _Thread_local bool in_receive; +static bool allocation_fails; +static unsigned int irq_allocator_calls; +static __maybe_unused void *test_alloc(size_t n) +{ + irq_allocator_calls += in_receive; + return allocation_fails ? NULL : calloc(1, n); +} +static void test_free(void *p) +{ + irq_allocator_calls += in_receive; + free(p); +} +#define kzalloc(n, flags) ((void)(flags), test_alloc(n)) +#define kfree test_free +#define lockdep_assert_held(p) ((void)(p)) + +struct mk_shared_data; +struct mk_instance { + int id; + atomic_int refs; + struct mk_shared_data *ipi_data; +}; +struct pci_bus { int number; }; +struct pci_dev { struct pci_bus *bus; int devfn, dev; }; +struct work_struct { void (*fn)(struct work_struct *); }; +struct mk_pciback_dev { + struct list_head list; + atomic_int ref; + struct mk_instance *instance; + struct pci_dev *pdev; + pthread_mutex_t io_lock; + bool revoked; +}; +static LIST_HEAD(mk_pciback_devs); +static pthread_mutex_t mk_pciback_lock = PTHREAD_MUTEX_INITIALIZER; +static struct mk_instance old = { .id = 7, .refs = 1 }; +static struct mk_instance replacement = { .id = 7, .refs = 1 }; +static struct mk_instance self = { .id = 0, .refs = 1 }; +static struct mk_instance *mk_self = &self, *lookup = &old, *answer_to; +static struct pci_bus bus = { 2 }; +static struct pci_dev pdev = { .bus = &bus, .devfn = 3 }; +static struct work_struct *queued; +static struct work_struct *jobs[MK_IPI_RING_SIZE + 2]; +static unsigned int nr_jobs; +static bool nested = true; +static pthread_mutex_t gate = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t changed = PTHREAD_COND_INITIALIZER; +static bool hold_access, entered, release_access, retiring, io_active, reset_during_io; +static _Thread_local bool retire_thread; +static unsigned int accesses, resets, failures, checks; +static u32 config_value; + +static void mutex_lock(pthread_mutex_t *p) +{ + if (retire_thread && p != &mk_pciback_lock) { + assert(!pthread_mutex_lock(&gate)); + retiring = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + } + assert(!pthread_mutex_lock(p)); +} +static void mutex_unlock(pthread_mutex_t *p) { assert(!pthread_mutex_unlock(p)); } +static pthread_mutex_t *lock(pthread_mutex_t *p) { mutex_lock(p); return p; } +static void unlock(pthread_mutex_t **p) +{ + if (*p) { + mutex_unlock(*p); + *p = NULL; + } +} +#define guard(kind) pthread_mutex_t *held_##kind __attribute__((cleanup(unlock))) = lock +#define scoped_guard(kind, p) \ + for (pthread_mutex_t *scope __attribute__((cleanup(unlock))) = lock(p); scope; unlock(&scope)) +#define INIT_WORK(w, f) ((w)->fn = (f)) +static void schedule_work(struct work_struct *w) +{ + assert(nr_jobs < MK_IPI_RING_SIZE + 2); + jobs[nr_jobs++] = w; + queued = jobs[0]; +} +static struct mk_instance *mk_instance_find(int id) +{ + if (!lookup || lookup->id != id) + return NULL; + lookup->refs++; + return lookup; +} +static void mk_instance_get(struct mk_instance *p) { p->refs++; } +static void mk_instance_put(struct mk_instance *p) { assert(p->refs > 1); p->refs--; } +#define refcount_inc(p) (++*(p)) +#define refcount_dec_and_test(p) (!--*(p)) +#define pm_runtime_put(p) ((void)(p)) +#define pci_dev_put(p) ((void)(p)) +#define pci_domain_nr(p) 0 +#define mk_manifest_phys() nested +static void mk_send_message_to(struct mk_instance *p, int type, int subtype, + void *data, size_t len) +{ + answer_to = p; +} +static void mk_pciback_do(struct mk_pciback_dev *d, u32 subtype, + struct mk_pci_cfg_payload *req) +{ + assert(!pthread_mutex_lock(&gate)); + accesses++; + config_value = req->val; + req->result = 0; + io_active = entered = true; + assert(!pthread_cond_broadcast(&changed)); + while (hold_access && !release_access) + assert(!pthread_cond_wait(&changed, &gate)); + io_active = false; + assert(!pthread_mutex_unlock(&gate)); +} +static int pci_reset_function(struct pci_dev *p) +{ + assert(!pthread_mutex_lock(&gate)); + resets++; + reset_during_io |= io_active; + retiring = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + return 0; +} +#define pci_clear_master(p) ((void)(p)) +static u64 answer_seq; +static struct mk_pci_cfg_payload sync_answer; +static void mk_request_answer(u64 seq, const void *data, u32 len) +{ + assert(len == sizeof(sync_answer)); + assert((uintptr_t)data % _Alignof(struct mk_pci_cfg_payload) == 0); + answer_seq = seq; + memcpy(&sync_answer, data, len); +} +static int register_error; +static bool handler_registered; +static int mk_register_msg_handler(u32 type, + void (*fn)(u32, u32, u64, void *, u32, void *), void *ctx) +{ + handler_registered = !register_error; + return register_error; +} +#include "pciback-production.h" + +static void receive(const struct mk_pci_cfg_payload *req, u32 subtype) +{ + in_receive = true; + mk_pciback_request(req, subtype); + in_receive = false; +} +#define mk_pciback_request receive + +#define CHECK(cond, why) do { \ + checks++; \ + if (!(cond)) { \ + failures++; \ + fprintf(stderr, "FAIL: %s\n", why); \ + } \ +} while (0) +static struct mk_pci_cfg_payload request = { .sender_instance_id = 7, .bus = 2, .devfn = 3 }; +static void lend(struct mk_instance *p) +{ + struct mk_pciback_dev *d = calloc(1, sizeof(*d)); + + assert(d); + d->ref = 1; + d->instance = p; + d->pdev = &pdev; + assert(!pthread_mutex_init(&d->io_lock, NULL)); + mk_instance_get(p); + list_add(&d->list, &mk_pciback_devs); +} +static void *run_work(void *arg) +{ + struct work_struct *work = queued; + + assert(work); + for (unsigned int i = 1; i < nr_jobs; i++) + jobs[i - 1] = jobs[i]; + nr_jobs--; + queued = nr_jobs ? jobs[0] : NULL; + work->fn(work); + return NULL; +} +static void *retire(void *arg) +{ + retire_thread = true; + mk_pciback_unlend(&old); + return NULL; +} + +#define MK_MAX_DATA_SIZE 64 +#define pr_warn_once(...) do { } while (0) +#define rcu_read_lock() do { } while (0) +#define rcu_read_unlock() do { } while (0) +#define rcu_dereference(p) (p) +#define atomic_read(p) (*(p)) +/* Controlled arrivals model publication; this fixture does not test barriers. */ +#define smp_load_acquire(p) (*(p)) +#define this_cpu_read(p) (p) +#define this_cpu_write(p, v) ((p) = (v)) +#define raw_spin_lock_irqsave(m, flags) do { (flags) = 0; mutex_lock(m); } while (0) +#define raw_spin_unlock_irqrestore(m, flags) do { (void)(flags); mutex_unlock(m); } while (0) +struct mk_ipi_data { + size_t data_size; + unsigned long type; + char buffer[MK_MAX_DATA_SIZE]; +}; + +struct mk_shared_data { + struct { + unsigned int head, tail; + struct mk_ipi_data entries[MK_IPI_RING_SIZE]; + } ring; +}; + +typedef void (*mk_ipi_callback_t)(struct mk_ipi_data *, void *); +struct mk_ipi_handler { + unsigned long ipi_type; + mk_ipi_callback_t callback; + void *context; + struct mk_ipi_handler *next; +}; + +static struct mk_shared_data inbox; +static struct mk_ipi_handler *mk_handlers; +static pthread_mutex_t mk_drain_lock = PTHREAD_MUTEX_INITIALIZER; +static _Thread_local bool mk_draining; +static unsigned int peer_callbacks, releases, append_valid, append_invalid; +static int recursive_result; +static bool reenter; + +static void enqueue(unsigned long type, size_t size) +{ + unsigned int head = inbox.ring.head; + unsigned int next = (head + 1) % MK_IPI_RING_SIZE; + struct mk_ipi_data *slot = &inbox.ring.entries[head]; + + assert(next != inbox.ring.tail && !slot->data_size); + slot->type = type; + memcpy(slot->buffer, &request, sizeof(request)); + slot->data_size = size; + inbox.ring.head = next; +} + +static void atomic_set_release(unsigned int *p, unsigned int value) +{ + assert(p == &inbox.ring.tail); + assert(!inbox.ring.entries[*p].data_size); + *p = value; + releases++; + if (append_invalid) { + append_invalid--; + enqueue(9, MK_MAX_DATA_SIZE + 1); + } +} + +bool mk_ipi_in_drain(void); +#include "pciback-ring.h" +#include "pciback-poll-api.h" +static int poll_result(void) +{ +#ifdef POLL_HAS_RESULT + return mk_ipi_poll(); +#else + mk_ipi_poll(); + return 0; +#endif +} + +static void pci_message(struct mk_ipi_data *slot, void *ctx) +{ + struct mk_pci_cfg_payload req; + + assert(mk_draining); + memcpy(&req, slot->buffer, sizeof(req)); + mk_pciback_request(&req, MK_PCI_CFG_WRITE); +} + +static void peer_message(struct mk_ipi_data *slot, void *ctx) +{ + assert(mk_draining); + peer_callbacks++; + if (append_valid) { + append_valid--; + enqueue(9, 1); + } + if (reenter) + recursive_result = poll_result(); +} + +static void reset_ring(void) +{ + static struct mk_ipi_handler peer = { .ipi_type = 9, .callback = peer_message }; + static struct mk_ipi_handler pci = { + .ipi_type = MK_MSG_PCI, .callback = pci_message, .next = &peer, + }; + + assert(!mk_draining && !queued && list_empty(&mk_pciback_devs)); + memset(&inbox, 0, sizeof(inbox)); + self.ipi_data = &inbox; + mk_handlers = &pci; + peer_callbacks = 0; + releases = 0; + append_valid = 0; + append_invalid = 0; + reenter = false; + answer_to = NULL; + accesses = 0; + resets = 0; + nested = true; +} + +static void ring_retirement(void) +{ + reset_ring(); + lookup = &old; + lend(&old); + enqueue(MK_MSG_PCI, sizeof(request)); + enqueue(9, 1); + CHECK(!poll_result(), "queued old PCI request dispatched before retirement"); + CHECK(queued && peer_callbacks == 1, "retirement preserves another peer's message"); + mk_pciback_unlend(&old); + lookup = &replacement; + lend(&replacement); + run_work(NULL); + CHECK(!accesses && !answer_to, "old ring request cannot reach replacement grant or inbox"); + mk_pciback_unlend(&replacement); + + reset_ring(); + lookup = &old; + lend(&old); + enqueue(9, 0); + enqueue(MK_MSG_PCI, sizeof(request)); + CHECK(poll_result() == -EBUSY, "unpublished slot must refuse retirement"); + CHECK(!releases && !queued && !resets, "blocked prefix must not skip or retire a grant"); + CHECK(inbox.ring.head == 2 && !inbox.ring.tail, "blocked shared ring remains intact"); + inbox.ring.entries[0].data_size = 1; + CHECK(!poll_result(), "late publication allows bounded retry"); + CHECK(peer_callbacks == 1 && queued, "late publication preserves FIFO dispatch"); + mk_pciback_unlend(&old); + run_work(NULL); + CHECK(!accesses && !answer_to, "retired late work cannot execute or reply"); + + reset_ring(); + enqueue(9, 1); + enqueue(9, 0); + enqueue(9, 1); + CHECK(poll_result() == -EBUSY && releases == 1 && peer_callbacks == 1, + "completed prefix does not hide a later unpublished slot"); + CHECK(inbox.ring.tail == 1 && inbox.ring.head == 3, + "partial drain retains blocked and following slots"); + inbox.ring.entries[1].data_size = 1; + CHECK(!poll_result() && releases == 3 && peer_callbacks == 3, + "retry does not dispatch the completed prefix twice"); + + reset_ring(); + inbox.ring.head = MK_IPI_RING_SIZE - 2; + inbox.ring.tail = MK_IPI_RING_SIZE - 2; + enqueue(9, 1); + enqueue(9, 1); + enqueue(9, 1); + CHECK(!poll_result() && inbox.ring.tail == 1 && peer_callbacks == 3, + "snapshot retirement handles wraparound"); + + reset_ring(); + enqueue(9, 1); + enqueue(9, 1); + append_valid = MK_IPI_RING_SIZE * 2; + CHECK(!poll_result() && peer_callbacks == 2 && releases == 2, + "new peer traffic cannot extend the retirement snapshot"); + CHECK(inbox.ring.tail != inbox.ring.head, "new peer messages remain queued"); + append_valid = 0; + CHECK(!poll_result() && peer_callbacks == 4 && releases == 4, + "next drain processes new peer traffic exactly once"); + + reset_ring(); + enqueue(9, MK_MAX_DATA_SIZE + 1); + enqueue(9, MK_MAX_DATA_SIZE + 1); + append_invalid = MK_IPI_RING_SIZE * 2; + CHECK(!poll_result() && releases == 2 && !peer_callbacks, + "malformed peer traffic also obeys the snapshot bound"); + +#ifdef POLL_HAS_RESULT + reset_ring(); + enqueue(9, 1); + reenter = true; + CHECK(!poll_result() && recursive_result == -EBUSY, + "callback reentry refuses recursive consumer locking"); +#endif + reset_ring(); + self.ipi_data = NULL; + CHECK(!poll_result(), "no initialized inbox has nothing to retire"); + CHECK(old.refs == 1 && replacement.refs == 1, "ring dispatch pins are released"); +} + +static void packed_messages(void) +{ + _Alignas(struct mk_pci_cfg_payload) unsigned char wire[sizeof(request) + 1]; + struct mk_pci_cfg_payload req = request; + + req.seq = 0x123456789abcdef0ULL; + req.val = 0x89abcdef; + memcpy(wire + 1, &req, sizeof(req)); + mk_pciback_msg_handler(MK_MSG_PCI, MK_PCI_CFG_ACK, 0, + wire + 1, sizeof(req), NULL); + CHECK(answer_seq == req.seq && sync_answer.val == req.val, + "packed ACK preserves full sequence and value"); + answer_seq = 0; + mk_pciback_msg_handler(MK_MSG_PCI, MK_PCI_CFG_ACK, 0, + wire + 1, sizeof(req) - 1, NULL); + CHECK(!answer_seq, "short ACK does not complete a request"); + lend(&old); + mk_pciback_msg_handler(MK_MSG_PCI, MK_PCI_CFG_WRITE, 0, + wire + 1, sizeof(req), NULL); + memset(wire, 0, sizeof(wire)); + run_work(NULL); + CHECK(config_value == req.val, "packed request is copied before callback return"); + mk_pciback_unlend(&old); + accesses = 0; + answer_to = NULL; +} + +static void saturation(void) +{ + struct work_struct *first; + struct mk_pciback_dev *d; + unsigned int i; + + assert(!queued && !nr_jobs && list_empty(&mk_pciback_devs)); + nested = true; + lookup = &old; + lend(&old); + d = mk_pciback_find(7, 0, 2, 3); + for (i = 0; i < MK_IPI_RING_SIZE; i++) { + request.val = i + 1; + mk_pciback_request(&request, MK_PCI_CFG_WRITE); + } + CHECK(nr_jobs == MK_IPI_RING_SIZE && d->ref == MK_IPI_RING_SIZE + 1, + "full pool pins every admitted grant"); + answer_to = NULL; + request.val = 0xffffffff; + mk_pciback_request(&request, MK_PCI_CFG_WRITE); + CHECK(nr_jobs == MK_IPI_RING_SIZE && d->ref == MK_IPI_RING_SIZE + 1 && !answer_to, + "overflow neither overwrites work nor pins or acknowledges"); + first = queued; + run_work(NULL); + CHECK(config_value == 1, "overflow preserves oldest request"); + mk_pciback_request(&request, MK_PCI_CFG_WRITE); + CHECK(jobs[MK_IPI_RING_SIZE - 1] == first && first->fn == mk_pciback_work_fn, + "released slot reuses its initialized callback on the same queue"); + for (i = 1; i < MK_IPI_RING_SIZE; i++) { + run_work(NULL); + CHECK(config_value == i + 1, "admitted request copy remains intact"); + } + run_work(NULL); + CHECK(config_value == request.val && d->ref == 1, "reused slot releases its pin"); + mk_pciback_unlend(&old); + + lend(&old); + for (i = 0; i < MK_IPI_RING_SIZE; i++) + mk_pciback_request(&request, MK_PCI_CFG_WRITE); + mk_pciback_unlend(&old); + lookup = &replacement; + lend(&replacement); + accesses = 0; + answer_to = NULL; + mk_pciback_request(&request, MK_PCI_CFG_WRITE); + CHECK(nr_jobs == MK_IPI_RING_SIZE && !answer_to, + "replacement cannot steal a revoked but occupied slot"); + while (nr_jobs) + run_work(NULL); + CHECK(!accesses && !answer_to && old.refs == 1, + "saturated revoked work cannot access or answer through replacement"); + mk_pciback_request(&request, MK_PCI_CFG_WRITE); + run_work(NULL); + CHECK(accesses == 1 && answer_to == &replacement, + "replacement gets a fresh request only after capacity returns"); + mk_pciback_unlend(&replacement); + CHECK(replacement.refs == 1 && !irq_allocator_calls, + "all pins retired and no receive allocator calls"); +} + +int main(void) +{ + pthread_t worker, reclaimer; + + register_error = -EIO; + CHECK(mk_pciback_init() == -EIO && !handler_registered, + "early registration error is preserved"); + register_error = 0; + assert(!mk_pciback_init()); + packed_messages(); + + lend(&old); + allocation_fails = true; + mk_pciback_request(&request, MK_PCI_CFG_WRITE); + CHECK(queued, "deferred admission survives allocator exhaustion"); + CHECK(!irq_allocator_calls, "deferred receive must not call allocator"); + allocation_fails = false; + if (queued) + run_work(NULL); + mk_pciback_unlend(&old); + accesses = 0; + answer_to = NULL; + + /* A queued request cannot acquire a replacement grant with the same ID/BDF. */ + lend(&old); + mk_pciback_request(&request, MK_PCI_CFG_WRITE); + mk_pciback_unlend(&old); + lookup = &replacement; + lend(&replacement); + run_work(NULL); + CHECK(!accesses, "retired queued request must not access a replacement grant"); + CHECK(!answer_to, "revoked work must not send an ACK into a later boot"); + mk_pciback_unlend(&replacement); + CHECK(old.refs == 1 && replacement.refs == 1, "queued pins released exactly once"); + + /* Negative replies have no config work and never queue an ID lookup. */ + lookup = &old; + mk_pciback_request(&request, MK_PCI_CFG_WRITE); + lookup = &replacement; + CHECK(!queued && answer_to == &old, "negative reply must not defer an instance ID lookup"); + + lookup = &old; + lend(&old); + hold_access = true; + entered = retiring = false; + mk_pciback_request(&request, MK_PCI_CFG_WRITE); + assert(!pthread_create(&worker, NULL, run_work, NULL)); + assert(!pthread_mutex_lock(&gate)); + while (!entered) + assert(!pthread_cond_wait(&changed, &gate)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_create(&reclaimer, NULL, retire, NULL)); + assert(!pthread_mutex_lock(&gate)); + while (!retiring) + assert(!pthread_cond_wait(&changed, &gate)); + CHECK(!reset_during_io, "device reset must wait for admitted config access"); + release_access = true; + assert(!pthread_cond_broadcast(&changed)); + assert(!pthread_mutex_unlock(&gate)); + assert(!pthread_join(worker, NULL)); + assert(!pthread_join(reclaimer, NULL)); + CHECK(old.refs == 1 && list_empty(&mk_pciback_devs), "retirement releases all request pins"); + /* Deferred and interrupt-context writes must preserve the request's value. */ + hold_access = false; + request.val = 0x1234abcd; + lend(&old); + mk_pciback_request(&request, MK_PCI_CFG_WRITE); + run_work(NULL); + CHECK(config_value == request.val, "deferred config write preserves its value"); + nested = false; + request.val = 0x4321dcba; + mk_pciback_request(&request, MK_PCI_CFG_WRITE); + CHECK(!queued && config_value == request.val, "direct config write preserves its value"); + mk_pciback_unlend_one(&old, 0, 2, 3); + CHECK(old.refs == 1 && list_empty(&mk_pciback_devs), "single-device retirement releases pins"); + ring_retirement(); + saturation(); + if (failures) + return 1; + printf("PCI/ring lifetime: %u checks, 0 failures (config, IRQ and workqueue modeled)\n", + checks); + return 0; +} diff --git a/tools/testing/selftests/multikernel/pciback_lifetime_test.sh b/tools/testing/selftests/multikernel/pciback_lifetime_test.sh new file mode 100644 index 00000000000000..28ea546e538825 --- /dev/null +++ b/tools/testing/selftests/multikernel/pciback_lifetime_test.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-pciback-lifetime.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^#define MK_IPI_RING_SIZE / { print } + /^#define MK_MSG_PCI / { print } + /^#define MK_PCI_CFG_/ { print } + /^struct mk_pci_cfg_payload \{/ { copy = 1 } + copy { print } + copy && /^};/ { copy = 0 }' "$src/include/linux/multikernel.h" > "$build/pciback-types.h" +awk '/^struct mk_pciback_work \{/ { copy = 1 } + /^static struct mk_pciback_work mk_pciback_requests\[/ { print } + /^static int __init mk_pciback_init\(/ { copy = 1 } + /^static void mk_pciback_msg_handler\(/ { copy = 1 } + /^static struct mk_pciback_dev \*mk_pciback_find\(/ { copy = 1 } + /^static void mk_pciback_(put|answer|work_fn|defer|request|free)\(/ { copy = 1 } + /^void mk_pciback_unlend(_one)?\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/pciback.c" > "$build/pciback-production.h" +awk '/^static inline unsigned int mk_ring_idx\(/ { copy = 1 } + /^static (void|int) mk_ipi_drain_ring\(void\)$/ { copy = 1 } + /^static (void|int) multikernel_interrupt_handler\(void\)$/ { copy = 1 } + /^bool mk_ipi_in_drain\(void\)$/ { copy = 1 } + /^(void|int) mk_ipi_poll\(void\)$/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/ipi.c" > "$build/pciback-ring.h" +# Permit regression runs on the old void API: it cannot report blocked retirement. +awk '/^int mk_ipi_poll\(void\)$/ { print "#define POLL_HAS_RESULT 1" }' \ + "$src/kernel/multikernel/ipi.c" > "$build/pciback-poll-api.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -pthread -Wall -Wextra -Werror \ + -Wno-unused-parameter -I"$build" -I"$src/tools/include" \ + "$test_dir/pciback_lifetime_test.c" -o "$build/pciback_lifetime_test" +timeout 15 "$build/pciback_lifetime_test" diff --git a/tools/testing/selftests/multikernel/pool_return_test.c b/tools/testing/selftests/multikernel/pool_return_test.c new file mode 100644 index 00000000000000..d2b4019db49d8e --- /dev/null +++ b/tools/testing/selftests/multikernel/pool_return_test.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real retirement predicate with retained local ownership after CPUHP failure. */ +#include +#include +#include +typedef unsigned long mk_phys_cpu_t; +struct mk_cpu_set { + unsigned int nr; + mk_phys_cpu_t ids[2]; +}; + +struct mk_instance { + struct mk_cpu_set *cpus, *cpus_on_slot; + struct mk_instance *next; +}; + +static struct mk_cpu_set local_cpus = { .nr = 1, .ids = { 0x10001 } }, + free_cpus; +static struct mk_instance self = { .cpus = &local_cpus }, child; +static struct mk_instance *mk_self = &self, *mk_instance_list = &self; +static struct { + struct mk_cpu_set *cpus; +} pool = { .cpus = &free_cpus }, *mk_pool = &pool; +static bool mk_overlay_mutex = true, mk_instance_mutex, online; +static int logical_cpu = 1; +static const unsigned int nr_cpu_ids = 4; +#define mk_cpu_set_empty(p) (!(p) || !(p)->nr) +#define mk_cpu_set_for_each(i, id, set) \ + for ((i) = 0; \ + (set) && (i) < (set)->nr && (((id) = (set)->ids[i]), true); \ + (i)++) +#define list_for_each_entry(p, head, member) \ + for ((p) = *(head); (p); (p) = (p)->next) +#define lockdep_assert_held(p) assert(*(p)) +static void mutex_lock(bool *p) +{ + assert(!*p); + *p = true; +} + +static void mutex_unlock(bool *p) +{ + assert(*p); + *p = false; +} + +static int arch_cpu_from_physical_id(mk_phys_cpu_t id) +{ + assert(id == 0x10001); + return logical_cpu; +} + +static bool cpu_online(unsigned int cpu) +{ + assert(cpu < nr_cpu_ids); + return online; +} + +#include "pool-return.h" + +int main(void) +{ + unsigned int failures = 0; + + if (mk_pool_cpus_returned()) { + puts("FAIL: locally owned offline CPU must pin the pool park area"); + failures++; + } + online = true; + assert(mk_pool_cpus_returned()); + logical_cpu = -1; + if (mk_pool_cpus_returned()) { + puts("FAIL: unmapped retained CPU must pin the pool park area"); + failures++; + } + logical_cpu = nr_cpu_ids; + if (mk_pool_cpus_returned()) { + puts("FAIL: invalid retained CPU must pin the pool park area"); + failures++; + } + logical_cpu = 1; + free_cpus.nr = 1; + assert(!mk_pool_cpus_returned()); + free_cpus.nr = 0; + self.next = &child; + child.cpus_on_slot = &local_cpus; + assert(!mk_pool_cpus_returned()); + assert(!mk_instance_mutex); + printf("Pool return: 6 scenarios, %u failures\n", failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/multikernel/pool_return_test.sh b/tools/testing/selftests/multikernel/pool_return_test.sh new file mode 100644 index 00000000000000..bd3deece4fa202 --- /dev/null +++ b/tools/testing/selftests/multikernel/pool_return_test.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-pool-return.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^bool mk_pool_cpus_returned\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/core.c" > "$build/pool-return.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror \ + -Wno-unused-function -Wno-unused-variable -I"$build" \ + "$test_dir/pool_return_test.c" -o "$build/pool_return_test" +"$build/pool_return_test" diff --git a/tools/testing/selftests/multikernel/repark_arrival_test.c b/tools/testing/selftests/multikernel/repark_arrival_test.c new file mode 100644 index 00000000000000..b77792578eb402 --- /dev/null +++ b/tools/testing/selftests/multikernel/repark_arrival_test.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real handoff/tracking code; mailbox claim and destination arrival controlled. */ +#include +#include +#include +#include +#include + +typedef uint32_t u32; +typedef uint64_t mk_phys_cpu_t; +#define MK_SPAWN_F_REPARK 1 +struct mk_spawn_context { + unsigned long repark_park_phys, repark_park_cr3, repark_slot_phys, flags; +}; +struct mk_cpu_set { bool tracked; }; +struct mk_instance { + struct { struct mk_spawn_context *spawn_ctx; } arch; + struct mk_cpu_set *cpus_on_slot; +}; +struct mk_pool { + bool park_lock; + struct { + struct mk_spawn_context *slot; + unsigned long park_phys, cr3, slot_phys; + } arch; +}; +static struct mk_spawn_context source, destination; +static struct mk_pool pool = { .arch = { .slot = &destination, .park_phys = 0x9000, + .cr3 = 0xa000, .slot_phys = (unsigned long)&destination } }, *mk_pool = &pool; +static bool destination_arrived; +static int source_error; +static unsigned int wakes, destination_wakes, checks, failures; +#define virt_to_phys(p) ((unsigned long)(p)) +#define __va(p) ((void *)(unsigned long)(p)) +#define lockdep_assert_held(p) assert(*(p)) +static bool *lock(bool *p) { assert(!*p); *p = true; return p; } +static void unlock(bool **p) +{ + assert(**p); + **p = false; +} +#define guard(kind) bool *held_##kind __attribute__((cleanup(unlock))) = lock +static bool mk_cpu_set_contains(struct mk_cpu_set *set, mk_phys_cpu_t cpu) +{ + assert(cpu == 17); + return set->tracked; +} +static void mk_cpu_set_del(struct mk_cpu_set *set, mk_phys_cpu_t cpu) +{ + assert(cpu == 17); + set->tracked = false; +} + +/* Mailbox state ownership is tested separately by mailbox_test.sh. */ +static int mk_slot_begin(struct mk_spawn_context *slot) +{ + return slot ? 0 : -ENODEV; +} + +static bool mk_slot_idle(struct mk_spawn_context *slot) +{ + (void)slot; + return true; +} +static int mk_slot_wake(struct mk_spawn_context *slot, u32 cpu, const char *what) +{ + (void)what; + assert(pool.park_lock && cpu == 17); + wakes++; + if (slot == &source) + return source_error; + assert(slot == &destination); + destination_wakes++; + return destination_arrived ? 0 : -ETIMEDOUT; +} +#include "repark-production.h" + +static void expect(const char *name, bool passed) +{ + checks++; + if (!passed) { + fprintf(stderr, "FAIL %s\n", name); + failures++; + } +} +int main(void) +{ + struct mk_cpu_set cpus = { .tracked = true }; + struct mk_instance instance = { .arch.spawn_ctx = &source, .cpus_on_slot = &cpus }; + int ret; + + ret = mk_repark_cpu_to_host(&instance, 17); + expect("source claim without destination arrival refuses retirement", ret == -ETIMEDOUT); + expect("delayed departure retains old-context tracking", cpus.tracked); + expect("destination must be probed", destination_wakes == 1); + cpus.tracked = true; + wakes = destination_wakes = 0; + destination_arrived = true; + ret = mk_repark_cpu_to_host(&instance, 17); + expect("destination arrival permits retirement", !ret && !cpus.tracked); + expect("successful transfer claims both source and destination", wakes == 2 && destination_wakes == 1); + cpus.tracked = true; + wakes = destination_wakes = 0; + source_error = -ETIMEDOUT; + ret = mk_repark_cpu_to_host(&instance, 17); + expect("failed source claim retains tracking", ret == -ETIMEDOUT && cpus.tracked); + expect("failed source does not republish destination", wakes == 1 && !destination_wakes); + source_error = 0; + wakes = destination_wakes = 0; + pool.park_lock = true; + ret = mk_publish_repark(&destination, 17, pool.arch.park_phys, + pool.arch.cr3, pool.arch.slot_phys, "self-confirm"); + pool.park_lock = false; + expect("self-confirm needs one claim, not recursion", !ret && wakes == 1); + printf("Repark arrival: %u checks, %u failures\n", checks, failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/multikernel/repark_arrival_test.sh b/tools/testing/selftests/multikernel/repark_arrival_test.sh new file mode 100644 index 00000000000000..c3bc922c4d8f44 --- /dev/null +++ b/tools/testing/selftests/multikernel/repark_arrival_test.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-repark-arrival.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^static int mk_publish_repark\(/ || /^int mk_repark_cpu_to_host\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/arch/x86/multikernel/spawn.c" > "$build/repark-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -I"$build" \ + "$test_dir/repark_arrival_test.c" -o "$build/repark_arrival_test" +"$build/repark_arrival_test" diff --git a/tools/testing/selftests/multikernel/resource_transfer_test.c b/tools/testing/selftests/multikernel/resource_transfer_test.c new file mode 100644 index 00000000000000..7b4b8178b0a19e --- /dev/null +++ b/tools/testing/selftests/multikernel/resource_transfer_test.c @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual senders; injected transport, native hotplug and grant bookkeeping. */ +#include +#include +#include +#include +#include +#define CONFIG_PHYS_ADDR_T_64BIT +#include +#include +#include +#include + +#define u64 __u64 +#include "resource-types.h" +#define MK_STATE_ACTIVE 2 +#define MK_STATE_LOADED 1 +#define MK_STATE_FAILED 3 +#define PFN_PHYS(pfn) ((pfn) << 12) +#define pr_warn(...) do { } while (0) +#define pr_err(...) do { } while (0) +#define pr_info(...) do { } while (0) +#define strscpy(dst, src, len) snprintf(dst, len, "%s", src) +#define lockdep_assert_held(m) assert(*(m)) +struct mk_instance { + int id, state; + bool resource_uncertain, pci_devices_valid; + struct list_head list, pci_devices; +}; + +struct mk_pci_device { + u16 domain; + u8 bus, slot, func; + struct list_head list; +}; + +#define PCI_SLOT(fn) ((fn) >> 3) +#define PCI_FUNC(fn) ((fn) & 7) +static LIST_HEAD(mk_instance_list); +static struct mk_pci_device grant; +static struct mk_instance self = { .id = 0 }, target; +static struct mk_instance *mk_self = &self; +static void *mk_pool; +static bool mk_overlay_mutex = true; +static bool mk_instance_mutex; + +static bool *lock(bool *held) +{ + assert(mk_overlay_mutex && !*held); + *held = true; + return held; +} + +static void unlock(bool **held) +{ + assert(**held); + **held = false; +} + +#define guard(kind) bool *held_##kind __attribute__((cleanup(unlock))) = lock +#include "resource-ownership.h" +struct pci_dev { u16 vendor, device; void *bus; }; +static struct pci_dev device = { .vendor = 0x1234, .device = 0x5678 }; +struct mk_pending_msg { int live; }; +static struct mk_pending_msg request; +static int send_error, wait_error, alloc_error, reserve_error, release_error; +static int lend_error, references, publishes, lends, resets; +static bool owned, removal_reserved, lent, absent_device, owned_at_lend; +static bool owned_at_publish; +static bool pool_owned; +static unsigned int detaches; +static unsigned int checks, failures; +enum operation { MEM_ADD, MEM_REMOVE, DEVICE_ADD, DEVICE_REMOVE }; + +static void expect(const char *why, int got, int want) +{ + checks++; + if (got != want) { + fprintf(stderr, "FAIL %s: %d != %d\n", why, got, want); + failures++; + } +} + +static struct mk_instance *mk_instance_find(int id) +{ + assert(id == target.id); + references++; + return ⌖ +} + +static void mk_instance_put(struct mk_instance *instance) +{ + assert(instance == &target && references == 1); + references--; +} + +static int reserve(void) +{ + if (reserve_error) + return reserve_error; + if (owned) + return -EBUSY; + owned = true; + return 0; +} + +static int release(void) +{ + assert(owned); + if (release_error) + return release_error; + owned = false; + return 0; +} + +static void range(phys_addr_t start, size_t size) +{ + assert(start == PFN_PHYS(0x40000ULL) && size == PFN_PHYS(32ULL)); +} + +static int mk_instance_add_memory_region(struct mk_instance *p, phys_addr_t start, + size_t size) +{ + range(start, size); + return reserve(); +} + +static int mk_instance_remove_memory_region(struct mk_instance *p, phys_addr_t start, + size_t size) +{ + range(start, size); + return release(); +} + +static int mk_instance_begin_memory_removal(struct mk_instance *p, phys_addr_t start, + size_t size) +{ + range(start, size); + assert(owned && !removal_reserved); + if (reserve_error) + return reserve_error; + removal_reserved = true; + return 0; +} + +static void mk_instance_finish_memory_removal(struct mk_instance *p, phys_addr_t start, + size_t size) +{ + range(start, size); + assert(owned && removal_reserved); + owned = false; + removal_reserved = false; +} + +static void mk_instance_abort_memory_removal(struct mk_instance *p, phys_addr_t start, + size_t size) +{ + range(start, size); + assert(owned && removal_reserved); + removal_reserved = false; +} + +static int mk_instance_add_pci_device(struct mk_instance *p, u16 dom, u8 bus, u8 fn) +{ + int ret; + + assert(dom == 0x8001 && bus == 2 && fn == 3); + ret = reserve(); + if (!ret) + list_add(&grant.list, &p->pci_devices); + return ret; +} + +static int mk_instance_remove_pci_device(struct mk_instance *p, u16 dom, u8 bus, u8 fn) +{ + int ret = release(); + + if (!ret) + list_del_init(&grant.list); + return ret; +} + +static int mk_pciback_lend_one(struct mk_instance *p, u16 dom, u8 bus, u8 fn) +{ + lends++; + owned_at_lend = owned; + if (!lend_error) + lent = true; + return lend_error; +} + +static void mk_pciback_unlend_one(struct mk_instance *p, u16 dom, u8 bus, u8 fn) +{ + resets++; + lent = false; +} + +static struct pci_dev *pci_get_domain_bus_and_slot(u16 dom, u8 bus, u8 fn) +{ + return absent_device ? NULL : &device; +} + +static void pci_dev_put(struct pci_dev *p) { } +static void mk_pci_bus_describe(void *bus, struct mk_pci_bus_desc *desc) { } +static struct mk_pending_msg *mk_msg_pending_add(u32 type, u32 op, u64 id) +{ + assert(!request.live); + assert(id == (op == MK_RES_MEM_ADD || op == MK_RES_MEM_REMOVE ? + 0x40000ULL : 0x80010203ULL)); + if (alloc_error) + return ERR_PTR(alloc_error); + request.live = 1; + return &request; +} + +static u64 mk_msg_pending_id(struct mk_pending_msg *p) { return 17; } +static void mk_msg_pending_cancel(struct mk_pending_msg *p) +{ + assert(p->live && !publishes); + p->live = 0; +} + +static int mk_send_message_id(int id, u32 type, u32 op, u64 msg_id, + void *payload, size_t size) +{ + assert(request.live && msg_id == 17); + if (send_error) + return send_error; + publishes++; + owned_at_publish = owned; + return 0; +} + +static int mk_msg_pending_wait(struct mk_pending_msg *p, int timeout) +{ + assert(p->live && publishes == 1); + p->live = 0; + return wait_error; +} + +/* Local native paths are not exercised by this remote ownership fixture. */ +#define mk_do_mem_add(...) (-EOPNOTSUPP) +#define mk_do_mem_remove(...) (-EOPNOTSUPP) +#define mk_pool_device_remove(...) (-EOPNOTSUPP) +#define mk_do_device_add(...) (-EOPNOTSUPP) +#define mk_manifest_phys() 0 +#define mk_pcifront_adopt_function(...) (-EOPNOTSUPP) + +static int mk_do_device_remove(u16 dom, u8 bus, u8 fn) +{ + detaches++; + return 0; +} + +static int mk_root_add_pci_device(u16 dom, u8 bus, u8 fn, const char *alias) +{ + pool_owned = true; + return 0; +} + +#include "resource-production.h" + +static void reset(enum operation op) +{ + guard(mutex)(&mk_instance_mutex); + + assert(mk_overlay_mutex && !references && !request.live); + target = (struct mk_instance) { .id = 2, .state = MK_STATE_ACTIVE, + .pci_devices_valid = true }; + INIT_LIST_HEAD(&mk_instance_list); + INIT_LIST_HEAD(&target.pci_devices); + INIT_LIST_HEAD(&self.pci_devices); + list_add(&target.list, &mk_instance_list); + list_add(&self.list, &mk_instance_list); + grant = (struct mk_pci_device) { .domain = 0x8001, .bus = 2, .slot = 0, .func = 3 }; + INIT_LIST_HEAD(&grant.list); + if (op == DEVICE_REMOVE) + list_add(&grant.list, &target.pci_devices); + pool_owned = false; + detaches = 0; + send_error = 0; + wait_error = 0; + alloc_error = 0; + reserve_error = 0; + release_error = 0; + lend_error = 0; + publishes = 0; + lends = 0; + resets = 0; + owned = op == MEM_REMOVE || op == DEVICE_REMOVE; + lent = op == DEVICE_REMOVE; + removal_reserved = false; + absent_device = false; + owned_at_lend = false; + owned_at_publish = false; +} + +static int run(enum operation op) +{ + switch (op) { + case MEM_ADD: + return mk_send_mem_add(2, 0x40000, 32, 0, 0); + case MEM_REMOVE: + return mk_send_mem_remove(2, 0x40000, 32); + case DEVICE_ADD: + return mk_send_device_add(2, 0x8001, 2, 3, "test", 0); + case DEVICE_REMOVE: + return mk_send_device_remove(2, 0x8001, 2, 3); + } + assert(false); + return -EINVAL; +} + +int main(void) +{ + const int errors[] = { -ETIMEDOUT, -EIO, -EBUSY }; + enum operation op; + unsigned int i; + + for (op = MEM_ADD; op <= DEVICE_REMOVE; op++) { + for (i = 0; i < ARRAY_SIZE(errors); i++) { + reset(op); + wait_error = errors[i]; + expect("published failure returned", run(op), errors[i]); + expect("published failure retains ownership", owned, true); + expect("published failure quarantines instance", + target.resource_uncertain, true); + if (op == MEM_REMOVE) + expect("late removal cannot race allocation", + removal_reserved, true); + if (op >= DEVICE_ADD) { + expect("late probe/removal retains lending", lent, true); + expect("uncertain device never reset", resets, 0); + expect("pool insertion cannot bypass retained grant", + mk_pool_device_add(0x8001, 2, 3, NULL), -EBUSY); + expect("quarantined device not detached locally", detaches, 0); + expect("quarantined device unavailable for another grant", + pool_owned, false); + } + /* A late worker can now finish; another grant must still fail. */ + expect("competing reservation rejected", reserve(), -EBUSY); + if (target.resource_uncertain) { + expect("quarantined request rejected", run(op), -EBUSY); + expect("no new publication", publishes, 1); + } + } + for (i = 0; i < 2; i++) { + reset(op); + if (i) + send_error = -ENOSPC; + else + alloc_error = -ENOMEM; + expect("unpublished error returned", run(op), i ? -ENOSPC : -ENOMEM); + expect("unpublished error restores grant", + owned, op == MEM_REMOVE || op == DEVICE_REMOVE); + expect("unpublished remove reservation undone", removal_reserved, false); + expect("unpublished request never quarantined", + target.resource_uncertain, false); + expect("unpublished request absent", publishes, 0); + } + reset(op); + expect("normal request succeeds", run(op), 0); + expect("ownership precedes publication", owned_at_publish, true); + expect("normal request leaves expected owner", + owned, op == MEM_ADD || op == DEVICE_ADD); + if (op == DEVICE_ADD) + expect("ownership precedes lend", owned_at_lend, true); + reset(op); + target.resource_uncertain = true; + target.state = MK_STATE_LOADED; + expect("state change cannot clear quarantine", run(op), -EBUSY); + expect("quarantined state publishes nothing", publishes, 0); + } + reset(DEVICE_ADD); + reserve_error = -ENOENT; + expect("unowned device rejected", run(DEVICE_ADD), -ENOENT); + expect("unowned device never lent", lends, 0); + expect("unowned device never published", publishes, 0); + reset(DEVICE_ADD); + lend_error = -EIO; + expect("lend failure returned", run(DEVICE_ADD), -EIO); + expect("lend failure restores grant", owned, false); + reset(DEVICE_ADD); + absent_device = true; + expect("missing device returned", run(DEVICE_ADD), -ENODEV); + expect("missing device restores grant", owned, false); + expect("missing device unlent", lent, false); + for (op = MEM_ADD; op <= DEVICE_REMOVE; op++) { + if (op == MEM_REMOVE) + continue; + reset(op); + release_error = -EBUSY; + if (op != DEVICE_REMOVE) + send_error = -ENOSPC; + expect("failed grant cleanup returned", run(op), -EBUSY); + expect("failed grant cleanup retains owner", owned, true); + expect("failed grant cleanup quarantined", target.resource_uncertain, true); + } + assert(mk_overlay_mutex && !references && !request.live); + reset(DEVICE_REMOVE); + expect("ordinary live grant also blocks pool insertion", + mk_pool_device_add(0x8001, 2, 3, NULL), -EBUSY); + expect("different domain is a different function", + mk_pool_device_add(0x8002, 2, 3, NULL), 0); + expect("different bus is a different function", + mk_pool_device_add(0x8001, 3, 3, NULL), 0); + expect("different function is a different device", + mk_pool_device_add(0x8001, 2, 4, NULL), 0); + reset(DEVICE_ADD); + self.pci_devices_valid = true; + list_add(&grant.list, &self.pci_devices); + expect("self pool entry does not imply another owner", + mk_pool_device_add(0x8001, 2, 3, NULL), 0); + printf("Resource transfers: %u checks, %u failures (transport/native operations modeled)\n", + checks, failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/multikernel/resource_transfer_test.sh b/tools/testing/selftests/multikernel/resource_transfer_test.sh new file mode 100644 index 00000000000000..c44b43d9b5a7d9 --- /dev/null +++ b/tools/testing/selftests/multikernel/resource_transfer_test.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-resource-transfer.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^#define MK_MSG_IO / { copy = 1 } + /^\/\* Opaque type/ { copy = 0 } + copy { print }' "$src/include/linux/multikernel.h" > "$build/resource-types.h" +awk '/^bool mk_pci_device_assigned\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/core.c" \ + > "$build/resource-ownership.h" +awk '/^int mk_pool_device_add\(/ { copy = 1 } + /^int mk_send_(mem|device)_(add|remove)\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/hotplug.c" \ + > "$build/resource-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror \ + -Wno-unused-parameter -I"$build" -I"$src/tools/include" \ + "$test_dir/resource_transfer_test.c" -o "$build/resource_transfer_test" +"$build/resource_transfer_test" diff --git a/tools/testing/selftests/multikernel/start_lifetime_test.c b/tools/testing/selftests/multikernel/start_lifetime_test.c new file mode 100644 index 00000000000000..7d26573fdf79e4 --- /dev/null +++ b/tools/testing/selftests/multikernel/start_lifetime_test.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual startup/state functions; architecture and PCI hardware are modeled. */ +#include +#include +#include +#include + +enum mk_instance_state { MK_STATE_READY, MK_STATE_LOADED, MK_STATE_ACTIVE, + MK_STATE_FAILED }; +struct mk_instance { + enum mk_instance_state state; + int id; + int resource_uncertain; + int halted_pending; + int *ipi_data; +}; +struct kimage { int unused; }; +static int lend_error, start_error, starts, unlends, failures; +static enum mk_instance_state state_at_start, state_at_unlend; +#define pr_debug(...) do { } while (0) +#define pr_info(...) do { } while (0) +#define pr_err(...) do { } while (0) +#define mk_instance_track_dump(...) do { } while (0) +#define lockdep_assert_held(...) do { } while (0) +#define atomic_read(p) (*(p)) +#define mk_instance_ipi_reset(instance) (*(instance)->ipi_data = 0) + +static int mk_pciback_lend(struct mk_instance *instance) +{ + (void)instance; + return lend_error; +} + +static void mk_pciback_unlend(struct mk_instance *instance) +{ + state_at_unlend = instance->state; + unlends++; +} + +static int mk_arch_spawn_instance(struct kimage *image, + struct mk_instance *instance, int cpu) +{ + (void)image; + assert(cpu == 3); + starts++; + state_at_start = instance->state; + return start_error; +} + +#include "start-production.h" + +#define CHECK(cond, why) do { \ + if (!(cond)) { fprintf(stderr, "FAIL: %s\n", why); failures++; } \ +} while (0) + +int main(void) +{ + int ring = 1; + struct mk_instance instance = { .state = MK_STATE_LOADED, .ipi_data = &ring }; + struct kimage image = { 0 }; + + instance.halted_pending = 1; + CHECK(mk_instance_start(&instance, &image, 3) == -EBUSY, + "queued halt notice excludes a new execution generation"); + CHECK(!starts && ring == 1, "pending halt leaves ring and execution untouched"); + instance.halted_pending = 0; + starts = 0; + ring = 1; + instance.state = MK_STATE_LOADED; + + instance.resource_uncertain = 1; + CHECK(mk_instance_start(&instance, &image, 3) == -EBUSY, + "uncertain resource ownership rejects direct start"); + CHECK(!starts && ring == 1, "quarantine leaves ring and execution untouched"); + instance.resource_uncertain = 0; + starts = 0; + instance.state = MK_STATE_LOADED; + ring = 1; + + lend_error = -ENOMEM; + CHECK(mk_instance_start(&instance, &image, 3) == -ENOMEM, + "return lend failure"); + CHECK(!starts && instance.state == MK_STATE_LOADED, + "lend failure cannot start or mark the instance executing"); + lend_error = 0; + start_error = -EIO; + CHECK(mk_instance_start(&instance, &image, 3) == -EIO, + "return architecture failure"); + CHECK(state_at_start == MK_STATE_ACTIVE, + "potential execution must be recorded before architecture start"); + CHECK(instance.state == MK_STATE_FAILED && !unlends, + "partial start retains device reservations while execution is uncertain"); + + instance.state = MK_STATE_ACTIVE; + mk_instance_set_state(&instance, MK_STATE_FAILED); + CHECK(!unlends, "FAILED alone cannot reset or release devices"); + /* Caller supplies CPU/work settlement before publishing LOADED. */ + mk_instance_set_state(&instance, MK_STATE_LOADED); + CHECK(unlends == 1, "settled failure retires previously lent devices"); + CHECK(state_at_unlend == MK_STATE_FAILED, + "PCI retirement completes before publishing reusable state"); + start_error = 0; + CHECK(!mk_instance_start(&instance, &image, 3), "successful start"); + CHECK(instance.state == MK_STATE_ACTIVE && state_at_start == MK_STATE_ACTIVE, + "successful start preserves potential-execution state"); + mk_instance_set_state(&instance, MK_STATE_LOADED); + CHECK(unlends == 2, "settled ordinary shutdown retires devices"); + if (failures) + return 1; + puts("Startup lifetime: PASS (modeled architecture/PCI; caller settlement assumed)"); + return 0; +} diff --git a/tools/testing/selftests/multikernel/start_lifetime_test.sh b/tools/testing/selftests/multikernel/start_lifetime_test.sh new file mode 100644 index 00000000000000..6acb9affe5a69b --- /dev/null +++ b/tools/testing/selftests/multikernel/start_lifetime_test.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-start-lifetime.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^void mk_instance_set_state\(/ { copy = 1 } + /^int mk_instance_start\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/core.c" \ + > "$build/start-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror \ + -I"$build" "$test_dir/start_lifetime_test.c" -o "$build/start_lifetime_test" +"$build/start_lifetime_test" diff --git a/tools/testing/selftests/multikernel/teardown_test.c b/tools/testing/selftests/multikernel/teardown_test.c new file mode 100644 index 00000000000000..a868ef5f6875c3 --- /dev/null +++ b/tools/testing/selftests/multikernel/teardown_test.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real destruction; firmware, kernfs drain and resource retirement injected. */ +#include +#include +#include +#include +#include + +#define MK_STATE_READY 1 +#define MK_STATE_LOADED 2 +#define MK_STATE_ACTIVE 3 +#define MK_STATE_FAILED 4 +#define pr_err(...) do { } while (0) +struct kernfs_node { int unused; }; +struct mk_instance { + int id, state; + const char *name; + struct kernfs_node *kn; + void *kimage; + struct list_head list; +}; +static bool mk_instance_mutex, mk_overlay_mutex = true; +static int mk_instance_idr; +static unsigned int ref_drops, retired, removed, locked_drains, checks, failures; +static int prepare_error, retire_error; +#define lockdep_assert_held(m) assert(*(m)) +#define mutex_lock(m) do { assert(!*(m)); *(m) = true; } while (0) +#define mutex_unlock(m) do { assert(*(m)); *(m) = false; } while (0) +#define idr_remove(p, id) ((void)(p), (void)(id), removed++) +#define mk_instance_idr_remove(id) idr_remove(&mk_instance_idr, id) +static void mk_instance_put(struct mk_instance *p) { ref_drops++; } +static void kernfs_remove(struct kernfs_node *kn) +{ + locked_drains += mk_instance_mutex; +} +static __attribute__((unused)) int mk_instance_prepare_destroy(struct mk_instance *p) +{ + assert(mk_overlay_mutex && !mk_instance_mutex); + return prepare_error; +} +static __attribute__((unused)) int mk_instance_retire_resources(struct mk_instance *p) +{ + assert(mk_overlay_mutex && !mk_instance_mutex); + retired++; + return retire_error; +} +#include "destroy-production.h" + +static void expect(const char *name, int got, int want) +{ + checks++; + if (got != want) { + fprintf(stderr, "FAIL %s: %d != %d\n", name, got, want); + failures++; + } +} + +static int destroy(struct mk_instance *p) +{ + /* The original API required the caller's registry lock. */ + int ret; + + mk_instance_mutex = true; + ret = mk_instance_destroy(p); + mk_instance_mutex = false; + return ret; +} + +int main(void) +{ + struct kernfs_node kn; + struct mk_instance p = { .id = 7, .state = MK_STATE_ACTIVE, .name = "spawn", .kn = &kn }; + LIST_HEAD(instances); + + INIT_LIST_HEAD(&p.list); + list_add(&p.list, &instances); + expect("active destruction refused", destroy(&p), -EBUSY); + p.state = MK_STATE_FAILED; + expect("uncertain failure destruction refused", destroy(&p), -EBUSY); + expect("uncertain record retained", removed, 0); + if (removed) { + INIT_LIST_HEAD(&instances); + INIT_LIST_HEAD(&p.list); + list_add(&p.list, &instances); + p.kn = &kn; + removed = ref_drops = 0; + } + p.state = MK_STATE_READY; + prepare_error = -EBUSY; + expect("failed stop/repark proof refuses destruction", destroy(&p), -EBUSY); + expect("failed stop retains registry membership", removed, 0); + expect("failed stop retains resources", retired, 0); + expect("failed stop retains references", ref_drops, 0); + if (removed) { + INIT_LIST_HEAD(&instances); + INIT_LIST_HEAD(&p.list); + list_add(&p.list, &instances); + p.kn = &kn; + removed = ref_drops = 0; + } + prepare_error = 0; + retire_error = -EIO; + expect("failed unmap refuses destruction", destroy(&p), -EIO); + expect("failed unmap retains registry membership", removed, 0); + expect("failed unmap retains references", ref_drops, 0); + if (removed) { + INIT_LIST_HEAD(&instances); + INIT_LIST_HEAD(&p.list); + list_add(&p.list, &instances); + p.kn = &kn; + removed = ref_drops = 0; + } + retired = 0; + retire_error = 0; + expect("confirmed destruction", destroy(&p), 0); + expect("record unregistered once", removed, 1); + expect("registry and kernfs references released", ref_drops, 2); + expect("kernfs drains outside registry lock", locked_drains, 0); + expect("resources reusable before overlay returns", retired, 1); + printf("Teardown: %u checks, %u failures\n", checks, failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/multikernel/teardown_test.sh b/tools/testing/selftests/multikernel/teardown_test.sh new file mode 100644 index 00000000000000..f1d2e80a05817d --- /dev/null +++ b/tools/testing/selftests/multikernel/teardown_test.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-teardown.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^int mk_instance_destroy\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/kernfs.c" \ + > "$build/destroy-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -Wno-unused-parameter \ + -I"$build" -I"$src/tools/include" "$test_dir/teardown_test.c" -o "$build/teardown_test" +"$build/teardown_test" diff --git a/tools/testing/selftests/multikernel/voluntary_shutdown_test.c b/tools/testing/selftests/multikernel/voluntary_shutdown_test.c new file mode 100644 index 00000000000000..8e32c51ab5fa6a --- /dev/null +++ b/tools/testing/selftests/multikernel/voluntary_shutdown_test.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real shutdown admission/notification functions; controlled kernel edges. */ +#include +#include +#include +#include +#include +#include + +typedef uint32_t u32; +typedef uint64_t u64; +#define __init +#define __noreturn __attribute__((noreturn)) +#define pr_info(...) \ + do { \ + } while (0) +#define pr_err(...) \ + do { \ + } while (0) +#define MK_MSG_SYSTEM 1 +#define MK_SYS_SHUTDOWN 2 +#define MK_SYS_SHUTDOWN_ACK 3 +#define MK_SYS_HALTED 4 +/* These stubs model publication order, not hardware memory ordering. */ +#define smp_store_release(p, v) (*(p) = (v)) +#define smp_load_acquire(p) (*(p)) + +enum { SYSTEM_RUNNING, SYSTEM_HALT, SYSTEM_POWER_OFF, SYSTEM_RESTART }; +static int system_state = SYSTEM_RUNNING; +static bool panic_now, kexec_in_progress, mk_overlay_mutex; +static void *mk_boot_context; +static int panic_timeout; +static struct { + void (*halt)(void); + void (*power_off)(void); + void (*restart)(char *); + void (*emergency_restart)(void); +} machine_ops; +static struct { + void (*stop_this_cpu)(void); +} smp_ops; +static bool native_hotplug_disabled; +static unsigned int notices, remote_parks, failures; +static jmp_buf parked; +static struct { + int id; +} self = { 7 }, *mk_self = &self; +struct mk_resource_ack { + u32 operation, result; + u64 resource_id; +}; +struct syscore_ops { + void (*shutdown)(void *); +}; +struct syscore { + const struct syscore_ops *ops; +}; +static struct syscore *registered; + +static void expect(const char *name, bool ok) +{ + if (!ok) { + fprintf(stderr, "FAIL %s\n", name); + failures++; + } +} + +static inline bool panic_in_progress(void) +{ + return panic_now; +} +static inline bool mutex_trylock(bool *lock) +{ + if (*lock) + return false; + *lock = true; + return true; +} + +static inline void register_syscore(struct syscore *ops) +{ + registered = ops; +} + +static int mk_send_message_id(int target, u32 type, u32 subtype, u64 id, + const void *data, u32 len) +{ + const struct mk_resource_ack *ack = data; + + expect("HALTED wire tuple", target == 0 && type == MK_MSG_SYSTEM && + subtype == MK_SYS_HALTED && + id == 0 && len == sizeof(*ack) && + ack->operation == MK_SYS_SHUTDOWN && + ack->result == 0 && + ack->resource_id == 7); + expect("notice has native CPUHP barrier", native_hotplug_disabled); + expect("notice excludes controllers", mk_overlay_mutex); + notices++; + return 0; +} + +static void __noreturn mk_enter_pool_state(void *data) +{ + longjmp(parked, 1); +} +static void smp_call_function(void (*fn)(void *), void *data, int wait) +{ + expect("remote nonwaiting park", + fn == mk_enter_pool_state && !data && !wait); + remote_parks++; +} + +#include "voluntary-production.h" + +int main(int argc, char **argv) +{ + const char *mode; + bool qualified, spawn; + + assert(argc == 2); + mode = argv[1]; + spawn = strcmp(mode, "host") != 0; + mk_boot_context = spawn ? &self : NULL; + expect("reboot init", mk_spawn_reboot_init() == 0); + expect("shutdown preparation registered only in spawn", + !!registered == spawn); + if (!spawn) + return failures ? 1 : 0; + system_state = SYSTEM_HALT; + if (!strcmp(mode, "poweroff")) + system_state = SYSTEM_POWER_OFF; + if (!strcmp(mode, "restart")) + system_state = SYSTEM_RESTART; + if (!strcmp(mode, "running")) + system_state = SYSTEM_RUNNING; + panic_now = !strcmp(mode, "panic"); + kexec_in_progress = !strcmp(mode, "kexec"); + mk_overlay_mutex = !strcmp(mode, "busy"); + qualified = !strcmp(mode, "halt") || !strcmp(mode, "poweroff") || + !strcmp(mode, "restart"); + if (strcmp(mode, "direct")) { + /* Normal reboot has already run migrate_to_reboot_cpu(). */ + native_hotplug_disabled = true; + if (registered) + registered->ops->shutdown(NULL); + } + if (!strcmp(mode, "panic-late")) + panic_now = true; + if (!strcmp(mode, "kexec-late")) + kexec_in_progress = true; + if (!setjmp(parked)) { + if (panic_now || !strcmp(mode, "direct")) + machine_ops.emergency_restart(); + else if (!strcmp(mode, "restart")) + machine_ops.restart(NULL); + else if (!strcmp(mode, "poweroff")) + machine_ops.power_off(); + else + machine_ops.halt(); + } + expect("local stop retains remote parking", remote_parks == 1); + expect("only qualified normal exit sends HALTED", + notices == (unsigned int)qualified); + printf("voluntary shutdown %s: %u failures\n", mode, failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/multikernel/voluntary_shutdown_test.sh b/tools/testing/selftests/multikernel/voluntary_shutdown_test.sh new file mode 100644 index 00000000000000..fa98112e5385df --- /dev/null +++ b/tools/testing/selftests/multikernel/voluntary_shutdown_test.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=${MK_SOURCE:-$(CDPATH= cd -- "$test_dir/../../../.." && pwd)} +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-voluntary-shutdown.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM + +awk '/^static void __noreturn mk_notify_down_and_park\(/ { copy = 1 } + /^void __noreturn mk_halt_to_pool\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/kernel/multikernel/core.c" \ + > "$build/voluntary-production.h" +awk '/^static bool mk_halt_quiesced;/ { print } + /^static void mk_halt_shutdown_prepare\(/ { copy = 1 } + /^static (const struct syscore_ops|struct syscore) mk_halt_syscore/ { copy = 1 } + /^static void __noreturn mk_(machine_halt|machine_restart|stop_this_cpu)\(/ { copy = 1 } + /^static int __init mk_spawn_reboot_init\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/arch/x86/multikernel/spawn.c" \ + >> "$build/voluntary-production.h" + +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -Wno-unused-parameter \ + -I"$build" "$test_dir/voluntary_shutdown_test.c" -o "$build/test" +result=0 +# A separate process represents a fresh kernel, including its static permit. +for scenario in direct halt poweroff restart panic panic-late kexec kexec-late busy running host; do + "$build/test" "$scenario" || result=1 +done +exit "$result" diff --git a/tools/testing/selftests/multikernel/vsock_rx_test.c b/tools/testing/selftests/multikernel/vsock_rx_test.c new file mode 100644 index 00000000000000..64e6728ec467fb --- /dev/null +++ b/tools/testing/selftests/multikernel/vsock_rx_test.c @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real receive/init/worker functions; FIFO, sockets and scheduling modeled. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef uint32_t u32; +typedef uint64_t u64; +typedef int raw_spinlock_t; +#include "vsock-wire.h" +#define le32_to_cpu(x) le32toh(x) +#define MK_MAX_DATA_SIZE 4096 +#define MK_IPI_RING_SIZE 64 +#define MK_MSG_NETWORK 1 +#define MK_NET_VSOCK_PKT 1 +#define GFP_KERNEL 1 +#define WQ_UNBOUND 1 +#define VSOCK_TRANSPORT_F_MULTIKERNEL 1 +#define __init +#define __exit +#define pr_err(...) ((void)0) +#define pr_info(...) ((void)0) +#define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) +struct mk_message { u64 id; u32 fields[4]; }; +static unsigned int checks, failures, raw_depth, allocs, delivered, runs; +static bool in_irq, core_live, handler_live, queue_live; +static bool fail_queue, fail_alloc, tail_inject, inject_on_unlock, short_copy; +static int fail_core, fail_handler; +static void (*allocation_hook)(void), (*unregister_hook)(void); +static void send_packet(unsigned int seq, unsigned int len, u32 claimed); +#define EXPECT(x) do { checks++; if (!(x)) { \ + fprintf(stderr, "line %d: %s\n", __LINE__, #x); failures++; \ +} } while (0) +#define raw_spin_lock_init(p) (*(p) = 0) +#define raw_spin_lock_irqsave(p, f) do { \ + assert(!*(p)); *(p) = 1; raw_depth++; (f) = 0; \ +} while (0) +static void raw_unlock(raw_spinlock_t *p) +{ + assert(*p && raw_depth); + *p = 0; + raw_depth--; + if (inject_on_unlock) { + inject_on_unlock = false; + send_packet(99, sizeof(struct virtio_vsock_hdr), 0); + } +} +#define raw_spin_unlock_irqrestore(p, f) do { (void)(f); raw_unlock(p); } while (0) + +/* Record FIFO API boundary: atomic whole-record insert, one consumer. */ +struct fifo { + unsigned char data[MK_IPI_RING_SIZE * MK_MAX_DATA_SIZE]; + unsigned int in, out; +}; +#define STRUCT_KFIFO_REC_2(n) struct fifo +#define INIT_KFIFO(f) memset(&(f), 0, sizeof(f)) +#define kfifo_is_empty(f) ((f)->in == (f)->out) +static unsigned int kfifo_peek_len(struct fifo *f) +{ + assert(raw_depth); + if (kfifo_is_empty(f)) { + if (tail_inject) { + tail_inject = false; + inject_on_unlock = true; + } + return 0; + } + return f->data[f->out % sizeof(f->data)] | + (f->data[(f->out + 1) % sizeof(f->data)] << 8); +} +static unsigned int kfifo_in(struct fifo *f, const void *p, unsigned int n) +{ + const unsigned char *data = p; + unsigned int i; + + assert(raw_depth && n && n <= UINT16_MAX); + if (n + 2 > sizeof(f->data) - (f->in - f->out)) + return 0; + f->data[f->in++ % sizeof(f->data)] = n; + f->data[f->in++ % sizeof(f->data)] = n >> 8; + for (i = 0; i < n; i++) + f->data[f->in++ % sizeof(f->data)] = data[i]; + return n; +} +static unsigned int kfifo_out(struct fifo *f, void *p, unsigned int n) +{ + unsigned char *data = p; + unsigned int i; + + assert(n == kfifo_peek_len(f)); + f->out += 2; + for (i = 0; i < n; i++) + data[i] = f->data[f->out++ % sizeof(f->data)]; + if (short_copy) { + short_copy = false; + return n - 1; + } + return n; +} +static void kfifo_skip(struct fifo *f) +{ + unsigned int n = kfifo_peek_len(f); + + assert(n); + f->out += n + 2; +} + +struct work_struct { void (*fn)(struct work_struct *); bool pending; }; +struct workqueue_struct { int unused; }; +static struct workqueue_struct queue; +static struct work_struct *queued; +#define INIT_WORK(w, cb) do { (w)->fn = (cb); (w)->pending = false; } while (0) +static bool queue_work(struct workqueue_struct *q, struct work_struct *w) +{ + bool fresh = !w->pending; + + assert(raw_depth && queue_live && q == &queue); + w->pending = true; + queued = w; + return fresh; +} +static void run_one(void) +{ + assert(queued && queued->pending && !in_irq && !raw_depth); + queued->pending = false; + runs++; + queued->fn(queued); +} +static void drain(void) +{ + unsigned int budget = 100; + + while (queued && queued->pending) { + assert(budget--); + run_one(); + } +} +static struct workqueue_struct *alloc_workqueue(const char *s, int f, int n) +{ + assert(!in_irq && !raw_depth && !queue_live); + queue_live = !fail_queue; + return queue_live ? &queue : NULL; +} +static void cancel_work_sync(struct work_struct *w) +{ + assert(!in_irq && !raw_depth && !handler_live && core_live); + w->pending = false; +} +static void destroy_workqueue(struct workqueue_struct *q) +{ + assert(!handler_live && (!queued || !queued->pending)); + queue_live = false; +} +static int mk_transport; +static int vsock_core_register(void *t, int flags) +{ + assert(queue_live && !core_live); + core_live = !fail_core; + return fail_core; +} +static void vsock_core_unregister(void *t) +{ + assert(core_live && !handler_live && (!queued || !queued->pending)); + core_live = false; +} +typedef void (*handler_t)(u32, u32, u64, void *, u32, void *); +static int mk_register_msg_handler(u32 type, handler_t fn, void *ctx) +{ + assert(core_live && queue_live && !handler_live); + handler_live = !fail_handler; + return fail_handler; +} +static int mk_unregister_msg_handler(u32 type, handler_t fn) +{ + assert(handler_live && core_live && queue_live); + if (unregister_hook) + unregister_hook(); + handler_live = false; + return 0; +} + +struct sk_buff { unsigned int len; unsigned char data[MK_MAX_DATA_SIZE]; }; +static struct sk_buff *alloc_skb(unsigned int len, int flags) +{ + assert(!in_irq && !raw_depth && flags == GFP_KERNEL && core_live); + assert(len <= MK_MAX_DATA_SIZE); + allocs++; + if (allocation_hook) { + void (*hook)(void) = allocation_hook; + + allocation_hook = NULL; + hook(); + } + if (fail_alloc) { + fail_alloc = false; + return NULL; + } + return calloc(1, sizeof(struct sk_buff)); +} +static void *skb_put(struct sk_buff *skb, unsigned int n) +{ + assert(!skb->len && n <= sizeof(skb->data)); + skb->len = n; + return skb->data; +} +static void kfree_skb(struct sk_buff *skb) +{ + assert(!in_irq && !raw_depth); + free(skb); +} +static unsigned int received[1024], lengths[1024]; +static void mk_vsock_rx_pkt(struct sk_buff *skb) +{ + struct virtio_vsock_hdr *h = (void *)skb->data; + unsigned int seq = le32toh(h->src_port), i; + + assert(!in_irq && !raw_depth && core_live && delivered < 1024); + for (i = sizeof(*h); i < skb->len; i++) + assert(skb->data[i] == (unsigned char)seq); + received[delivered] = seq; + lengths[delivered++] = skb->len; + free(skb); +} +#include "vsock-production.h" + +static void send_packet(unsigned int seq, unsigned int len, u32 claimed) +{ + unsigned char buffer[MK_MAX_DATA_SIZE + 1]; + struct virtio_vsock_hdr h = { .src_port = htole32(seq), + .len = htole32(claimed) }; + + assert(handler_live && !in_irq && !raw_depth && len <= MK_MAX_DATA_SIZE); + memset(buffer, seq, sizeof(buffer)); + memcpy(buffer + 1, &h, sizeof(h)); + in_irq = true; + mk_vsock_ipi_handler(MK_MSG_NETWORK, MK_NET_VSOCK_PKT, 0, + buffer + 1, len, NULL); + in_irq = false; + memset(buffer, 0xff, sizeof(buffer)); +} +static void inject(void) +{ + send_packet(42, sizeof(struct virtio_vsock_hdr), 0); +} +static void start(void) +{ + allocs = delivered = runs = 0; + queued = NULL; + EXPECT(!mk_vsock_transport_init()); +} + +int main(void) +{ + unsigned int i, header = sizeof(struct virtio_vsock_hdr); + + fail_queue = true; + EXPECT(mk_vsock_transport_init() == -ENOMEM); + EXPECT(!queue_live && !core_live && !handler_live); + fail_queue = false; + fail_core = -EIO; + EXPECT(mk_vsock_transport_init() == -EIO); + EXPECT(!queue_live && !core_live && !handler_live); + fail_core = 0; + fail_handler = -EINVAL; + EXPECT(mk_vsock_transport_init() == -EINVAL); + EXPECT(!queue_live && !core_live && !handler_live); + fail_handler = 0; + start(); + send_packet(1, header, 0); + send_packet(2, MK_VSOCK_MAX_PKT_SIZE, MK_VSOCK_MAX_PKT_SIZE - header); + EXPECT(!allocs && !delivered); + drain(); + EXPECT(delivered == 2 && allocs == 2); + EXPECT(received[0] == 1 && received[1] == 2); + EXPECT(lengths[0] == header && lengths[1] == MK_VSOCK_MAX_PKT_SIZE); + mk_vsock_transport_exit(); + + start(); + mk_vsock_ipi_handler(99, MK_NET_VSOCK_PKT, 0, NULL, UINT32_MAX, NULL); + mk_vsock_ipi_handler(MK_MSG_NETWORK, 99, 0, NULL, UINT32_MAX, NULL); + send_packet(1, 0, 0); + send_packet(2, header - 1, 0); + send_packet(3, MK_VSOCK_MAX_PKT_SIZE + 1, 0); + send_packet(4, header, UINT32_MAX); + send_packet(5, header + 1, 2); + EXPECT(!allocs && !queued); + send_packet(6, header + 8, 0); + drain(); + EXPECT(delivered == 1 && received[0] == 6); + mk_vsock_transport_exit(); + + start(); + for (i = 0; i < 65; i++) + send_packet(i, MK_VSOCK_MAX_PKT_SIZE, MK_VSOCK_MAX_PKT_SIZE - header); + EXPECT(!allocs); + drain(); + EXPECT(delivered == 64 && allocs == 64); + for (i = 0; i < 64; i++) + EXPECT(received[i] == i); + send_packet(70, MK_VSOCK_MAX_PKT_SIZE, MK_VSOCK_MAX_PKT_SIZE - header); + drain(); + EXPECT(delivered == 65 && received[64] == 70); + mk_vsock_transport_exit(); + + start(); + for (i = 0; i < 130; i++) + send_packet(i, header, 0); + run_one(); + EXPECT(delivered == 64 && queued->pending); + drain(); + EXPECT(delivered == 130 && runs == 3); + for (i = 0; i < 130; i++) + EXPECT(received[i] == i); + mk_vsock_transport_exit(); + + start(); + fail_alloc = true; + send_packet(1, header, 0); + send_packet(2, header, 0); + drain(); + EXPECT(allocs == 2 && delivered == 1 && received[0] == 2); + mk_vsock_transport_exit(); + start(); + short_copy = true; + send_packet(1, header, 0); + send_packet(2, header, 0); + drain(); + EXPECT(allocs == 2 && delivered == 1 && received[0] == 2); + mk_vsock_transport_exit(); + start(); + allocation_hook = inject; + send_packet(1, header, 0); + drain(); + EXPECT(delivered == 2 && received[0] == 1 && received[1] == 42); + tail_inject = true; + send_packet(3, header, 0); + drain(); + EXPECT(delivered == 4 && received[2] == 3 && received[3] == 99); + mk_vsock_transport_exit(); + + start(); + send_packet(1, header, 0); + unregister_hook = inject; + mk_vsock_transport_exit(); + unregister_hook = NULL; + EXPECT(!allocs && !delivered && !queue_live && !core_live && !handler_live); + start(); + send_packet(7, header, 0); + drain(); + EXPECT(delivered == 1 && received[0] == 7); + mk_vsock_transport_exit(); + printf("VSOCK receive: %u checks, %u failures (FIFO, socket and IRQ boundaries modeled)\n", + checks, failures); + return failures != 0; +} diff --git a/tools/testing/selftests/multikernel/vsock_rx_test.sh b/tools/testing/selftests/multikernel/vsock_rx_test.sh new file mode 100644 index 00000000000000..ad157fbcf5fe22 --- /dev/null +++ b/tools/testing/selftests/multikernel/vsock_rx_test.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-vsock-rx.XXXXXX") +trap 'rm -rf -- "$build"' EXIT HUP INT TERM +awk '/^#define MK_VSOCK_MAX_PKT_SIZE/ { print } + /^struct mk_vsock \{/ { copy = 1 } + /^static struct mk_vsock mk_vsock_dev;/ { print } + /^static void mk_vsock_(rx_work|ipi_handler)\(/ { copy = 1 } + /^static (int __init|void __exit) mk_vsock_transport_(init|exit)\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/net/vmw_vsock/mk_transport.c" > "$build/vsock-production.h" +awk '/^struct virtio_vsock_hdr \{/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/include/uapi/linux/virtio_vsock.h" > "$build/vsock-wire.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -Wno-unused-parameter \ + -I"$build" "$test_dir/vsock_rx_test.c" -o "$build/vsock_rx_test" +"$build/vsock_rx_test" diff --git a/tools/testing/selftests/multikernel/wake_slot_test.c b/tools/testing/selftests/multikernel/wake_slot_test.c new file mode 100644 index 00000000000000..3610156c066b11 --- /dev/null +++ b/tools/testing/selftests/multikernel/wake_slot_test.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Extracted production dispatch; mocked mapping is not native CPU proof. */ +#include +#include +#include +#include +#include +#include + +typedef uint32_t u32; +struct mk_spawn_context { + unsigned long trampoline_phys, trampoline_virt, + secondary_trampoline_phys; + unsigned long identity_cr3, kernel_entry, gs_base, stack, spawn_cr3; + u32 target_apic_id, flags, ready; +}; +struct mk_cpu_set { + bool present; + u32 id; +}; +struct mk_ident_pgtable { + int unused; +}; +static struct mk_ident_pgtable park_pgt; +struct mk_pool { + struct mk_cpu_set *cpus; + struct { + unsigned long slot_phys; + struct mk_spawn_context *slot; + struct mk_ident_pgtable *pgt; + } arch; + bool park_lock, cpus_lock; +}; +struct mk_instance { + unsigned long pool_slot_phys; + struct mk_cpu_set *cpus_on_slot; +}; +static struct mk_spawn_context own, local, adopted; +static struct mk_cpu_set free_cpus = { .id = 17 }, adopted_cpus = { .id = 17 }; +static struct mk_pool pool = { .cpus = &free_cpus, .arch.slot_phys = 0x9000 }; +static struct mk_pool *mk_pool; +static struct mk_instance parent = { .pool_slot_phys = 0xa000, + .cpus_on_slot = &adopted_cpus }, + *host_instance; +static unsigned int maps, unmaps, publishes, checks, failures; +static bool map_failure, publication_locked; +static bool park_trampoline_mapped, dispatch_fetch_mapped; +static int park_map_error; +static unsigned long mapped_phys; +static struct { + void *pgd; +} init_mm; +static struct { + struct { + unsigned long sp; + } thread; +} idle = { .thread.sp = 0x123000 }; +static unsigned char trampoline_blob[32]; +#define mk_secondary_trampoline (trampoline_blob + 8) +#define multikernel_relocate_kernel_start trampoline_blob +#define MEMREMAP_WB 1 +#define MK_SPAWN_F_SECONDARY 2 +#define __PAGE_KERNEL_LARGE_EXEC 0x123 +#define _PAGE_RW 2 +#define __va(p) ((void *)(p)) +#define __sme_pa(p) ((unsigned long)(p)) +#define pr_err(...) ((void)0) +#define lockdep_assert_held(p) assert(*(p)) +static inline void mutex_lock(bool *p) +{ + assert(!*p); + *p = true; +} +static inline void mutex_unlock(bool *p) +{ + assert(*p); + *p = false; +} +static bool mk_cpu_set_contains(struct mk_cpu_set *set, u32 cpu) +{ + assert(set != &free_cpus || pool.cpus_lock); + return set && set->present && set->id == cpu; +} +static void *memremap(unsigned long phys, size_t size, int flags) +{ + (void)flags; + assert(size == sizeof(struct mk_spawn_context)); + maps++; + mapped_phys = phys; + if (map_failure) + return NULL; + return phys == 0x9000 ? &local : phys == 0xa000 ? &adopted : NULL; +} +static void memunmap(void *pointer) +{ + assert(pointer == &local || pointer == &adopted); + unmaps++; +} +static struct mk_spawn_context *mk_own_context(void) +{ + return &own; +} +static int mk_build_trampoline_pgtable(unsigned long va, unsigned long other, + unsigned long phys, unsigned long *out) +{ + assert(phys == own.trampoline_phys); + *out = 0x111000; + return 0; +} +static int mk_add_2mb_mapping(void *pgd, unsigned long va, unsigned long phys, + unsigned long flags) +{ + assert(pgd == init_mm.pgd && phys == own.trampoline_phys && + !(flags & _PAGE_RW)); + return 0; +} +static inline int mk_ident_map_range(struct mk_ident_pgtable *pgt, + unsigned long start, unsigned long end) +{ + assert(pool.park_lock && pgt == &park_pgt); + assert(start == own.trampoline_phys && end == start + 4096); + if (!park_map_error) + park_trampoline_mapped = true; + return park_map_error; +} +#define PAGE_SIZE 4096 +static unsigned long per_cpu_offset(unsigned int cpu) +{ + return 0x321000 + cpu; +} +#define idle_task(cpu) (&idle) +#define multikernel_secondary_startup 0xabc000UL +#define cmpxchg(p, from, to) \ + do { \ + if (*(p) == (from)) \ + *(p) = (to); \ + } while (0) +#define smp_wmb() ((void)0) +#define WRITE_ONCE(p, v) \ + do { \ + publication_locked = !mk_pool || (mk_pool->park_lock && \ + mk_pool->cpus_lock); \ + dispatch_fetch_mapped = &(p) != &local.ready || \ + park_trampoline_mapped; \ + (p) = (v); \ + publishes++; \ + } while (0) + +/* Publication protocol itself is covered by mailbox_test.sh. */ +static int mk_slot_begin(struct mk_spawn_context *slot) +{ + if (slot->ready) + return -EBUSY; + slot->ready = 2; + return 0; +} + +static bool mk_slot_idle(struct mk_spawn_context *slot) +{ + return !slot || !slot->ready; +} + +static int mk_slot_wake(struct mk_spawn_context *slot, u32 cpu, const char *what) +{ + assert(slot->ready == 2 && slot->target_apic_id == cpu); + WRITE_ONCE(slot->ready, 1); + slot->ready = 0; + return 0; +} +#include "wake-production.h" + +static void expect(const char *name, bool passed) +{ + checks++; + if (!passed) { + fprintf(stderr, "FAIL %s\n", name); + failures++; + } +} +static void reset(void) +{ + assert(!pool.park_lock && !pool.cpus_lock); + memset(&own, 0, sizeof(own)); + memset(&local, 0, sizeof(local)); + memset(&adopted, 0, sizeof(adopted)); + own.trampoline_phys = 0x222000; + own.trampoline_virt = 0x333000; + init_mm.pgd = (void *)0x444000; + maps = unmaps = publishes = 0; + mapped_phys = 0; + publication_locked = map_failure = false; + park_trampoline_mapped = dispatch_fetch_mapped = false; + park_map_error = 0; + pool.park_lock = false; + pool.arch.slot_phys = 0x9000; + pool.arch.slot = &local; + pool.arch.pgt = &park_pgt; + free_cpus.present = adopted_cpus.present = false; + mk_pool = &pool; + host_instance = &parent; +} +int main(void) +{ + int ret; + reset(); + free_cpus.present = true; + adopted_cpus.present = + true; /* Local donation supersedes old parent tracking. */ + ret = multikernel_wakeup_secondary_cpu_64(17, 0, 2); + expect("local free CPU wakes on local pool slot", + !ret && publishes == 1 && !local.ready && !own.ready && !adopted.ready); + expect("local mapping paired", + maps == 1 && unmaps == 1 && mapped_phys == 0x9000); + expect("dispatch is child kernel, not park kernel", + local.trampoline_phys == own.trampoline_phys && + local.kernel_entry == multikernel_secondary_startup && + local.stack == idle.thread.sp && + local.spawn_cr3 == (unsigned long)init_mm.pgd && + local.gs_base == per_cpu_offset(2)); + expect("complete dispatch targets selected CPU and trampoline continuation", + local.target_apic_id == 17 && + local.flags == MK_SPAWN_F_SECONDARY && + local.secondary_trampoline_phys == + own.trampoline_phys + 8 && + local.identity_cr3 == 0x111000); + expect("initial instruction fetch mapped in old park table before publication", + dispatch_fetch_mapped); + expect("publication holds park lifetime lock", + publication_locked && !pool.park_lock); + reset(); + adopted_cpus.present = true; + ret = multikernel_wakeup_secondary_cpu_64(17, 0, 2); + expect("adopted parent-slot path preserved", + !ret && publishes == 1 && !adopted.ready && !local.ready && maps == unmaps); + reset(); + ret = multikernel_wakeup_secondary_cpu_64(17, 0, 2); + expect("ordinary secondary uses own context", + !ret && publishes == 1 && !own.ready && !maps && !unmaps); + reset(); + mk_pool = NULL; + ret = multikernel_wakeup_secondary_cpu_64(17, 0, 2); + expect("initial secondary works before baseline", !ret && publishes == 1 && !own.ready); + reset(); + free_cpus.present = true; + map_failure = true; + ret = multikernel_wakeup_secondary_cpu_64(17, 0, 2); + expect("failed local mapping does not publish or change free ownership", + ret == -ENODEV && !publishes && !unmaps && free_cpus.present && + !pool.park_lock); + reset(); + free_cpus.present = true; + pool.arch.slot_phys = 0; + ret = multikernel_wakeup_secondary_cpu_64(17, 0, 2); + expect("missing local slot cannot fall back to wrong mailbox", + ret == -ENODEV && !publishes && !own.ready && + free_cpus.present && !pool.park_lock); + reset(); + free_cpus.present = true; + park_map_error = -ENOMEM; + ret = multikernel_wakeup_secondary_cpu_64(17, 0, 2); + expect("park mapping failure leaves free ownership and publishes nothing", + ret == -ENOMEM && !publishes && free_cpus.present && + !pool.park_lock); + reset(); + free_cpus.present = true; + pool.arch.pgt = NULL; + ret = multikernel_wakeup_secondary_cpu_64(17, 0, 2); + expect("missing park table refuses dispatch without changing ownership", + ret == -ENODEV && !publishes && free_cpus.present && + !pool.park_lock); + printf("Wake slot: %u checks, %u failures (non-native mapping boundaries)\n", + checks, failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/multikernel/wake_slot_test.sh b/tools/testing/selftests/multikernel/wake_slot_test.sh new file mode 100644 index 00000000000000..b0c139e1d64cd9 --- /dev/null +++ b/tools/testing/selftests/multikernel/wake_slot_test.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu +test_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +src=$(CDPATH= cd -- "$test_dir/../../../.." && pwd) +build=$(mktemp -d "${TMPDIR:-/tmp}/mk-wake-slot.XXXXXX") +trap 'rm -r -- "$build"' EXIT HUP INT TERM +awk '/^static struct mk_spawn_context \*mk_wake_slot\(/ || + /^static void mk_put_wake_slot\(/ || + /^int multikernel_wakeup_secondary_cpu_64\(/ { copy = 1 } + copy { print } + copy && /^}/ { copy = 0 }' "$src/arch/x86/multikernel/spawn.c" > "$build/wake-production.h" +${CC:-cc} ${CFLAGS:-} -std=gnu11 -Wall -Wextra -Werror -Wno-unused-parameter \ + -I"$build" "$test_dir/wake_slot_test.c" -o "$build/wake_slot_test" +"$build/wake_slot_test"