diff options
Diffstat (limited to 'arch/riscv/kernel')
-rw-r--r-- | arch/riscv/kernel/asm-offsets.c | 3 | ||||
-rw-r--r-- | arch/riscv/kernel/cpu.c | 3 | ||||
-rw-r--r-- | arch/riscv/kernel/entry.S | 22 | ||||
-rw-r--r-- | arch/riscv/kernel/head.S | 33 | ||||
-rw-r--r-- | arch/riscv/kernel/irq.c | 19 | ||||
-rw-r--r-- | arch/riscv/kernel/perf_event.c | 4 | ||||
-rw-r--r-- | arch/riscv/kernel/reset.c | 15 | ||||
-rw-r--r-- | arch/riscv/kernel/setup.c | 6 | ||||
-rw-r--r-- | arch/riscv/kernel/signal.c | 6 | ||||
-rw-r--r-- | arch/riscv/kernel/smp.c | 61 | ||||
-rw-r--r-- | arch/riscv/kernel/smpboot.c | 22 | ||||
-rw-r--r-- | arch/riscv/kernel/stacktrace.c | 14 | ||||
-rw-r--r-- | arch/riscv/kernel/traps.c | 30 | ||||
-rw-r--r-- | arch/riscv/kernel/vdso/Makefile | 2 |
14 files changed, 115 insertions, 125 deletions
diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index dac98348c6a3..578bb5efc085 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -312,9 +312,6 @@ void asm_offsets(void) - offsetof(struct task_struct, thread.fstate.f[0]) ); - /* The assembler needs access to THREAD_SIZE as well. */ - DEFINE(ASM_THREAD_SIZE, THREAD_SIZE); - /* * We allocate a pt_regs on the stack when entering the kernel. This * ensures the alignment is sane. diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index cf2fca12414a..c8d2a3223099 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -136,8 +136,7 @@ static void c_stop(struct seq_file *m, void *v) static int c_show(struct seq_file *m, void *v) { unsigned long cpu_id = (unsigned long)v - 1; - struct device_node *node = of_get_cpu_node(cpuid_to_hartid_map(cpu_id), - NULL); + struct device_node *node = of_get_cpu_node(cpu_id, NULL); const char *compat, *isa, *mmu; seq_printf(m, "processor\t: %lu\n", cpu_id); diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index fd9b57c8b4ce..1c1ecc238cfa 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -37,11 +37,11 @@ * the kernel thread pointer. If we came from the kernel, sscratch * will contain 0, and we should continue on the current TP. */ - csrrw tp, sscratch, tp + csrrw tp, CSR_SSCRATCH, tp bnez tp, _save_context _restore_kernel_tpsp: - csrr tp, sscratch + csrr tp, CSR_SSCRATCH REG_S sp, TASK_TI_KERNEL_SP(tp) _save_context: REG_S sp, TASK_TI_USER_SP(tp) @@ -87,11 +87,11 @@ _save_context: li t0, SR_SUM | SR_FS REG_L s0, TASK_TI_USER_SP(tp) - csrrc s1, sstatus, t0 - csrr s2, sepc - csrr s3, sbadaddr - csrr s4, scause - csrr s5, sscratch + csrrc s1, CSR_SSTATUS, t0 + csrr s2, CSR_SEPC + csrr s3, CSR_STVAL + csrr s4, CSR_SCAUSE + csrr s5, CSR_SSCRATCH REG_S s0, PT_SP(sp) REG_S s1, PT_SSTATUS(sp) REG_S s2, PT_SEPC(sp) @@ -107,8 +107,8 @@ _save_context: .macro RESTORE_ALL REG_L a0, PT_SSTATUS(sp) REG_L a2, PT_SEPC(sp) - csrw sstatus, a0 - csrw sepc, a2 + csrw CSR_SSTATUS, a0 + csrw CSR_SEPC, a2 REG_L x1, PT_RA(sp) REG_L x3, PT_GP(sp) @@ -155,7 +155,7 @@ ENTRY(handle_exception) * Set sscratch register to 0, so that if a recursive exception * occurs, the exception vector knows it came from the kernel */ - csrw sscratch, x0 + csrw CSR_SSCRATCH, x0 /* Load the global pointer */ .option push @@ -248,7 +248,7 @@ resume_userspace: * Save TP into sscratch, so we can find the kernel data structures * again. */ - csrw sscratch, tp + csrw CSR_SSCRATCH, tp restore_all: RESTORE_ALL diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index fe884cd69abd..370c66ce187a 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -23,7 +23,8 @@ __INIT ENTRY(_start) /* Mask all interrupts */ - csrw sie, zero + csrw CSR_SIE, zero + csrw CSR_SIP, zero /* Load the global pointer */ .option push @@ -68,14 +69,10 @@ clear_bss_done: /* Restore C environment */ la tp, init_task sw zero, TASK_TI_CPU(tp) - - la sp, init_thread_union - li a0, ASM_THREAD_SIZE - add sp, sp, a0 + la sp, init_thread_union + THREAD_SIZE /* Start the kernel */ - mv a0, s0 - mv a1, s1 + mv a0, s1 call parse_dtb tail start_kernel @@ -89,7 +86,7 @@ relocate: /* Point stvec to virtual address of intruction after satp write */ la a0, 1f add a0, a0, a1 - csrw stvec, a0 + csrw CSR_STVEC, a0 /* Compute satp for kernel page tables, but don't load it yet */ la a2, swapper_pg_dir @@ -99,18 +96,20 @@ relocate: /* * Load trampoline page directory, which will cause us to trap to - * stvec if VA != PA, or simply fall through if VA == PA + * stvec if VA != PA, or simply fall through if VA == PA. We need a + * full fence here because setup_vm() just wrote these PTEs and we need + * to ensure the new translations are in use. */ la a0, trampoline_pg_dir srl a0, a0, PAGE_SHIFT or a0, a0, a1 sfence.vma - csrw sptbr, a0 + csrw CSR_SATP, a0 .align 2 1: /* Set trap vector to spin forever to help debug */ la a0, .Lsecondary_park - csrw stvec, a0 + csrw CSR_STVEC, a0 /* Reload the global pointer */ .option push @@ -118,8 +117,14 @@ relocate: la gp, __global_pointer$ .option pop - /* Switch to kernel page tables */ - csrw sptbr, a2 + /* + * Switch to kernel page tables. A full fence is necessary in order to + * avoid using the trampoline translations, which are only correct for + * the first superpage. Fetching the fence is guarnteed to work + * because that first superpage is translated the same way. + */ + csrw CSR_SATP, a2 + sfence.vma ret @@ -130,7 +135,7 @@ relocate: /* Set trap vector to spin forever to help debug */ la a3, .Lsecondary_park - csrw stvec, a3 + csrw CSR_STVEC, a3 slli a3, a0, LGREG la a1, __cpu_up_stack_pointer diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index 48e6b7db83a1..6d8659388c49 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -14,17 +14,9 @@ /* * Possible interrupt causes: */ -#define INTERRUPT_CAUSE_SOFTWARE 1 -#define INTERRUPT_CAUSE_TIMER 5 -#define INTERRUPT_CAUSE_EXTERNAL 9 - -/* - * The high order bit of the trap cause register is always set for - * interrupts, which allows us to differentiate them from exceptions - * quickly. The INTERRUPT_CAUSE_* macros don't contain that bit, so we - * need to mask it off. - */ -#define INTERRUPT_CAUSE_FLAG (1UL << (__riscv_xlen - 1)) +#define INTERRUPT_CAUSE_SOFTWARE IRQ_S_SOFT +#define INTERRUPT_CAUSE_TIMER IRQ_S_TIMER +#define INTERRUPT_CAUSE_EXTERNAL IRQ_S_EXT int arch_show_interrupts(struct seq_file *p, int prec) { @@ -37,7 +29,7 @@ asmlinkage void __irq_entry do_IRQ(struct pt_regs *regs) struct pt_regs *old_regs = set_irq_regs(regs); irq_enter(); - switch (regs->scause & ~INTERRUPT_CAUSE_FLAG) { + switch (regs->scause & ~SCAUSE_IRQ_FLAG) { case INTERRUPT_CAUSE_TIMER: riscv_timer_interrupt(); break; @@ -54,7 +46,8 @@ asmlinkage void __irq_entry do_IRQ(struct pt_regs *regs) handle_arch_irq(regs); break; default: - panic("unexpected interrupt cause"); + pr_alert("unexpected interrupt cause 0x%lx", regs->scause); + BUG(); } irq_exit(); diff --git a/arch/riscv/kernel/perf_event.c b/arch/riscv/kernel/perf_event.c index 667ee70defea..91626d9ae5f2 100644 --- a/arch/riscv/kernel/perf_event.c +++ b/arch/riscv/kernel/perf_event.c @@ -185,10 +185,10 @@ static inline u64 read_counter(int idx) switch (idx) { case RISCV_PMU_CYCLE: - val = csr_read(cycle); + val = csr_read(CSR_CYCLE); break; case RISCV_PMU_INSTRET: - val = csr_read(instret); + val = csr_read(CSR_INSTRET); break; default: WARN_ON_ONCE(idx < 0 || idx > RISCV_MAX_COUNTERS); diff --git a/arch/riscv/kernel/reset.c b/arch/riscv/kernel/reset.c index 2a53d26ffdd6..ed637aee514b 100644 --- a/arch/riscv/kernel/reset.c +++ b/arch/riscv/kernel/reset.c @@ -12,11 +12,15 @@ */ #include <linux/reboot.h> -#include <linux/export.h> #include <asm/sbi.h> -void (*pm_power_off)(void) = machine_power_off; -EXPORT_SYMBOL(pm_power_off); +static void default_power_off(void) +{ + sbi_shutdown(); + while (1); +} + +void (*pm_power_off)(void) = default_power_off; void machine_restart(char *cmd) { @@ -26,11 +30,10 @@ void machine_restart(char *cmd) void machine_halt(void) { - machine_power_off(); + pm_power_off(); } void machine_power_off(void) { - sbi_shutdown(); - while (1); + pm_power_off(); } diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 540a331d1376..d93bcce004e3 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -52,9 +52,11 @@ struct screen_info screen_info = { atomic_t hart_lottery; unsigned long boot_cpu_hartid; -void __init parse_dtb(unsigned int hartid, void *dtb) +void __init parse_dtb(phys_addr_t dtb_phys) { - if (early_init_dt_scan(__va(dtb))) + void *dtb = __va(dtb_phys); + + if (early_init_dt_scan(dtb)) return; pr_err("No DTB passed to the kernel\n"); diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c index 837e1646091a..804d6ee4f3c5 100644 --- a/arch/riscv/kernel/signal.c +++ b/arch/riscv/kernel/signal.c @@ -234,6 +234,9 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) /* Are we from a system call? */ if (regs->scause == EXC_SYSCALL) { + /* Avoid additional syscall restarting via ret_from_exception */ + regs->scause = -1UL; + /* If so, check system call restarting.. */ switch (regs->a0) { case -ERESTART_RESTARTBLOCK: @@ -272,6 +275,9 @@ static void do_signal(struct pt_regs *regs) /* Did we come from a system call? */ if (regs->scause == EXC_SYSCALL) { + /* Avoid additional syscall restarting via ret_from_exception */ + regs->scause = -1UL; + /* Restart the system call - no handlers present */ switch (regs->a0) { case -ERESTARTNOHAND: diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index 0c41d07ec281..b2537ffa855c 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -42,7 +42,7 @@ unsigned long __cpuid_to_hartid_map[NR_CPUS] = { void __init smp_setup_processor_id(void) { - cpuid_to_hartid_map(0) = boot_cpu_hartid; + cpuid_to_hartid_map(0) = boot_cpu_hartid; } /* A collection of single bit ipi messages. */ @@ -53,7 +53,7 @@ static struct { int riscv_hartid_to_cpuid(int hartid) { - int i = -1; + int i; for (i = 0; i < NR_CPUS; i++) if (cpuid_to_hartid_map(i) == hartid) @@ -70,6 +70,12 @@ void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out) for_each_cpu(cpu, in) cpumask_set_cpu(cpuid_to_hartid_map(cpu), out); } + +bool arch_match_cpu_phys_id(int cpu, u64 phys_id) +{ + return phys_id == cpuid_to_hartid_map(cpu); +} + /* Unsupported */ int setup_profiling_timer(unsigned int multiplier) { @@ -89,7 +95,7 @@ void riscv_software_interrupt(void) unsigned long *stats = ipi_data[smp_processor_id()].stats; /* Clear pending IPI */ - csr_clear(sip, SIE_SSIE); + csr_clear(CSR_SIP, SIE_SSIE); while (true) { unsigned long ops; @@ -199,52 +205,3 @@ void smp_send_reschedule(int cpu) send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); } -/* - * Performs an icache flush for the given MM context. RISC-V has no direct - * mechanism for instruction cache shoot downs, so instead we send an IPI that - * informs the remote harts they need to flush their local instruction caches. - * To avoid pathologically slow behavior in a common case (a bunch of - * single-hart processes on a many-hart machine, ie 'make -j') we avoid the - * IPIs for harts that are not currently executing a MM context and instead - * schedule a deferred local instruction cache flush to be performed before - * execution resumes on each hart. - */ -void flush_icache_mm(struct mm_struct *mm, bool local) -{ - unsigned int cpu; - cpumask_t others, hmask, *mask; - - preempt_disable(); - - /* Mark every hart's icache as needing a flush for this MM. */ - mask = &mm->context.icache_stale_mask; - cpumask_setall(mask); - /* Flush this hart's I$ now, and mark it as flushed. */ - cpu = smp_processor_id(); - cpumask_clear_cpu(cpu, mask); - local_flush_icache_all(); - - /* - * Flush the I$ of other harts concurrently executing, and mark them as - * flushed. - */ - cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu)); - local |= cpumask_empty(&others); - if (mm != current->active_mm || !local) { - cpumask_clear(&hmask); - riscv_cpuid_to_hartid_mask(&others, &hmask); - sbi_remote_fence_i(hmask.bits); - } else { - /* - * It's assumed that at least one strongly ordered operation is - * performed on this hart between setting a hart's cpumask bit - * and scheduling this MM context on that hart. Sending an SBI - * remote message will do this, but in the case where no - * messages are sent we still need to order this hart's writes - * with flush_icache_deferred(). - */ - smp_mb(); - } - - preempt_enable(); -} diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index eb533b5c2c8c..7a0b62252524 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -47,6 +47,17 @@ void __init smp_prepare_boot_cpu(void) void __init smp_prepare_cpus(unsigned int max_cpus) { + int cpuid; + + /* This covers non-smp usecase mandated by "nosmp" option */ + if (max_cpus == 0) + return; + + for_each_possible_cpu(cpuid) { + if (cpuid == smp_processor_id()) + continue; + set_cpu_present(cpuid, true); + } } void __init setup_smp(void) @@ -73,12 +84,19 @@ void __init setup_smp(void) } cpuid_to_hartid_map(cpuid) = hart; - set_cpu_possible(cpuid, true); - set_cpu_present(cpuid, true); cpuid++; } BUG_ON(!found_boot_cpu); + + if (cpuid > nr_cpu_ids) + pr_warn("Total number of cpus [%d] is greater than nr_cpus option value [%d]\n", + cpuid, nr_cpu_ids); + + for (cpuid = 1; cpuid < nr_cpu_ids; cpuid++) { + if (cpuid_to_hartid_map(cpuid) != INVALID_HARTID) + set_cpu_possible(cpuid, true); + } } int __cpu_up(unsigned int cpu, struct task_struct *tidle) diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index 4d403274c2e8..e80a5e8da119 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -33,9 +33,9 @@ static void notrace walk_stackframe(struct task_struct *task, unsigned long fp, sp, pc; if (regs) { - fp = GET_FP(regs); - sp = GET_USP(regs); - pc = GET_IP(regs); + fp = frame_pointer(regs); + sp = user_stack_pointer(regs); + pc = instruction_pointer(regs); } else if (task == NULL || task == current) { const register unsigned long current_sp __asm__ ("sp"); fp = (unsigned long)__builtin_frame_address(0); @@ -64,12 +64,8 @@ static void notrace walk_stackframe(struct task_struct *task, frame = (struct stackframe *)fp - 1; sp = fp; fp = frame->fp; -#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR pc = ftrace_graph_ret_addr(current, NULL, frame->ra, (unsigned long *)(fp - 8)); -#else - pc = frame->ra - 0x4; -#endif } } @@ -82,8 +78,8 @@ static void notrace walk_stackframe(struct task_struct *task, unsigned long *ksp; if (regs) { - sp = GET_USP(regs); - pc = GET_IP(regs); + sp = user_stack_pointer(regs); + pc = instruction_pointer(regs); } else if (task == NULL || task == current) { const register unsigned long current_sp __asm__ ("sp"); sp = current_sp; diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 24a9333dda2c..3d1a651dc54c 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -70,7 +70,7 @@ void do_trap(struct pt_regs *regs, int signo, int code, && printk_ratelimit()) { pr_info("%s[%d]: unhandled signal %d code 0x%x at 0x" REG_FMT, tsk->comm, task_pid_nr(tsk), signo, code, addr); - print_vma_addr(KERN_CONT " in ", GET_IP(regs)); + print_vma_addr(KERN_CONT " in ", instruction_pointer(regs)); pr_cont("\n"); show_regs(regs); } @@ -118,6 +118,17 @@ DO_ERROR_INFO(do_trap_ecall_s, DO_ERROR_INFO(do_trap_ecall_m, SIGILL, ILL_ILLTRP, "environment call from M-mode"); +#ifdef CONFIG_GENERIC_BUG +static inline unsigned long get_break_insn_length(unsigned long pc) +{ + bug_insn_t insn; + + if (probe_kernel_address((bug_insn_t *)pc, insn)) + return 0; + return (((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) ? 4UL : 2UL); +} +#endif /* CONFIG_GENERIC_BUG */ + asmlinkage void do_trap_break(struct pt_regs *regs) { #ifdef CONFIG_GENERIC_BUG @@ -129,8 +140,8 @@ asmlinkage void do_trap_break(struct pt_regs *regs) case BUG_TRAP_TYPE_NONE: break; case BUG_TRAP_TYPE_WARN: - regs->sepc += sizeof(bug_insn_t); - return; + regs->sepc += get_break_insn_length(regs->sepc); + break; case BUG_TRAP_TYPE_BUG: die(regs, "Kernel BUG"); } @@ -145,11 +156,14 @@ int is_valid_bugaddr(unsigned long pc) { bug_insn_t insn; - if (pc < PAGE_OFFSET) + if (pc < VMALLOC_START) return 0; if (probe_kernel_address((bug_insn_t *)pc, insn)) return 0; - return (insn == __BUG_INSN); + if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) + return (insn == __BUG_INSN_32); + else + return ((insn & __COMPRESSED_INSN_MASK) == __BUG_INSN_16); } #endif /* CONFIG_GENERIC_BUG */ @@ -159,9 +173,9 @@ void __init trap_init(void) * Set sup0 scratch register to 0, indicating to exception vector * that we are presently executing in the kernel */ - csr_write(sscratch, 0); + csr_write(CSR_SSCRATCH, 0); /* Set the exception vector address */ - csr_write(stvec, &handle_exception); + csr_write(CSR_STVEC, &handle_exception); /* Enable all interrupts */ - csr_write(sie, -1); + csr_write(CSR_SIE, -1); } diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index fec62b24df89..b07b765f312a 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -36,7 +36,7 @@ $(obj)/vdso.so.dbg: $(src)/vdso.lds $(obj-vdso) FORCE # these symbols in the kernel code rather than hand-coded addresses. SYSCFLAGS_vdso.so.dbg = -shared -s -Wl,-soname=linux-vdso.so.1 \ - $(call cc-ldoption, -Wl$(comma)--hash-style=both) + -Wl,--hash-style=both $(obj)/vdso-dummy.o: $(src)/vdso.lds $(obj)/rt_sigreturn.o FORCE $(call if_changed,vdsold) |