Callers of global_doorbell and core_doorbell must now be aware of their
capabilities and which addressing mode their msgsnd instruction uses,
and use the correct one.
---
arch/powerpc/include/asm/dbell.h | 41 ++++++++++++++++++++++++++----------
arch/powerpc/include/asm/smp.h | 1 +
arch/powerpc/kernel/dbell.c | 11 +++++-----
arch/powerpc/kernel/smp.c | 13 +++++++++---
arch/powerpc/platforms/powernv/smp.c | 18 ++++++++++++++--
arch/powerpc/platforms/pseries/smp.c | 8 ++++++-
6 files changed, 69 insertions(+), 23 deletions(-)
diff --git a/arch/powerpc/include/asm/dbell.h b/arch/powerpc/include/asm/dbell.h
index 78b8ba575ec2..0ce54014632b 100644
--- a/arch/powerpc/include/asm/dbell.h
+++ b/arch/powerpc/include/asm/dbell.h
@@ -38,10 +38,24 @@ enum ppc_dbell {
static inline void _ppc_msgsnd(u32 msg)
{
- if (cpu_has_feature(CPU_FTR_HVMODE))
- __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
- else
- __asm__ __volatile__ (PPC_MSGSNDP(%0) : : "r" (msg));
+ __asm__ __volatile__ (ASM_FTR_IFSET(PPC_MSGSND(%1), PPC_MSGSNDP(%1), %0)
+ : : "i" (CPU_FTR_HVMODE), "r" (msg));
+}
+
+/* sync before sending message */
+static inline void ppc_msgsnd_sync(void)
+{
+ /* Could use lwsync for msgsndp? (does not order non-cacheable) */
+ __asm__ __volatile__ ("sync" : : : "memory");
+}
+
+/* sync after taking message interrupt */
+static inline void ppc_msgsync(void)
+{
+ /* sync is not required when taking messages from the same core */
+ __asm__ __volatile__ (ASM_FTR_IFSET(PPC_MSGSYNC, "", %0)
+ : : "i" (CPU_FTR_ARCH_300|CPU_FTR_HVMODE)
+ : "memory");
}
#else /* CONFIG_PPC_BOOK3S */
@@ -55,15 +69,20 @@ static inline void _ppc_msgsnd(u32 msg)
#endif /* CONFIG_PPC_BOOK3S */
+/*
+ * Doorbells must only be used if CPU_FTR_DBELL is available.
+ *
+ * Global doorbell IPIs must only be used on CPUs that address the msgsnd
+ * (or msgsndp if in HVMODE) with PIR (hard_smp_processor_id)
+ */
extern void global_doorbell_cause_ipi(int cpu);
+
+/*
+ * Core doorbell IPIs must only be used on CPUs that address the msgsnd
+ * (or msgsndp if in HVMODE) with TIR (cpu_thread_in_core)
+ */
extern void core_doorbell_cause_ipi(int cpu);
-extern int __try_core_doorbell_cause_ipi(int cpu);
-static inline int try_core_doorbell_cause_ipi(int cpu)
-{
- if (!cpu_has_feature(CPU_FTR_DBELL))
- return 0;
- return __try_core_doorbell_cause_ipi(cpu);
-}
+extern int try_core_doorbell_cause_ipi(int cpu);
extern void doorbell_exception(struct pt_regs *regs);
static inline void ppc_msgsnd(enum ppc_dbell type, u32 flags, u32 tag)
diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h
index 0ada12e61fd7..0ee8a6cb1d87 100644
--- a/arch/powerpc/include/asm/smp.h
+++ b/arch/powerpc/include/asm/smp.h
@@ -128,6 +128,7 @@ extern const char *smp_ipi_name[];
extern void smp_muxed_ipi_message_pass(int cpu, int msg);
extern void smp_muxed_ipi_set_message(int cpu, int msg);
extern irqreturn_t smp_ipi_demux(void);
+extern irqreturn_t smp_ipi_demux_relaxed(void);
void smp_init_pSeries(void);
void smp_init_cell(void);
diff --git a/arch/powerpc/kernel/dbell.c b/arch/powerpc/kernel/dbell.c
index 4df23ba7c3d3..95409bca76b0 100644
--- a/arch/powerpc/kernel/dbell.c
+++ b/arch/powerpc/kernel/dbell.c
@@ -34,18 +34,15 @@ void core_doorbell_cause_ipi(int cpu)
{
u32 tag;
- if (cpu_has_feature(CPU_FTR_ARCH_300) &&
cpu_has_feature(CPU_FTR_HVMODE))
- global_doorbell_cause_ipi(cpu);
-
tag = cpu_thread_in_core(cpu);
kvmppc_set_host_ipi(cpu, 1);
/* Order previous accesses vs. msgsnd, which is treated as a store */
- mb();
+ ppc_msgsnd_sync();
ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag);
}
-int __try_core_doorbell_cause_ipi(int cpu)
+int try_core_doorbell_cause_ipi(int cpu)
{
int this_cpu = get_cpu();
int ret = 0;
@@ -66,12 +63,14 @@ void doorbell_exception(struct pt_regs *regs)
irq_enter();
+ ppc_msgsync();
+
may_hard_irq_enable();
kvmppc_set_host_ipi(smp_processor_id(), 0);
__this_cpu_inc(irq_stat.doorbell_irqs);
- smp_ipi_demux();
+ smp_ipi_demux_relaxed(); /* already performed the barrier */
irq_exit();
set_irq_regs(old_regs);
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index bb882fa1f762..fd2441591b81 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -246,11 +246,18 @@ void smp_muxed_ipi_message_pass(int cpu, int msg)
irqreturn_t smp_ipi_demux(void)
{
- struct cpu_messages *info = this_cpu_ptr(&ipi_message);
- unsigned long all;
-
mb(); /* order any irq clear */
+ return smp_ipi_demux_relaxed();
+}
+
+/* sync-free variant. Callers should ensure synchronization */
+irqreturn_t smp_ipi_demux_relaxed(void)
+{
+ struct cpu_messages *info;
+ unsigned long all;
+
+ info = this_cpu_ptr(&ipi_message);
do {
all = xchg(&info->messages, 0);
#if defined(CONFIG_KVM_XICS) && defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE)
diff --git a/arch/powerpc/platforms/powernv/smp.c
b/arch/powerpc/platforms/powernv/smp.c
index e796c97914c8..660212d3c2d9 100644
--- a/arch/powerpc/platforms/powernv/smp.c
+++ b/arch/powerpc/platforms/powernv/smp.c
@@ -254,10 +254,24 @@ static void pnv_cause_ipi(int cpu)
icp_ops->cause_ipi(cpu);
}
+static __init void pnv_smp_probe(void)
+{
+ xics_smp_probe();
+
+ if (cpu_has_feature(CPU_FTR_DBELL)) {
+ if (cpu_has_feature(CPU_FTR_ARCH_300))
+ smp_ops->cause_ipi = global_doorbell_cause_ipi;
+ else
+ smp_ops->cause_ipi = pnv_cause_ipi;
+ } else {
+ smp_ops->cause_ipi = icp_ops->cause_ipi;
+ }
+}
+
static struct smp_ops_t pnv_smp_ops = {
.message_pass = NULL, /* Use smp_muxed_ipi_message_pass */
- .cause_ipi = pnv_cause_ipi,
- .probe = xics_smp_probe,
+ .cause_ipi = NULL, /* Filled at runtime by pnv_smp_probe() */
+ .probe = pnv_smp_probe,
.kick_cpu = pnv_smp_kick_cpu,
.setup_cpu = pnv_smp_setup_cpu,
.cpu_bootable = pnv_cpu_bootable,
diff --git a/arch/powerpc/platforms/pseries/smp.c
b/arch/powerpc/platforms/pseries/smp.c
index a26b3158555e..87b5700b217b 100644
--- a/arch/powerpc/platforms/pseries/smp.c
+++ b/arch/powerpc/platforms/pseries/smp.c
@@ -190,11 +190,17 @@ static void smp_pseries_cause_ipi(int cpu)
static __init void pSeries_smp_probe(void)
{
xics_smp_probe();
+
+ if (cpu_has_feature(CPU_FTR_DBELL))
+ smp_ops->cause_ipi = pseries_cause_ipi;
+ else
+ smp_ops->cause_ipi = icp_ops->cause_ipi;
+
}
static struct smp_ops_t pseries_smp_ops = {
.message_pass = NULL, /* Use smp_muxed_ipi_message_pass */
- .cause_ipi = smp_pseries_cause_ipi,
+ .cause_ipi = NULL, /* Filled at runtime by pSeries_smp_probe */
.probe = pSeries_smp_probe,
.kick_cpu = smp_pSeries_kick_cpu,
.setup_cpu = smp_setup_cpu,
--
2.11.0
