On Fri, Dec 6, 2019 at 2:50 PM Rick Chen <rickche...@gmail.com> wrote:
>
> Hi Lukas,
>
> > From: Lukas Auer [mailto:lukas.a...@aisec.fraunhofer.de]
> > Sent: Wednesday, December 04, 2019 5:40 AM
> > To: u-boot@lists.denx.de
> > Cc: Rick Jian-Zhi Chen(陳建志); Anup Patel; Bin Meng; Lukas Auer; Anup Patel;
> > Atish Patra; Marcus Comstedt
> > Subject: [PATCH 3/4] riscv: add option to wait for ack from secondary harts
> > in smp functions
> >
> > Add a wait option to smp_call_function() to wait for the secondary harts to
> > acknowledge the call-function request. The request is considered to be
> > acknowledged once each secondary hart has cleared the corresponding IPI.
> >
> > As part of the call-function request, the secondary harts invalidate the
> > instruction cache after clearing the IPI. This adds a delay between
> > acknowledgment (clear IPI) and fulfillment (call function) of the request.
> > We want to use the acknowledgment to be able to judge when the request has
> > been completed. Remove the delay by clearing the IPI after cache
> > invalidation and just before calling the function from the request.
> >
> > Signed-off-by: Lukas Auer <lukas.a...@aisec.fraunhofer.de>
> > ---
> >
>
> Reviewed-by: Rick Chen <r...@andestech.com>
> Tested-by: Rick Chen <r...@andestech.com>
This is inline what we last discussed. Looks good to me.
Reviewed-by: Anup Patel <anup.pa...@wdc.com>
Regards,
Anup
>
> > arch/riscv/cpu/start.S | 2 ++
> > arch/riscv/include/asm/smp.h | 3 ++-
> > arch/riscv/lib/bootm.c | 2 +-
> > arch/riscv/lib/smp.c | 31 ++++++++++++++++++++++---------
> > arch/riscv/lib/spl.c | 2 +-
> > common/spl/spl_opensbi.c | 2 +-
> > 6 files changed, 29 insertions(+), 13 deletions(-)
> >
> > diff --git a/arch/riscv/cpu/start.S b/arch/riscv/cpu/start.S index
> > 0a2ce6d691..60631638dd 100644
> > --- a/arch/riscv/cpu/start.S
> > +++ b/arch/riscv/cpu/start.S
> > @@ -197,6 +197,7 @@ spl_secondary_hart_stack_gd_setup:
> > la a0, secondary_hart_relocate
> > mv a1, s0
> > mv a2, s0
> > + mv a3, zero
> > jal smp_call_function
> >
> > /* hang if relocation of secondary harts has failed */ @@ -337,6
> > +338,7 @@ relocate_secondary_harts:
> >
> > mv a1, s2
> > mv a2, s3
> > + mv a3, zero
> > jal smp_call_function
> >
> > /* hang if relocation of secondary harts has failed */ diff --git
> > a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index
> > bc863fdbaf..74de92ed13 100644
> > --- a/arch/riscv/include/asm/smp.h
> > +++ b/arch/riscv/include/asm/smp.h
> > @@ -46,8 +46,9 @@ void handle_ipi(ulong hart);
> > * @addr: Address of function
> > * @arg0: First argument of function
> > * @arg1: Second argument of function
> > + * @wait: Wait for harts to acknowledge request
> > * @return 0 if OK, -ve on error
> > */
> > -int smp_call_function(ulong addr, ulong arg0, ulong arg1);
> > +int smp_call_function(ulong addr, ulong arg0, ulong arg1, int wait);
> >
> > #endif
> > diff --git a/arch/riscv/lib/bootm.c b/arch/riscv/lib/bootm.c index
> > efbd3e23e7..e96137a50c 100644
> > --- a/arch/riscv/lib/bootm.c
> > +++ b/arch/riscv/lib/bootm.c
> > @@ -99,7 +99,7 @@ static void boot_jump_linux(bootm_headers_t *images, int
> > flag)
> > if (IMAGE_ENABLE_OF_LIBFDT && images->ft_len) { #ifdef
> > CONFIG_SMP
> > ret = smp_call_function(images->ep,
> > - (ulong)images->ft_addr, 0);
> > + (ulong)images->ft_addr, 0,
> > 0);
> > if (ret)
> > hang();
> > #endif
> > diff --git a/arch/riscv/lib/smp.c b/arch/riscv/lib/smp.c index
> > 6ff0de4b74..d575e904c2 100644
> > --- a/arch/riscv/lib/smp.c
> > +++ b/arch/riscv/lib/smp.c
> > @@ -43,11 +43,11 @@ extern int riscv_clear_ipi(int hart);
> > */
> > extern int riscv_get_ipi(int hart, int *pending);
> >
> > -static int send_ipi_many(struct ipi_data *ipi)
> > +static int send_ipi_many(struct ipi_data *ipi, int wait)
> > {
> > ofnode node, cpus;
> > u32 reg;
> > - int ret;
> > + int ret, pending;
> >
> > cpus = ofnode_path("/cpus");
> > if (!ofnode_valid(cpus)) {
> > @@ -90,6 +90,15 @@ static int send_ipi_many(struct ipi_data *ipi)
> > pr_err("Cannot send IPI to hart %d\n", reg);
> > return ret;
> > }
> > +
> > + if (wait) {
> > + pending = 1;
> > + while (pending) {
> > + ret = riscv_get_ipi(reg, &pending);
> > + if (ret)
> > + return ret;
> > + }
> > + }
> > }
> >
> > return 0;
> > @@ -103,21 +112,25 @@ void handle_ipi(ulong hart)
> > if (hart >= CONFIG_NR_CPUS)
> > return;
> >
> > + __smp_mb();
> > +
> > + smp_function = (void (*)(ulong, ulong,
> > ulong))gd->arch.ipi[hart].addr;
> > + invalidate_icache_all();
> > +
> > + /*
> > + * Clear the IPI to acknowledge the request before jumping to the
> > + * requested function.
> > + */
> > ret = riscv_clear_ipi(hart);
> > if (ret) {
> > pr_err("Cannot clear IPI of hart %ld\n", hart);
> > return;
> > }
> >
> > - __smp_mb();
> > -
> > - smp_function = (void (*)(ulong, ulong,
> > ulong))gd->arch.ipi[hart].addr;
> > - invalidate_icache_all();
> > -
> > smp_function(hart, gd->arch.ipi[hart].arg0,
> > gd->arch.ipi[hart].arg1); }
> >
> > -int smp_call_function(ulong addr, ulong arg0, ulong arg1)
> > +int smp_call_function(ulong addr, ulong arg0, ulong arg1, int wait)
> > {
> > int ret = 0;
> > struct ipi_data ipi;
> > @@ -126,7 +139,7 @@ int smp_call_function(ulong addr, ulong arg0, ulong
> > arg1)
> > ipi.arg0 = arg0;
> > ipi.arg1 = arg1;
> >
> > - ret = send_ipi_many(&ipi);
> > + ret = send_ipi_many(&ipi, wait);
> >
> > return ret;
> > }
> > diff --git a/arch/riscv/lib/spl.c b/arch/riscv/lib/spl.c index
> > bea8695987..fd3c2efc5b 100644
> > --- a/arch/riscv/lib/spl.c
> > +++ b/arch/riscv/lib/spl.c
> > @@ -40,7 +40,7 @@ void __noreturn jump_to_image_no_args(struct
> > spl_image_info *spl_image)
> >
> > debug("image entry point: 0x%lX\n", spl_image->entry_point);
> > #ifdef CONFIG_SMP
> > - ret = smp_call_function(spl_image->entry_point, (ulong)fdt_blob, 0);
> > + ret = smp_call_function(spl_image->entry_point, (ulong)fdt_blob, 0,
> > +0);
> > if (ret)
> > hang();
> > #endif
> > diff --git a/common/spl/spl_opensbi.c b/common/spl/spl_opensbi.c index
> > 79ee7edcf9..91a411a3db 100644
> > --- a/common/spl/spl_opensbi.c
> > +++ b/common/spl/spl_opensbi.c
> > @@ -77,7 +77,7 @@ void spl_invoke_opensbi(struct spl_image_info *spl_image)
> > #ifdef CONFIG_SMP
> > ret = smp_call_function((ulong)spl_image->entry_point,
> > (ulong)spl_image->fdt_addr,
> > - (ulong)&opensbi_info);
> > + (ulong)&opensbi_info, 0);
> > if (ret)
> > hang();
> > #endif
> > --
> > 2.21.0
> >
