https://github.com/Lukacma updated
https://github.com/llvm/llvm-project/pull/100128
>From ea280c4177aa64c29a921ea40c95fc7024ce24ed Mon Sep 17 00:00:00 2001
From: Marian Lukac <marian.lu...@arm.com>
Date: Tue, 23 Jul 2024 14:11:49 +0000
Subject: [PATCH 1/2] [AArch64] Implement intrinsics for SME2 FSCALE
---
clang/include/clang/Basic/arm_sve.td | 10 +
.../acle_sme2_fp8_scale.c | 452 ++++++++++++++++++
llvm/include/llvm/IR/IntrinsicsAArch64.td | 25 +
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 28 ++
.../CodeGen/AArch64/sme2-intrinsics-fscale.ll | 186 +++++++
5 files changed, 701 insertions(+)
create mode 100644
clang/test/CodeGen/aarch64-fp8-intrinsics/acle_sme2_fp8_scale.c
create mode 100644 llvm/test/CodeGen/AArch64/sme2-intrinsics-fscale.ll
diff --git a/clang/include/clang/Basic/arm_sve.td
b/clang/include/clang/Basic/arm_sve.td
index 94c093d8911562..2fb93e091aa5c1 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -2377,6 +2377,16 @@ let SVETargetGuard = InvalidMode, SMETargetGuard =
"sme2" in {
def SVUUNPK_X4 : SInst<"svunpk_{d}[_{3}_x4]", "42.h", "UsUiUl", MergeNone,
"aarch64_sve_uunpk_x4", [IsStreaming], []>;
}
+//
+// Multi-vector scaling
+//
+let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2,fp8" in {
+ def FSCALE_SINGLE_X2 : Inst<"svscale[_single_{d}_x2]", "22x", "fhd",
MergeNone, "aarch64_sme_fp8_scale_single_x2", [IsStreaming],[]>;
+ def FSCALE_SINGLE_X4 : Inst<"svscale[_single_{d}_x4]", "44x", "fhd",
MergeNone, "aarch64_sme_fp8_scale_single_x4", [IsStreaming],[]>;
+ def FSCALE_X2 : Inst<"svscale[_{d}_x2]", "222.x", "fhd", MergeNone,
"aarch64_sme_fp8_scale_x2", [IsStreaming],[]>;
+ def FSCALE_X4 : Inst<"svscale[_{d}_x4]", "444.x", "fhd", MergeNone,
"aarch64_sme_fp8_scale_x4", [IsStreaming],[]>;
+}
+
let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in {
// == BFloat16 multiply-subtract ==
def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone,
"aarch64_sve_bfmlslb", [IsOverloadNone, VerifyRuntimeMode], []>;
diff --git a/clang/test/CodeGen/aarch64-fp8-intrinsics/acle_sme2_fp8_scale.c
b/clang/test/CodeGen/aarch64-fp8-intrinsics/acle_sme2_fp8_scale.c
new file mode 100644
index 00000000000000..b733e772ba3075
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-fp8-intrinsics/acle_sme2_fp8_scale.c
@@ -0,0 +1,452 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme
-target-feature +sme2 -target-feature +fp8 -disable-O0-optnone -Werror -Wall
-emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme
-target-feature +sme2 -target-feature +fp8 -disable-O0-optnone -Werror -Wall
-emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
-check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu
-target-feature +sme -target-feature +sme2 -target-feature +fp8
-disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S
-passes=mem2reg,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu
-target-feature +sme -target-feature +sme2 -target-feature +fp8
-disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S
-passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme
-target-feature +sme2 -target-feature +fp8 -S -disable-O0-optnone -Werror -Wall
-o /dev/null %s
+#include <arm_sme.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED) A1
+#else
+#define SVE_ACLE_FUNC(A1,A2) A1##A2
+#endif
+
+
+// Single x2
+// CHECK-LABEL: @test_svscale_single_f16_x2(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[OP1:%.*]], i64 0)
+// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[OP1]], i64 8)
+// CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8
x half> } @llvm.aarch64.sme.fp8.scale.single.x2.nxv8f16(<vscale x 8 x half>
[[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x i16> [[OP2:%.*]])
+// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x
8 x half> } [[TMP2]], 0
+// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x half>
@llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> poison, <vscale x 8 x
half> [[TMP3]], i64 0)
+// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x
8 x half> } [[TMP2]], 1
+// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x half>
@llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP4]], <vscale x 8
x half> [[TMP5]], i64 8)
+// CHECK-NEXT: ret <vscale x 16 x half> [[TMP6]]
+//
+// CPP-CHECK-LABEL:
@_Z26test_svscale_single_f16_x213svfloat16x2_tu11__SVInt16_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[OP1:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[OP1]], i64 8)
+// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 8 x half>, <vscale
x 8 x half> } @llvm.aarch64.sme.fp8.scale.single.x2.nxv8f16(<vscale x 8 x half>
[[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x i16> [[OP2:%.*]])
+// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 8 x half>,
<vscale x 8 x half> } [[TMP2]], 0
+// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 16 x half>
@llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> poison, <vscale x 8 x
half> [[TMP3]], i64 0)
+// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 8 x half>,
<vscale x 8 x half> } [[TMP2]], 1
+// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x half>
@llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP4]], <vscale x 8
x half> [[TMP5]], i64 8)
+// CPP-CHECK-NEXT: ret <vscale x 16 x half> [[TMP6]]
+//
+svfloat16x2_t test_svscale_single_f16_x2(svfloat16x2_t op1, svint16_t op2)
__arm_streaming
+{
+ return SVE_ACLE_FUNC(svscale,_single_f16_x2)(op1, op2);
+}
+
+// CHECK-LABEL: @test_svscale_single_f32_x2(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[OP1:%.*]], i64 0)
+// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[OP1]], i64 4)
+// CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4
x float> } @llvm.aarch64.sme.fp8.scale.single.x2.nxv4f32(<vscale x 4 x float>
[[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x i32> [[OP2:%.*]])
+// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>, <vscale
x 4 x float> } [[TMP2]], 0
+// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x float>
@llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x
float> [[TMP3]], i64 0)
+// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale
x 4 x float> } [[TMP2]], 1
+// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x float>
@llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP4]], <vscale x 4
x float> [[TMP5]], i64 4)
+// CHECK-NEXT: ret <vscale x 8 x float> [[TMP6]]
+//
+// CPP-CHECK-LABEL:
@_Z26test_svscale_single_f32_x213svfloat32x2_tu11__SVInt32_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[OP1:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[OP1]], i64 4)
+// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 4 x float>, <vscale
x 4 x float> } @llvm.aarch64.sme.fp8.scale.single.x2.nxv4f32(<vscale x 4 x
float> [[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x i32> [[OP2:%.*]])
+// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 4 x float>,
<vscale x 4 x float> } [[TMP2]], 0
+// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x float>
@llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x
float> [[TMP3]], i64 0)
+// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>,
<vscale x 4 x float> } [[TMP2]], 1
+// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x float>
@llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP4]], <vscale x 4
x float> [[TMP5]], i64 4)
+// CPP-CHECK-NEXT: ret <vscale x 8 x float> [[TMP6]]
+//
+svfloat32x2_t test_svscale_single_f32_x2(svfloat32x2_t op1, svint32_t op2)
__arm_streaming
+{
+ return SVE_ACLE_FUNC(svscale,_single_f32_x2)(op1, op2);
+}
+
+// CHECK-LABEL: @test_svscale_single_f64_x2(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[OP1:%.*]], i64 0)
+// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[OP1]], i64 2)
+// CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 2 x double>, <vscale x
2 x double> } @llvm.aarch64.sme.fp8.scale.single.x2.nxv2f64(<vscale x 2 x
double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x i64>
[[OP2:%.*]])
+// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 2 x double>, <vscale
x 2 x double> } [[TMP2]], 0
+// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x double>
@llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x
double> [[TMP3]], i64 0)
+// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 2 x double>, <vscale
x 2 x double> } [[TMP2]], 1
+// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 4 x double>
@llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP4]], <vscale x 2
x double> [[TMP5]], i64 2)
+// CHECK-NEXT: ret <vscale x 4 x double> [[TMP6]]
+//
+// CPP-CHECK-LABEL:
@_Z26test_svscale_single_f64_x213svfloat64x2_tu11__SVInt64_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[OP1:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[OP1]], i64 2)
+// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call { <vscale x 2 x double>,
<vscale x 2 x double> } @llvm.aarch64.sme.fp8.scale.single.x2.nxv2f64(<vscale x
2 x double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x i64>
[[OP2:%.*]])
+// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double> } [[TMP2]], 0
+// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x double>
@llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x
double> [[TMP3]], i64 0)
+// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double> } [[TMP2]], 1
+// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 4 x double>
@llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP4]], <vscale x 2
x double> [[TMP5]], i64 2)
+// CPP-CHECK-NEXT: ret <vscale x 4 x double> [[TMP6]]
+//
+svfloat64x2_t test_svscale_single_f64_x2(svfloat64x2_t op1, svint64_t op2)
__arm_streaming
+{
+ return SVE_ACLE_FUNC(svscale,_single_f64_x2)(op1, op2);
+}
+
+// Single x4
+// CHECK-LABEL: @test_svscale_single_f16_x4(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1:%.*]], i64 0)
+// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1]], i64 8)
+// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1]], i64 16)
+// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1]], i64 24)
+// CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8
x half>, <vscale x 8 x half>, <vscale x 8 x half> }
@llvm.aarch64.sme.fp8.scale.single.x4.nxv8f16(<vscale x 8 x half> [[TMP0]],
<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half>
[[TMP3]], <vscale x 8 x i16> [[OP2:%.*]])
+// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x
8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP4]], 0
+// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> poison, <vscale x 8 x
half> [[TMP5]], i64 0)
+// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x
8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP4]], 1
+// CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP6]], <vscale x 8
x half> [[TMP7]], i64 8)
+// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x
8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP4]], 2
+// CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP8]], <vscale x 8
x half> [[TMP9]], i64 16)
+// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 8 x half>, <vscale
x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP4]], 3
+// CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP10]], <vscale x
8 x half> [[TMP11]], i64 24)
+// CHECK-NEXT: ret <vscale x 32 x half> [[TMP12]]
+//
+// CPP-CHECK-LABEL:
@_Z26test_svscale_single_f16_x413svfloat16x4_tu11__SVInt16_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1]], i64 8)
+// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1]], i64 16)
+// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1]], i64 24)
+// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 8 x half>, <vscale
x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> }
@llvm.aarch64.sme.fp8.scale.single.x4.nxv8f16(<vscale x 8 x half> [[TMP0]],
<vscale x 8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half>
[[TMP3]], <vscale x 8 x i16> [[OP2:%.*]])
+// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 8 x half>,
<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP4]], 0
+// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> poison, <vscale x 8 x
half> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 8 x half>,
<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP4]], 1
+// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP6]], <vscale x 8
x half> [[TMP7]], i64 8)
+// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 8 x half>,
<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP4]], 2
+// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP8]], <vscale x 8
x half> [[TMP9]], i64 16)
+// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 8 x half>,
<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP4]], 3
+// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP10]], <vscale x
8 x half> [[TMP11]], i64 24)
+// CPP-CHECK-NEXT: ret <vscale x 32 x half> [[TMP12]]
+//
+svfloat16x4_t test_svscale_single_f16_x4(svfloat16x4_t op1, svint16_t op2)
__arm_streaming
+{
+ return SVE_ACLE_FUNC(svscale,_single_f16_x4)(op1, op2);
+}
+
+// CHECK-LABEL: @test_svscale_single_f32_x4(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1:%.*]], i64 0)
+// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1]], i64 4)
+// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1]], i64 8)
+// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1]], i64 12)
+// CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4
x float>, <vscale x 4 x float>, <vscale x 4 x float> }
@llvm.aarch64.sme.fp8.scale.single.x4.nxv4f32(<vscale x 4 x float> [[TMP0]],
<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x
float> [[TMP3]], <vscale x 4 x i32> [[OP2:%.*]])
+// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale
x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 0
+// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4
x float> [[TMP5]], i64 0)
+// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale
x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 1
+// CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x
4 x float> [[TMP7]], i64 4)
+// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>, <vscale
x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 2
+// CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP8]], <vscale x
4 x float> [[TMP9]], i64 8)
+// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale
x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 3
+// CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP10]], <vscale x
4 x float> [[TMP11]], i64 12)
+// CHECK-NEXT: ret <vscale x 16 x float> [[TMP12]]
+//
+// CPP-CHECK-LABEL:
@_Z26test_svscale_single_f32_x413svfloat32x4_tu11__SVInt32_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1]], i64 4)
+// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1]], i64 8)
+// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1]], i64 12)
+// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 4 x float>, <vscale
x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> }
@llvm.aarch64.sme.fp8.scale.single.x4.nxv4f32(<vscale x 4 x float> [[TMP0]],
<vscale x 4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x
float> [[TMP3]], <vscale x 4 x i32> [[OP2:%.*]])
+// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>,
<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 0
+// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4
x float> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>,
<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 1
+// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP6]], <vscale x
4 x float> [[TMP7]], i64 4)
+// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>,
<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 2
+// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP8]], <vscale x
4 x float> [[TMP9]], i64 8)
+// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>,
<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP4]], 3
+// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP10]], <vscale x
4 x float> [[TMP11]], i64 12)
+// CPP-CHECK-NEXT: ret <vscale x 16 x float> [[TMP12]]
+//
+svfloat32x4_t test_svscale_single_f32_x4(svfloat32x4_t op1, svint32_t op2)
__arm_streaming
+{
+ return SVE_ACLE_FUNC(svscale,_single_f32_x4)(op1, op2);
+}
+
+// CHECK-LABEL: @test_svscale_single_f64_x4(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1:%.*]], i64 0)
+// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1]], i64 2)
+// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1]], i64 4)
+// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1]], i64 6)
+// CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 2 x double>, <vscale x
2 x double>, <vscale x 2 x double>, <vscale x 2 x double> }
@llvm.aarch64.sme.fp8.scale.single.x4.nxv2f64(<vscale x 2 x double> [[TMP0]],
<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x
double> [[TMP3]], <vscale x 2 x i64> [[OP2:%.*]])
+// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 2 x double>, <vscale
x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP4]], 0
+// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> poison, <vscale x 2 x
double> [[TMP5]], i64 0)
+// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 2 x double>, <vscale
x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP4]], 1
+// CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP6]], <vscale x 2
x double> [[TMP7]], i64 2)
+// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 2 x double>, <vscale
x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP4]], 2
+// CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP8]], <vscale x 2
x double> [[TMP9]], i64 4)
+// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP4]],
3
+// CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP10]], <vscale x
2 x double> [[TMP11]], i64 6)
+// CHECK-NEXT: ret <vscale x 8 x double> [[TMP12]]
+//
+// CPP-CHECK-LABEL:
@_Z26test_svscale_single_f64_x413svfloat64x4_tu11__SVInt64_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1]], i64 2)
+// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1]], i64 4)
+// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1]], i64 6)
+// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> }
@llvm.aarch64.sme.fp8.scale.single.x4.nxv2f64(<vscale x 2 x double> [[TMP0]],
<vscale x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x
double> [[TMP3]], <vscale x 2 x i64> [[OP2:%.*]])
+// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP4]], 0
+// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> poison, <vscale x 2 x
double> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP4]],
1
+// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP6]], <vscale x 2
x double> [[TMP7]], i64 2)
+// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP4]],
2
+// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP8]], <vscale x 2
x double> [[TMP9]], i64 4)
+// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP4]],
3
+// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP10]], <vscale x
2 x double> [[TMP11]], i64 6)
+// CPP-CHECK-NEXT: ret <vscale x 8 x double> [[TMP12]]
+//
+svfloat64x4_t test_svscale_single_f64_x4(svfloat64x4_t op1, svint64_t op2)
__arm_streaming
+{
+ return SVE_ACLE_FUNC(svscale,_single_f64_x4)(op1, op2);
+}
+
+// Multi x2
+// CHECK-LABEL: @test_svscale_f16_x2(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[OP1:%.*]], i64 0)
+// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[OP1]], i64 8)
+// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16>
@llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[OP2:%.*]], i64 0)
+// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16>
@llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[OP2]], i64 8)
+// CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8
x half> } @llvm.aarch64.sme.fp8.scale.x2.nxv8f16(<vscale x 8 x half> [[TMP0]],
<vscale x 8 x half> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16>
[[TMP3]])
+// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x
8 x half> } [[TMP4]], 0
+// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x half>
@llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> poison, <vscale x 8 x
half> [[TMP5]], i64 0)
+// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x
8 x half> } [[TMP4]], 1
+// CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x half>
@llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP6]], <vscale x 8
x half> [[TMP7]], i64 8)
+// CHECK-NEXT: ret <vscale x 16 x half> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svscale_f16_x213svfloat16x2_t11svint16x2_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[OP1:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv16f16(<vscale x 16 x half> [[OP1]], i64 8)
+// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x i16>
@llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[OP2:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x i16>
@llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[OP2]], i64 8)
+// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 8 x half>, <vscale
x 8 x half> } @llvm.aarch64.sme.fp8.scale.x2.nxv8f16(<vscale x 8 x half>
[[TMP0]], <vscale x 8 x half> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x
8 x i16> [[TMP3]])
+// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 8 x half>,
<vscale x 8 x half> } [[TMP4]], 0
+// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 16 x half>
@llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> poison, <vscale x 8 x
half> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 8 x half>,
<vscale x 8 x half> } [[TMP4]], 1
+// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 16 x half>
@llvm.vector.insert.nxv16f16.nxv8f16(<vscale x 16 x half> [[TMP6]], <vscale x 8
x half> [[TMP7]], i64 8)
+// CPP-CHECK-NEXT: ret <vscale x 16 x half> [[TMP8]]
+//
+svfloat16x2_t test_svscale_f16_x2(svfloat16x2_t op1, svint16x2_t op2)
__arm_streaming
+{
+ return SVE_ACLE_FUNC(svscale,_f16_x2)(op1, op2);
+}
+
+// CHECK-LABEL: @test_svscale_f32_x2(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[OP1:%.*]], i64 0)
+// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[OP1]], i64 4)
+// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x i32>
@llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[OP2:%.*]], i64 0)
+// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x i32>
@llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[OP2]], i64 4)
+// CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4
x float> } @llvm.aarch64.sme.fp8.scale.x2.nxv4f32(<vscale x 4 x float>
[[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x
4 x i32> [[TMP3]])
+// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>, <vscale
x 4 x float> } [[TMP4]], 0
+// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x float>
@llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x
float> [[TMP5]], i64 0)
+// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>, <vscale
x 4 x float> } [[TMP4]], 1
+// CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 8 x float>
@llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP6]], <vscale x 4
x float> [[TMP7]], i64 4)
+// CHECK-NEXT: ret <vscale x 8 x float> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svscale_f32_x213svfloat32x2_t11svint32x2_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[OP1:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv8f32(<vscale x 8 x float> [[OP1]], i64 4)
+// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x i32>
@llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[OP2:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x i32>
@llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[OP2]], i64 4)
+// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 4 x float>, <vscale
x 4 x float> } @llvm.aarch64.sme.fp8.scale.x2.nxv4f32(<vscale x 4 x float>
[[TMP0]], <vscale x 4 x float> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x
4 x i32> [[TMP3]])
+// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 4 x float>,
<vscale x 4 x float> } [[TMP4]], 0
+// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x float>
@llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> poison, <vscale x 4 x
float> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x float>,
<vscale x 4 x float> } [[TMP4]], 1
+// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 8 x float>
@llvm.vector.insert.nxv8f32.nxv4f32(<vscale x 8 x float> [[TMP6]], <vscale x 4
x float> [[TMP7]], i64 4)
+// CPP-CHECK-NEXT: ret <vscale x 8 x float> [[TMP8]]
+//
+svfloat32x2_t test_svscale_f32_x2(svfloat32x2_t op1, svint32x2_t op2)
__arm_streaming
+{
+ return SVE_ACLE_FUNC(svscale,_f32_x2)(op1, op2);
+}
+
+// CHECK-LABEL: @test_svscale_f64_x2(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[OP1:%.*]], i64 0)
+// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[OP1]], i64 2)
+// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x i64>
@llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[OP2:%.*]], i64 0)
+// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x i64>
@llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[OP2]], i64 2)
+// CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 2 x double>, <vscale x
2 x double> } @llvm.aarch64.sme.fp8.scale.x2.nxv2f64(<vscale x 2 x double>
[[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale
x 2 x i64> [[TMP3]])
+// CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 2 x double>, <vscale
x 2 x double> } [[TMP4]], 0
+// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 4 x double>
@llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x
double> [[TMP5]], i64 0)
+// CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 2 x double>, <vscale
x 2 x double> } [[TMP4]], 1
+// CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 4 x double>
@llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP6]], <vscale x 2
x double> [[TMP7]], i64 2)
+// CHECK-NEXT: ret <vscale x 4 x double> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svscale_f64_x213svfloat64x2_t11svint64x2_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[OP1:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv4f64(<vscale x 4 x double> [[OP1]], i64 2)
+// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x i64>
@llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[OP2:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x i64>
@llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[OP2]], i64 2)
+// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call { <vscale x 2 x double>,
<vscale x 2 x double> } @llvm.aarch64.sme.fp8.scale.x2.nxv2f64(<vscale x 2 x
double> [[TMP0]], <vscale x 2 x double> [[TMP1]], <vscale x 2 x i64> [[TMP2]],
<vscale x 2 x i64> [[TMP3]])
+// CPP-CHECK-NEXT: [[TMP5:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double> } [[TMP4]], 0
+// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 4 x double>
@llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x
double> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double> } [[TMP4]], 1
+// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call <vscale x 4 x double>
@llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP6]], <vscale x 2
x double> [[TMP7]], i64 2)
+// CPP-CHECK-NEXT: ret <vscale x 4 x double> [[TMP8]]
+//
+svfloat64x2_t test_svscale_f64_x2(svfloat64x2_t op1, svint64x2_t op2)
__arm_streaming
+{
+ return SVE_ACLE_FUNC(svscale,_f64_x2)(op1, op2);
+}
+
+// Multi x4
+// CHECK-LABEL: @test_svscale_f16_x4(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1:%.*]], i64 0)
+// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1]], i64 8)
+// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1]], i64 16)
+// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1]], i64 24)
+// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16>
@llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[OP2:%.*]], i64 0)
+// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16>
@llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[OP2]], i64 8)
+// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16>
@llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[OP2]], i64 16)
+// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16>
@llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[OP2]], i64 24)
+// CHECK-NEXT: [[TMP8:%.*]] = tail call { <vscale x 8 x half>, <vscale x 8
x half>, <vscale x 8 x half>, <vscale x 8 x half> }
@llvm.aarch64.sme.fp8.scale.x4.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x
8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]],
<vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16>
[[TMP6]], <vscale x 8 x i16> [[TMP7]])
+// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 8 x half>, <vscale x
8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP8]], 0
+// CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> poison, <vscale x 8 x
half> [[TMP9]], i64 0)
+// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 8 x half>, <vscale
x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP8]], 1
+// CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP10]], <vscale x
8 x half> [[TMP11]], i64 8)
+// CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 8 x half>, <vscale
x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP8]], 2
+// CHECK-NEXT: [[TMP14:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP12]], <vscale x
8 x half> [[TMP13]], i64 16)
+// CHECK-NEXT: [[TMP15:%.*]] = extractvalue { <vscale x 8 x half>, <vscale
x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP8]], 3
+// CHECK-NEXT: [[TMP16:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP14]], <vscale x
8 x half> [[TMP15]], i64 24)
+// CHECK-NEXT: ret <vscale x 32 x half> [[TMP16]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svscale_f16_x413svfloat16x4_t11svint16x4_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1]], i64 8)
+// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1]], i64 16)
+// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 8 x half>
@llvm.vector.extract.nxv8f16.nxv32f16(<vscale x 32 x half> [[OP1]], i64 24)
+// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 8 x i16>
@llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[OP2:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 8 x i16>
@llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[OP2]], i64 8)
+// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 8 x i16>
@llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[OP2]], i64 16)
+// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 8 x i16>
@llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[OP2]], i64 24)
+// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call { <vscale x 8 x half>, <vscale
x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> }
@llvm.aarch64.sme.fp8.scale.x4.nxv8f16(<vscale x 8 x half> [[TMP0]], <vscale x
8 x half> [[TMP1]], <vscale x 8 x half> [[TMP2]], <vscale x 8 x half> [[TMP3]],
<vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16>
[[TMP6]], <vscale x 8 x i16> [[TMP7]])
+// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 8 x half>,
<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP8]], 0
+// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> poison, <vscale x 8 x
half> [[TMP9]], i64 0)
+// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 8 x half>,
<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP8]], 1
+// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP10]], <vscale x
8 x half> [[TMP11]], i64 8)
+// CPP-CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 8 x half>,
<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP8]], 2
+// CPP-CHECK-NEXT: [[TMP14:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP12]], <vscale x
8 x half> [[TMP13]], i64 16)
+// CPP-CHECK-NEXT: [[TMP15:%.*]] = extractvalue { <vscale x 8 x half>,
<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } [[TMP8]], 3
+// CPP-CHECK-NEXT: [[TMP16:%.*]] = tail call <vscale x 32 x half>
@llvm.vector.insert.nxv32f16.nxv8f16(<vscale x 32 x half> [[TMP14]], <vscale x
8 x half> [[TMP15]], i64 24)
+// CPP-CHECK-NEXT: ret <vscale x 32 x half> [[TMP16]]
+//
+svfloat16x4_t test_svscale_f16_x4(svfloat16x4_t op1, svint16x4_t op2)
__arm_streaming
+{
+ return SVE_ACLE_FUNC(svscale,_f16_x4)(op1, op2);
+}
+
+// CHECK-LABEL: @test_svscale_f32_x4(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1:%.*]], i64 0)
+// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1]], i64 4)
+// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1]], i64 8)
+// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1]], i64 12)
+// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x i32>
@llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[OP2:%.*]], i64 0)
+// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 4 x i32>
@llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[OP2]], i64 4)
+// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 4 x i32>
@llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[OP2]], i64 8)
+// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 4 x i32>
@llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[OP2]], i64 12)
+// CHECK-NEXT: [[TMP8:%.*]] = tail call { <vscale x 4 x float>, <vscale x 4
x float>, <vscale x 4 x float>, <vscale x 4 x float> }
@llvm.aarch64.sme.fp8.scale.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x
4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float>
[[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4
x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]])
+// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>, <vscale
x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP8]], 0
+// CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4
x float> [[TMP9]], i64 0)
+// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale
x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP8]], 1
+// CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP10]], <vscale x
4 x float> [[TMP11]], i64 4)
+// CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale
x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP8]], 2
+// CHECK-NEXT: [[TMP14:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP12]], <vscale x
4 x float> [[TMP13]], i64 8)
+// CHECK-NEXT: [[TMP15:%.*]] = extractvalue { <vscale x 4 x float>, <vscale
x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP8]], 3
+// CHECK-NEXT: [[TMP16:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP14]], <vscale x
4 x float> [[TMP15]], i64 12)
+// CHECK-NEXT: ret <vscale x 16 x float> [[TMP16]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svscale_f32_x413svfloat32x4_t11svint32x4_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1]], i64 4)
+// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1]], i64 8)
+// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float>
@llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[OP1]], i64 12)
+// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x i32>
@llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[OP2:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 4 x i32>
@llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[OP2]], i64 4)
+// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 4 x i32>
@llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[OP2]], i64 8)
+// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 4 x i32>
@llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[OP2]], i64 12)
+// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call { <vscale x 4 x float>, <vscale
x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> }
@llvm.aarch64.sme.fp8.scale.x4.nxv4f32(<vscale x 4 x float> [[TMP0]], <vscale x
4 x float> [[TMP1]], <vscale x 4 x float> [[TMP2]], <vscale x 4 x float>
[[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4
x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]])
+// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>,
<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP8]], 0
+// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> poison, <vscale x 4
x float> [[TMP9]], i64 0)
+// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>,
<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP8]], 1
+// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP10]], <vscale x
4 x float> [[TMP11]], i64 4)
+// CPP-CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>,
<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP8]], 2
+// CPP-CHECK-NEXT: [[TMP14:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP12]], <vscale x
4 x float> [[TMP13]], i64 8)
+// CPP-CHECK-NEXT: [[TMP15:%.*]] = extractvalue { <vscale x 4 x float>,
<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } [[TMP8]], 3
+// CPP-CHECK-NEXT: [[TMP16:%.*]] = tail call <vscale x 16 x float>
@llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> [[TMP14]], <vscale x
4 x float> [[TMP15]], i64 12)
+// CPP-CHECK-NEXT: ret <vscale x 16 x float> [[TMP16]]
+//
+svfloat32x4_t test_svscale_f32_x4(svfloat32x4_t op1, svint32x4_t op2)
__arm_streaming
+{
+ return SVE_ACLE_FUNC(svscale,_f32_x4)(op1, op2);
+}
+
+// CHECK-LABEL: @test_svscale_f64_x4(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1:%.*]], i64 0)
+// CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1]], i64 2)
+// CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1]], i64 4)
+// CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1]], i64 6)
+// CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 2 x i64>
@llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[OP2:%.*]], i64 0)
+// CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 2 x i64>
@llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[OP2]], i64 2)
+// CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 2 x i64>
@llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[OP2]], i64 4)
+// CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 2 x i64>
@llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[OP2]], i64 6)
+// CHECK-NEXT: [[TMP8:%.*]] = tail call { <vscale x 2 x double>, <vscale x
2 x double>, <vscale x 2 x double>, <vscale x 2 x double> }
@llvm.aarch64.sme.fp8.scale.x4.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale
x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double>
[[TMP3]], <vscale x 2 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], <vscale x 2
x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]])
+// CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 2 x double>, <vscale
x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP8]], 0
+// CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> poison, <vscale x 2 x
double> [[TMP9]], i64 0)
+// CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP8]],
1
+// CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP10]], <vscale x
2 x double> [[TMP11]], i64 2)
+// CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP8]],
2
+// CHECK-NEXT: [[TMP14:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP12]], <vscale x
2 x double> [[TMP13]], i64 4)
+// CHECK-NEXT: [[TMP15:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP8]],
3
+// CHECK-NEXT: [[TMP16:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP14]], <vscale x
2 x double> [[TMP15]], i64 6)
+// CHECK-NEXT: ret <vscale x 8 x double> [[TMP16]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svscale_f64_x413svfloat64x4_t11svint64x4_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1]], i64 2)
+// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1]], i64 4)
+// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 2 x double>
@llvm.vector.extract.nxv2f64.nxv8f64(<vscale x 8 x double> [[OP1]], i64 6)
+// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 2 x i64>
@llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[OP2:%.*]], i64 0)
+// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call <vscale x 2 x i64>
@llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[OP2]], i64 2)
+// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call <vscale x 2 x i64>
@llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[OP2]], i64 4)
+// CPP-CHECK-NEXT: [[TMP7:%.*]] = tail call <vscale x 2 x i64>
@llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[OP2]], i64 6)
+// CPP-CHECK-NEXT: [[TMP8:%.*]] = tail call { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> }
@llvm.aarch64.sme.fp8.scale.x4.nxv2f64(<vscale x 2 x double> [[TMP0]], <vscale
x 2 x double> [[TMP1]], <vscale x 2 x double> [[TMP2]], <vscale x 2 x double>
[[TMP3]], <vscale x 2 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], <vscale x 2
x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]])
+// CPP-CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP8]], 0
+// CPP-CHECK-NEXT: [[TMP10:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> poison, <vscale x 2 x
double> [[TMP9]], i64 0)
+// CPP-CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP8]],
1
+// CPP-CHECK-NEXT: [[TMP12:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP10]], <vscale x
2 x double> [[TMP11]], i64 2)
+// CPP-CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP8]],
2
+// CPP-CHECK-NEXT: [[TMP14:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP12]], <vscale x
2 x double> [[TMP13]], i64 4)
+// CPP-CHECK-NEXT: [[TMP15:%.*]] = extractvalue { <vscale x 2 x double>,
<vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } [[TMP8]],
3
+// CPP-CHECK-NEXT: [[TMP16:%.*]] = tail call <vscale x 8 x double>
@llvm.vector.insert.nxv8f64.nxv2f64(<vscale x 8 x double> [[TMP14]], <vscale x
2 x double> [[TMP15]], i64 6)
+// CPP-CHECK-NEXT: ret <vscale x 8 x double> [[TMP16]]
+//
+svfloat64x4_t test_svscale_f64_x4(svfloat64x4_t op1, svint64x4_t op2)
__arm_streaming
+{
+ return SVE_ACLE_FUNC(svscale,_f64_x4)(op1, op2);
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td
b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 3735bf5222fce3..0ac3166aacd668 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -3698,6 +3698,31 @@ let TargetPrefix = "aarch64" in {
: DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
LLVMMatchType<0>, LLVMMatchType<0>],
[llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty],
[ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>,
IntrReadMem]>;
+
+ //
+ // Register scaling
+ //
+ def int_aarch64_sme_fp8_scale_single_x2
+ : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
+ [LLVMMatchType<0>, LLVMMatchType<0>,
LLVMVectorOfBitcastsToInt<0>],
+ [IntrNoMem]>;
+
+ def int_aarch64_sme_fp8_scale_single_x4
+ : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
LLVMMatchType<0>, LLVMMatchType<0>],
+ [LLVMMatchType<0>, LLVMMatchType<0>,
LLVMMatchType<0>, LLVMMatchType<0>, LLVMVectorOfBitcastsToInt<0>],
+ [IntrNoMem]>;
+
+ def int_aarch64_sme_fp8_scale_x2
+ : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
+ [LLVMMatchType<0>, LLVMMatchType<0>,
+ LLVMVectorOfBitcastsToInt<0>,
LLVMVectorOfBitcastsToInt<0>],
+ [IntrNoMem]>;
+
+ def int_aarch64_sme_fp8_scale_x4
+ : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
LLVMMatchType<0>, LLVMMatchType<0>],
+ [LLVMMatchType<0>, LLVMMatchType<0>,
LLVMMatchType<0>, LLVMMatchType<0>,
+ LLVMVectorOfBitcastsToInt<0>,
LLVMVectorOfBitcastsToInt<0>, LLVMVectorOfBitcastsToInt<0>,
LLVMVectorOfBitcastsToInt<0>],
+ [IntrNoMem]>;
}
// SVE2.1 - ZIPQ1, ZIPQ2, UZPQ1, UZPQ2
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 55cc106c08b956..3db1f1dbef07cc 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -5653,6 +5653,34 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
return;
+ case Intrinsic::aarch64_sme_fp8_scale_single_x2:
+ if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
+ Node->getValueType(0),
+ {0, AArch64::FSCALE_2ZZ_H, AArch64::FSCALE_2ZZ_S,
+ AArch64::FSCALE_2ZZ_D}))
+ SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
+ return;
+ case Intrinsic::aarch64_sme_fp8_scale_single_x4:
+ if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
+ Node->getValueType(0),
+ {0, AArch64::FSCALE_4ZZ_H, AArch64::FSCALE_4ZZ_S,
+ AArch64::FSCALE_4ZZ_D}))
+ SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
+ return;
+ case Intrinsic::aarch64_sme_fp8_scale_x2:
+ if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
+ Node->getValueType(0),
+ {0, AArch64::FSCALE_2Z2Z_H, AArch64::FSCALE_2Z2Z_S,
+ AArch64::FSCALE_2Z2Z_D}))
+ SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
+ return;
+ case Intrinsic::aarch64_sme_fp8_scale_x4:
+ if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
+ Node->getValueType(0),
+ {0, AArch64::FSCALE_4Z4Z_H, AArch64::FSCALE_4Z4Z_S,
+ AArch64::FSCALE_4Z4Z_D}))
+ SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
+ return;
case Intrinsic::aarch64_sve_whilege_x2:
if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
Node->getValueType(0),
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-fscale.ll
b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fscale.ll
new file mode 100644
index 00000000000000..2ac31de335309b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fscale.ll
@@ -0,0 +1,186 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
UTC_ARGS: --version 2
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+fp8
-force-streaming -verify-machineinstrs < %s | FileCheck %s
+
+; FSCALE (Single, x2)
+
+define { <vscale x 8 x half>, <vscale x 8 x half> }
@multi_vec_scale_single_x2_half( <vscale x 8 x half> %zdn1, <vscale x 8 x half>
%zdn2, <vscale x 8 x i16> %zm) {
+; CHECK-LABEL: multi_vec_scale_single_x2_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: fscale { z0.h, z1.h }, { z0.h, z1.h }, z2.h
+; CHECK-NEXT: ret
+ %res = call { <vscale x 8 x half>, <vscale x 8 x half> }
@llvm.aarch64.sme.fp8.scale.single.x2.nxv8f16(<vscale x 8 x half> %zdn1,
<vscale x 8 x half> %zdn2, <vscale x 8 x i16> %zm)
+ ret { <vscale x 8 x half>, <vscale x 8 x half> } %res
+}
+
+define { <vscale x 4 x float>, <vscale x 4 x float> }
@multi_vec_scale_single_x2_float( <vscale x 4 x float> %zdn1, <vscale x 4 x
float> %zdn2, <vscale x 4 x i32> %zm) {
+; CHECK-LABEL: multi_vec_scale_single_x2_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: fscale { z0.s, z1.s }, { z0.s, z1.s }, z2.s
+; CHECK-NEXT: ret
+ %res = call { <vscale x 4 x float>, <vscale x 4 x float> }
@llvm.aarch64.sme.fp8.scale.single.x2.nxv4f32(<vscale x 4 x float> %zdn1,
<vscale x 4 x float> %zdn2, <vscale x 4 x i32> %zm)
+ ret { <vscale x 4 x float>, <vscale x 4 x float> } %res
+}
+
+define { <vscale x 2 x double>, <vscale x 2 x double> }
@multi_vec_scale_single_x2_double( <vscale x 2 x double> %zdn1, <vscale x 2 x
double> %zdn2, <vscale x 2 x i64> %zm) {
+; CHECK-LABEL: multi_vec_scale_single_x2_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: fscale { z0.d, z1.d }, { z0.d, z1.d }, z2.d
+; CHECK-NEXT: ret
+ %res = call { <vscale x 2 x double>, <vscale x 2 x double> }
@llvm.aarch64.sme.fp8.scale.single.x2.nxv2f64(<vscale x 2 x double> %zdn1,
<vscale x 2 x double> %zdn2, <vscale x 2 x i64> %zm)
+ ret { <vscale x 2 x double>, <vscale x 2 x double> } %res
+}
+
+; FSCALE (Single, x4)
+
+define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>,
<vscale x 8 x half> } @multi_vec_scale_single_x4_half( <vscale x 8 x half>
%zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zdn3, <vscale x 8 x
half> %zdn4, <vscale x 8 x i16> %zm) {
+; CHECK-LABEL: multi_vec_scale_single_x4_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: fscale { z0.h - z3.h }, { z0.h - z3.h }, z4.h
+; CHECK-NEXT: ret
+ %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>,
<vscale x 8 x half> } @llvm.aarch64.sme.fp8.scale.single.x4.nxv8f16(<vscale x 8
x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zdn3, <vscale x
8 x half> %zdn4, <vscale x 8 x i16> %zm)
+ ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale
x 8 x half> } %res
+}
+
+define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>,
<vscale x 4 x float> } @multi_vec_scale_single_x4_float( <vscale x 4 x float>
%zdn1, <vscale x 4 x float> %zdn2, <vscale x 4 x float> %zdn3, <vscale x 4 x
float> %zdn4, <vscale x 4 x i32> %zm) {
+; CHECK-LABEL: multi_vec_scale_single_x4_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: fscale { z0.s - z3.s }, { z0.s - z3.s }, z4.s
+; CHECK-NEXT: ret
+ %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x
float>, <vscale x 4 x float> }
@llvm.aarch64.sme.fp8.scale.single.x4.nxv4f32(<vscale x 4 x float> %zdn1,
<vscale x 4 x float> %zdn2, <vscale x 4 x float> %zdn3, <vscale x 4 x float>
%zdn4, <vscale x 4 x i32> %zm)
+ ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>,
<vscale x 4 x float> } %res
+}
+
+define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>,
<vscale x 2 x double> } @multi_vec_scale_single_x4_double( <vscale x 2 x
double> %zdn1, <vscale x 2 x double> %zdn2, <vscale x 2 x double> %zdn3,
<vscale x 2 x double> %zdn4, <vscale x 2 x i64> %zm) {
+; CHECK-LABEL: multi_vec_scale_single_x4_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: fscale { z0.d - z3.d }, { z0.d - z3.d }, z4.d
+; CHECK-NEXT: ret
+ %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x
double>, <vscale x 2 x double> }
@llvm.aarch64.sme.fp8.scale.single.x4.nxv2f64(<vscale x 2 x double> %zdn1,
<vscale x 2 x double> %zdn2, <vscale x 2 x double> %zdn3, <vscale x 2 x double>
%zdn4, <vscale x 2 x i64> %zm)
+ ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>,
<vscale x 2 x double> } %res
+}
+
+; FSCALE (Multi, x2)
+define { <vscale x 8 x half>, <vscale x 8 x half> } @multi_vec_scale_x2_half(
<vscale x 8 x half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x i16> %zm1,
<vscale x 8 x i16> %zm2) {
+; CHECK-LABEL: multi_vec_scale_x2_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: fscale { z0.h, z1.h }, { z0.h, z1.h }, { z2.h, z3.h }
+; CHECK-NEXT: ret
+ %res = call { <vscale x 8 x half>, <vscale x 8 x half> }
@llvm.aarch64.sme.fp8.scale.x2.nxv8f16(<vscale x 8 x half> %zdn1, <vscale x 8 x
half> %zdn2, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2)
+ ret { <vscale x 8 x half>, <vscale x 8 x half> } %res
+}
+
+define { <vscale x 4 x float>, <vscale x 4 x float> }
@multi_vec_scale_x2_float( <vscale x 4 x float> %zdn1, <vscale x 4 x float>
%zdn2, <vscale x 4 x i32> %zm1, <vscale x 4 x i32> %zm2 ) {
+; CHECK-LABEL: multi_vec_scale_x2_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: fscale { z0.s, z1.s }, { z0.s, z1.s }, { z2.s, z3.s }
+; CHECK-NEXT: ret
+ %res = call { <vscale x 4 x float>, <vscale x 4 x float> }
@llvm.aarch64.sme.fp8.scale.x2.nxv4f32(<vscale x 4 x float> %zdn1, <vscale x 4
x float> %zdn2, <vscale x 4 x i32> %zm1, <vscale x 4 x i32> %zm2)
+ ret { <vscale x 4 x float>, <vscale x 4 x float> } %res
+}
+
+define { <vscale x 2 x double>, <vscale x 2 x double> }
@multi_vec_scale_x2_double( <vscale x 2 x double> %zdn1, <vscale x 2 x double>
%zdn2, <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm2) {
+; CHECK-LABEL: multi_vec_scale_x2_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: fscale { z0.d, z1.d }, { z0.d, z1.d }, { z2.d, z3.d }
+; CHECK-NEXT: ret
+ %res = call { <vscale x 2 x double>, <vscale x 2 x double> }
@llvm.aarch64.sme.fp8.scale.x2.nxv2f64(<vscale x 2 x double> %zdn1, <vscale x 2
x double> %zdn2, <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm2)
+ ret { <vscale x 2 x double>, <vscale x 2 x double> } %res
+}
+
+; FSCALE (Multi, x4)
+define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>,
<vscale x 8 x half> } @multi_vec_scale_x4_half( <vscale x 8 x half> %zdn1,
<vscale x 8 x half> %zdn2, <vscale x 8 x half> %zdn3, <vscale x 8 x half>
%zdn4, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x i16>
%zm3, <vscale x 8 x i16> %zm4) {
+; CHECK-LABEL: multi_vec_scale_x4_half:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def
$z4_z5_z6_z7
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def
$z4_z5_z6_z7
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def
$z4_z5_z6_z7
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def
$z4_z5_z6_z7
+; CHECK-NEXT: fscale { z0.h - z3.h }, { z0.h - z3.h }, { z4.h - z7.h }
+; CHECK-NEXT: ret
+ %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>,
<vscale x 8 x half> } @llvm.aarch64.sme.fp8.scale.x4.nxv8f16(<vscale x 8 x
half> %zdn1, <vscale x 8 x half> %zdn2, <vscale x 8 x half> %zdn3, <vscale x 8
x half> %zdn4, <vscale x 8 x i16> %zm1, <vscale x 8 x i16> %zm2, <vscale x 8 x
i16> %zm3, <vscale x 8 x i16> %zm4)
+ ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale
x 8 x half> } %res
+}
+
+define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>,
<vscale x 4 x float> } @multi_vec_scale_x4_float( <vscale x 4 x float> %zdn1,
<vscale x 4 x float> %zdn2, <vscale x 4 x float> %zdn3, <vscale x 4 x float>
%zdn4, <vscale x 4 x i32> %zm1, <vscale x 4 x i32> %zm2, <vscale x 4 x i32>
%zm3, <vscale x 4 x i32> %zm4) {
+; CHECK-LABEL: multi_vec_scale_x4_float:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def
$z4_z5_z6_z7
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def
$z4_z5_z6_z7
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def
$z4_z5_z6_z7
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def
$z4_z5_z6_z7
+; CHECK-NEXT: fscale { z0.s - z3.s }, { z0.s - z3.s }, { z4.s - z7.s }
+; CHECK-NEXT: ret
+ %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x
float>, <vscale x 4 x float> } @llvm.aarch64.sme.fp8.scale.x4.nxv4f32(<vscale x
4 x float> %zdn1, <vscale x 4 x float> %zdn2, <vscale x 4 x float> %zdn3,
<vscale x 4 x float> %zdn4, <vscale x 4 x i32> %zm1, <vscale x 4 x i32> %zm2,
<vscale x 4 x i32> %zm3, <vscale x 4 x i32> %zm4)
+ ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>,
<vscale x 4 x float> } %res
+}
+
+define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>,
<vscale x 2 x double> } @multi_vec_scale_x4_double( <vscale x 2 x double>
%zdn1, <vscale x 2 x double> %zdn2, <vscale x 2 x double> %zdn3, <vscale x 2 x
double> %zdn4, <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm2, <vscale x 2 x
i64> %zm3, <vscale x 2 x i64> %zm4) {
+; CHECK-LABEL: multi_vec_scale_x4_double:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z27.d, z7.d
+; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: mov z25.d, z6.d
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: mov z24.d, z4.d
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: mov z26.d, z25.d
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
+; CHECK-NEXT: fscale { z0.d - z3.d }, { z0.d - z3.d }, { z24.d - z27.d }
+; CHECK-NEXT: ret
+ %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x
double>, <vscale x 2 x double> } @llvm.aarch64.sme.fp8.scale.x4.nxv2f64(<vscale
x 2 x double> %zdn1, <vscale x 2 x double> %zdn2, <vscale x 2 x double> %zdn3,
<vscale x 2 x double> %zdn4, <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm3,
<vscale x 2 x i64> %zm3, <vscale x 2 x i64> %zm4)
+ ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>,
<vscale x 2 x double> } %res
+}
+
+declare { <vscale x 8 x half>, <vscale x 8 x half> }
@llvm.aarch64.sme.fp8.scale.single.x2.nxv8f16(<vscale x 8 x half>, <vscale x 8
x half>, <vscale x 8 x i16>)
+declare { <vscale x 4 x float>, <vscale x 4 x float> }
@llvm.aarch64.sme.fp8.scale.single.x2.nxv4f32(<vscale x 4 x float>, <vscale x 4
x float>, <vscale x 4 x i32>)
+declare { <vscale x 2 x double>, <vscale x 2 x double> }
@llvm.aarch64.sme.fp8.scale.single.x2.nxv2f64(<vscale x 2 x double>, <vscale x
2 x double>, <vscale x 2 x i64>)
+
+declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>,
<vscale x 8 x half> } @llvm.aarch64.sme.fp8.scale.single.x4.nxv8f16(<vscale x 8
x half>, <vscale x 8 x half>,<vscale x 8 x half>, <vscale x 8 x half>, <vscale
x 8 x i16>)
+declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>,
<vscale x 4 x float> } @llvm.aarch64.sme.fp8.scale.single.x4.nxv4f32(<vscale x
4 x float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>,
<vscale x 4 x i32>)
+declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>,
<vscale x 2 x double> } @llvm.aarch64.sme.fp8.scale.single.x4.nxv2f64(<vscale x
2 x double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x
double>, <vscale x 2 x i64>)
+
+declare { <vscale x 8 x half>, <vscale x 8 x half> }
@llvm.aarch64.sme.fp8.scale.x2.nxv8f16(<vscale x 8 x half>, <vscale x 8 x
half>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare { <vscale x 4 x float>, <vscale x 4 x float> }
@llvm.aarch64.sme.fp8.scale.x2.nxv4f32(<vscale x 4 x float>, <vscale x 4 x
float>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare { <vscale x 2 x double>, <vscale x 2 x double> }
@llvm.aarch64.sme.fp8.scale.x2.nxv2f64(<vscale x 2 x double>, <vscale x 2 x
double>, <vscale x 2 x i64>, <vscale x 2 x i64>)
+
+declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>,
<vscale x 8 x half> } @llvm.aarch64.sme.fp8.scale.x4.nxv8f16(<vscale x 8 x
half>, <vscale x 8 x half>,<vscale x 8 x half>, <vscale x 8 x half>, <vscale x
8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>)
+declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>,
<vscale x 4 x float> } @llvm.aarch64.sme.fp8.scale.x4.nxv4f32(<vscale x 4 x
float>, <vscale x 4 x float>,<vscale x 4 x float>, <vscale x 4 x float>,
<vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>,
<vscale x 2 x double> } @llvm.aarch64.sme.fp8.scale.x4.nxv2f64(<vscale x 2 x
double>, <vscale x 2 x double>,<vscale x 2 x double>, <vscale x 2 x double>,
<vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>)
>From 961a45e8bbd40dd99531bd105799b1f161d4bfc7 Mon Sep 17 00:00:00 2001
From: Marian Lukac <marian.lu...@arm.com>
Date: Mon, 2 Sep 2024 09:57:06 +0000
Subject: [PATCH 2/2] Fix test file
---
llvm/test/CodeGen/AArch64/sme2-intrinsics-fscale.ll | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-fscale.ll
b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fscale.ll
index 2ac31de335309b..591fe8da6b79cd 100644
--- a/llvm/test/CodeGen/AArch64/sme2-intrinsics-fscale.ll
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-fscale.ll
@@ -155,17 +155,17 @@ define { <vscale x 4 x float>, <vscale x 4 x float>,
<vscale x 4 x float>, <vsca
define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>,
<vscale x 2 x double> } @multi_vec_scale_x4_double( <vscale x 2 x double>
%zdn1, <vscale x 2 x double> %zdn2, <vscale x 2 x double> %zdn3, <vscale x 2 x
double> %zdn4, <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm2, <vscale x 2 x
i64> %zm3, <vscale x 2 x i64> %zm4) {
; CHECK-LABEL: multi_vec_scale_x4_double:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z27.d, z7.d
; CHECK-NEXT: // kill: def $z3 killed $z3 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
-; CHECK-NEXT: mov z25.d, z6.d
+; CHECK-NEXT: // kill: def $z7 killed $z7 killed $z4_z5_z6_z7 def
$z4_z5_z6_z7
; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
-; CHECK-NEXT: mov z24.d, z4.d
+; CHECK-NEXT: // kill: def $z6 killed $z6 killed $z4_z5_z6_z7 def
$z4_z5_z6_z7
; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
-; CHECK-NEXT: mov z26.d, z25.d
+; CHECK-NEXT: // kill: def $z5 killed $z5 killed $z4_z5_z6_z7 def
$z4_z5_z6_z7
; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2_z3 def
$z0_z1_z2_z3
-; CHECK-NEXT: fscale { z0.d - z3.d }, { z0.d - z3.d }, { z24.d - z27.d }
+; CHECK-NEXT: // kill: def $z4 killed $z4 killed $z4_z5_z6_z7 def
$z4_z5_z6_z7
+; CHECK-NEXT: fscale { z0.d - z3.d }, { z0.d - z3.d }, { z4.d - z7.d }
; CHECK-NEXT: ret
- %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x
double>, <vscale x 2 x double> } @llvm.aarch64.sme.fp8.scale.x4.nxv2f64(<vscale
x 2 x double> %zdn1, <vscale x 2 x double> %zdn2, <vscale x 2 x double> %zdn3,
<vscale x 2 x double> %zdn4, <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm3,
<vscale x 2 x i64> %zm3, <vscale x 2 x i64> %zm4)
+ %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x
double>, <vscale x 2 x double> } @llvm.aarch64.sme.fp8.scale.x4.nxv2f64(<vscale
x 2 x double> %zdn1, <vscale x 2 x double> %zdn2, <vscale x 2 x double> %zdn3,
<vscale x 2 x double> %zdn4, <vscale x 2 x i64> %zm1, <vscale x 2 x i64> %zm2,
<vscale x 2 x i64> %zm3, <vscale x 2 x i64> %zm4)
ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>,
<vscale x 2 x double> } %res
}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
