=?utf-8?q?Mikołaj_Piróg?= <mikolajpi...@gmail.com>,
=?utf-8?q?Mikołaj_Piróg?= <mikolajpi...@gmail.com>,
=?utf-8?q?Mikołaj_Piróg?= <mikolajpi...@gmail.com>,
=?utf-8?q?Mikołaj_Piróg?= <mikolajpi...@gmail.com>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/120...@github.com>
================
@@ -24,573 +24,3369 @@
__attribute__((__always_inline__, __nodebug__, __target__("avx10.2-256"),
\
__min_vector_width__(256)))
+//clang-format off
+
+/// Convert two 128-bit vectors, \a __A and \a __B, containing packed
+/// single-precision (32-bit) floating-point elements to a 128-bit vector
+/// containing FP16 elements.
+///
+/// \code{.operation}
+/// FOR i := 0 to 7
+/// IF i < 4
+/// dst.fp16[i] := convert_fp32_to_fp16(__B.fp32[i])
+/// ELSE
+/// dst.fp16[i] := convert_fp32_to_fp16(__A.fp32[i - 4])
+/// FI
+///
+/// dst[MAX:127] := 0
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VCVT2PS2PHX instruction.
+///
+/// \param __A
+/// A 128-bit vector of [4 x float].
+/// \param __B
+/// A 128-bit vector of [4 x float].
+/// \returns
+/// A 128-bit vector of [8 x fp16]. Lower elements correspond to the
+/// (converted) elements from \a __B; higher order elements correspond to
the
+/// (converted) elements from \a __A.
static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_cvtx2ps_ph(__m128 __A,
__m128 __B) {
return (__m128h)__builtin_ia32_vcvt2ps2phx128_mask(
(__v4sf)__A, (__v4sf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)(-1));
}
+/// Convert two 128-bit vectors, \a __A and \a __B, containing packed
+/// single-precision (32-bit) floating-point elements to a 128-bit vector
+/// containing FP16 elements. Merging mask \a __U is used to determine if
given
+/// element should be taken from \a __W instead.
+///
+/// \code{.operation}
+/// FOR i := 0 to 7
+/// IF __U[i]
+/// IF i < 4
+/// dst.fp16[i] := convert_fp32_to_fp16(__B.fp32[i])
+/// ELSE
+/// dst.fp16[i] := convert_fp32_to_fp16(__A.fp32[i - 4])
+/// FI
+/// ELSE
+/// dst.fp16[i] := __W.fp16[i]
+/// FI
+///
+/// dst[MAX:127] := 0
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VCVT2PS2PHX instruction.
+///
+/// \param __W
+/// A 128-bit vector of [8 x fp16].
+/// \param __U
+/// A 8-bit merging mask.
+/// \param __A
+/// A 128-bit vector of [4 x float].
+/// \param __B
+/// A 128-bit vector of [4 x float].
+/// \returns
+/// A 128-bit vector of [8 x fp16]. Lower elements correspond to the
+/// (converted) elements from \a __B; higher order elements correspond to
the
+/// (converted) elements from \a __A. If corresponding mask bit is not set,
then
+/// element from \a __W is taken instead.
static __inline__ __m128h __DEFAULT_FN_ATTRS128
_mm_mask_cvtx2ps_ph(__m128h __W, __mmask8 __U, __m128 __A, __m128 __B) {
return (__m128h)__builtin_ia32_vcvt2ps2phx128_mask(
(__v4sf)__A, (__v4sf)__B, (__v8hf)__W, (__mmask8)__U);
}
+/// Convert two 128-bit vectors, \a __A and \a __B, containing packed
+/// single-precision (32-bit) floating-point elements to a 128-bit vector
+/// containing FP16 elements. Zeroing mask \a __U is used to determine if
given
+/// element should be zeroed instead.
+///
+/// \code{.operation}
+/// FOR i := 0 to 7
+/// IF __U[i]
+/// IF i < 4
+/// dst.fp16[i] := convert_fp32_to_fp16(__B.fp32[i])
+/// ELSE
+/// dst.fp16[i] := convert_fp32_to_fp16(__A.fp32[i - 4])
+/// FI
+/// ELSE
+/// dst.fp16[i] := 0
+/// FI
+/// ENDFOR
+///
+/// dst[MAX:127] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VCVT2PS2PHX instruction.
+///
+/// \param __U
+/// A 8-bit zeroing mask.
+/// \param __A
+/// A 128-bit vector of [4 x float].
+/// \param __B
+/// A 128-bit vector of [4 x float].
+/// \returns
+/// A 128-bit vector of [8 x fp16]. Lower elements correspond to the
+/// (converted) elements from \a __B; higher order elements correspond to
the
+/// (converted) elements from \a __A. If corresponding mask bit is not set,
+/// then zero is taken instead.
static __inline__ __m128h __DEFAULT_FN_ATTRS128
_mm_maskz_cvtx2ps_ph(__mmask8 __U, __m128 __A, __m128 __B) {
return (__m128h)__builtin_ia32_vcvt2ps2phx128_mask(
(__v4sf)__A, (__v4sf)__B, (__v8hf)_mm_setzero_ph(), (__mmask8)__U);
}
+/// Convert two 256-bit vectors, \a __A and \a __B, containing packed
+/// single-precision (32-bit) floating-point elements to a 256-bit vector
+/// containing FP16 elements.
+///
+/// \code{.operation}
+/// FOR i := 0 to 15
+/// IF i < 8
+/// dst.fp16[i] := convert_fp32_to_fp16(__B.fp32[i])
+/// ELSE
+/// dst.fp16[i] := convert_fp32_to_fp16(__A.fp32[i - 8])
+/// FI
+/// ENDFOR
+///
+/// dst[MAX:256] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VCVT2PS2PHX instruction.
+///
+/// \param __A
+/// A 256-bit vector of [8 x float].
+/// \param __B
+/// A 256-bit vector of [8 x float].
+/// \returns
+/// A 256-bit vector of [16 x fp16]. Lower elements correspond to the
+/// (converted) elements from \a __B; higher order elements correspond to
the
+/// (converted) elements from \a __A.
static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_cvtx2ps_ph(__m256 __A,
__m256 __B) {
return (__m256h)__builtin_ia32_vcvt2ps2phx256_mask(
(__v8sf)__A, (__v8sf)__B, (__v16hf)_mm256_setzero_ph(), (__mmask16)(-1),
_MM_FROUND_CUR_DIRECTION);
}
+/// Convert two 256-bit vectors, \a __A and \a __B, containing packed
+/// single-precision (32-bit) floating-point elements to a 256-bit vector
+/// containing FP16 elements. Merging mask \a __U is used to determine if
given
+/// element should be taken from \a __W instead.
+///
+/// \code{.operation}
+/// FOR i := 0 to 15
+/// IF __U[i]
+/// IF i < 8
+/// dst.fp16[i] := convert_fp32_to_fp16(__B.fp32[i])
+/// ELSE
+/// dst.fp16[i] := convert_fp32_to_fp16(__A.fp32[i - 8])
+/// FI
+/// ELSE
+/// dst.fp16[i] := __W.fp16[i]
+/// FI
+/// ENDFOR
+///
+/// dst[MAX:256] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VCVT2PS2PHX instruction.
+///
+/// \param __W
+/// A 256-bit vector of [16 x fp16].
+/// \param __U
+/// A 16-bit merging mask.
+/// \param __A
+/// A 256-bit vector of [8 x float].
+/// \param __B
+/// A 256-bit vector of [8 x float].
+/// \returns
+/// A 256-bit vector of [16 x fp16]. Lower elements correspond to the
+/// (converted) elements from \a __B; higher order elements correspond to
the
+/// (converted) elements from \a __A. If corresponding mask bit is not set,
then
+/// element from \a __W is taken instead.
static __inline__ __m256h __DEFAULT_FN_ATTRS256
_mm256_mask_cvtx2ps_ph(__m256h __W, __mmask16 __U, __m256 __A, __m256 __B) {
return (__m256h)__builtin_ia32_vcvt2ps2phx256_mask(
(__v8sf)__A, (__v8sf)__B, (__v16hf)__W, (__mmask16)__U,
_MM_FROUND_CUR_DIRECTION);
}
+/// Convert two 256-bit vectors, \a __A and \a __B, containing packed
+/// single-precision (32-bit) floating-point elements to a 256-bit vector
+/// containing FP16 elements. Zeroing mask \a __U is used to determine if
given
+/// element should be zeroed instead.
+///
+/// \code{.operation}
+/// FOR i := 0 to 15
+/// IF __U[i]
+/// IF i < 8
+/// dst.fp16[i] := convert_fp32_to_fp16(__B.fp32[i])
+/// ELSE
+/// dst.fp16[i] := convert_fp32_to_fp16(__A.fp32[i - 8])
+/// FI
+/// ELSE
+/// dst.fp16[i] := 0
+/// FI
+/// ENDFOR
+///
+/// dst[MAX:256] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VCVT2PS2PHX instruction.
+///
+/// \param __U
+/// A 16-bit zeroing mask.
+/// \param __A
+/// A 256-bit vector of [8 x float].
+/// \param __B
+/// A 256-bit vector of [8 x float].
+/// \returns
+/// A 256-bit vector of [16 x fp16]. Lower elements correspond to the
+/// (converted) elements from \a __B; higher order elements correspond to
the
+/// (converted) elements from \a __A. If corresponding mask bit is not set,
+/// then zero is taken instead.
static __inline__ __m256h __DEFAULT_FN_ATTRS256
_mm256_maskz_cvtx2ps_ph(__mmask16 __U, __m256 __A, __m256 __B) {
return (__m256h)__builtin_ia32_vcvt2ps2phx256_mask(
(__v8sf)__A, (__v8sf)__B, (__v16hf)_mm256_setzero_ph(), (__mmask16)__U,
_MM_FROUND_CUR_DIRECTION);
}
+/// Convert two 256-bit vectors, \a __A and \a __B, containing packed
+/// single-precision (32-bit) floating-point elements to a 256-bit vector
+/// containing FP16 elements. Rounding mode \a __R needs to be provided.
+///
+/// \code{.operation}
+/// FOR i := 0 to 15
+/// IF i < 8
+/// dst.fp16[i] := convert_fp32_to_fp16(__B.fp32[i])
+/// ELSE
+/// dst.fp16[i] := convert_fp32_to_fp16(__A.fp32[i - 8])
+/// FI
+/// ENDFOR
+///
+/// dst[MAX:256] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VCVT2PS2PHX instruction.
+///
+/// \param __A
+/// A 256-bit vector of [8 x float].
+/// \param __B
+/// A 256-bit vector of [8 x float].
+/// \param __R
+/// Rounding mode. Valid inputs are: _MM_FROUND_CUR_DIRECTION or
+/// result of bitwise or of _MM_FROUND_NO_EXC with at most one of the
following:
+/// _MM_FROUND_TO_NEAREST_INT, _MM_FROUND_TO_NEG_INF, _MM_FROUND_TO_POS_INF,
+/// _MM_FROUND_TO_ZERO.
+/// \returns
+/// A 256-bit vector of [16 x fp16]. Lower elements correspond to the
+/// (converted) elements from \a __B; higher order elements correspond to
the
+/// (converted) elements from \a __A.
#define _mm256_cvtx_round2ps_ph(A, B, R)
\
((__m256h)__builtin_ia32_vcvt2ps2phx256_mask(
\
(__v8sf)(A), (__v8sf)(B), (__v16hf)_mm256_undefined_ph(),
\
(__mmask16)(-1), (const int)(R)))
+/// Convert two 256-bit vectors, \a __A and \a __B, containing packed
+/// single-precision (32-bit) floating-point elements to a 256-bit vector
+/// containing FP16 elements. Merging mask \a __U is used to determine if
given
+/// element should be taken from \a __W instead. Rounding mode \a __R needs
to
+/// be provided.
+///
+/// \code{.operation}
+/// FOR i := 0 to 15
+/// IF __U[i]
+/// IF i < 8
+/// dst.fp16[i] := convert_fp32_to_fp16(__B.fp32[i])
+/// ELSE
+/// dst.fp16[i] := convert_fp32_to_fp16(__A.fp32[i - 8])
+/// FI
+/// ELSE
+/// dst.fp16[i] := __W.fp16[i]
+/// FI
+/// ENDFOR
+///
+/// dst[MAX:256] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VCVT2PS2PHX instruction.
+///
+/// \param __W
+/// A 256-bit vector of [16 x fp16].
+/// \param __U
+/// A 16-bit merging mask.
+/// \param __A
+/// A 256-bit vector of [8 x float].
+/// \param __B
+/// A 256-bit vector of [8 x float].
+/// \param __R
+/// Rounding mode. Valid inputs are: _MM_FROUND_CUR_DIRECTION or
+/// result of bitwise or of _MM_FROUND_NO_EXC with at most one of the
following:
+/// _MM_FROUND_TO_NEAREST_INT, _MM_FROUND_TO_NEG_INF, _MM_FROUND_TO_POS_INF,
+/// _MM_FROUND_TO_ZERO.
+/// \returns
+/// A 256-bit vector of [16 x fp16]. Lower elements correspond to the
+/// (converted) elements from \a __B; higher order elements correspond to
the
+/// (converted) elements from \a __A. If corresponding mask bit is not set,
then
+/// element from \a __W is taken instead.
#define _mm256_mask_cvtx_round2ps_ph(W, U, A, B, R)
\
((__m256h)__builtin_ia32_vcvt2ps2phx256_mask(
\
(__v8sf)(A), (__v8sf)(B), (__v16hf)(W), (__mmask16)(U), (const int)(R)))
+/// Convert two 256-bit vectors, \a __A and \a __B, containing packed
+/// single-precision (32-bit) floating-point elements to a 256-bit vector
+/// containing FP16 elements. Zeroing mask \a __U is used to determine if
given
+/// element should be zeroed instead. Rounding mode \a __R needs to be
provided.
+///
+/// \code{.operation}
+/// FOR i := 0 to 15
+/// IF __U[i]
+/// IF i < 8
+/// dst.fp16[i] := convert_fp32_to_fp16(__B.fp32[i])
+/// ELSE
+/// dst.fp16[i] := convert_fp32_to_fp16(__A.fp32[i - 8])
+/// FI
+/// ELSE
+/// dst.fp16[i] := 0
+/// FI
+/// ENDFOR
+///
+/// dst[MAX:256] := 0
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VCVT2PS2PHX instruction.
+///
+/// \param __U
+/// A 16-bit zeroing mask.
+/// \param __A
+/// A 256-bit vector of [8 x float].
+/// \param __B
+/// A 256-bit vector of [8 x float].
+/// \param __R
+/// Rounding mode. Valid inputs are: _MM_FROUND_CUR_DIRECTION or
+/// result of bitwise or of _MM_FROUND_NO_EXC with at most one of the
following:
+/// _MM_FROUND_TO_NEAREST_INT, _MM_FROUND_TO_NEG_INF, _MM_FROUND_TO_POS_INF,
+/// _MM_FROUND_TO_ZERO.
+/// \returns
+/// A 256-bit vector of [16 x fp16]. Lower elements correspond to the
+/// (converted) elements from \a __B; higher order elements correspond to
the
+/// (converted) elements from \a __A. If corresponding mask bit is not set,
+/// then zero is taken instead.
#define _mm256_maskz_cvtx_round2ps_ph(U, A, B, R)
\
((__m256h)__builtin_ia32_vcvt2ps2phx256_mask(
\
(__v8sf)(A), (__v8sf)(B), (__v16hf)(_mm256_setzero_ph()),
\
(__mmask16)(U), (const int)(R)))
+/// Add two 128-bit vectors, \a __A and \a __B, containing packed FP16
+/// floating-point elements and 8-bit integers stored in the lower half of
+/// packed 16-bit integers, respectively. Results are converted to FP8 E5M2.
+///
+/// \code{.operation}
+/// FOR i := 0 to 7
+/// dst.fp8[i] := add_convert_fp16_to_fp8_bias(__A.fp16[i], __B.int8[2i])
----------------
phoebewang wrote:
2*i
https://github.com/llvm/llvm-project/pull/120766
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
