/* * Copyright 2024 The LibYuv Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "libyuv/row.h" #include "libyuv/row_sve.h" #ifdef __cplusplus namespace libyuv { extern "C" { #endif #if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ defined(__aarch64__) #define RGBTOARGB8_SVE_2X \ /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \ "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \ "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \ "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \ "uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \ "uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \ "uqshrnt z18.b, z22.h, #6 \n" /* R1 */ __arm_locally_streaming void I444ToARGBRow_SME( const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { // Streaming-SVE only, no use of ZA tile. uint64_t vl; asm volatile( "cntb %[vl] \n" "ptrue p0.b \n" // YUVTORGB_SVE_SETUP "dup z19.b, #255 \n" // A "subs %w[width], %w[width], %w[vl] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p1.b \n" "1: \n" // READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X "subs %w[width], %w[width], %w[vl] \n" "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" "incb %[dst_argb], all, mul #4 \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. "whilelt p1.b, wzr, %w[width] \n" // READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] [src_u] "+r"(src_u), // %[src_u] [src_v] "+r"(src_v), // %[src_v] [dst_argb] "+r"(dst_argb), // %[dst_argb] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] : "cc", "memory", YUVTORGB_SVE_REGS); } __arm_locally_streaming void I444ToRGB24Row_SME( const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { I444ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_rgb24, yuvconstants, width); } __arm_locally_streaming void I400ToARGBRow_SME( const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { // Streaming-SVE only, no use of ZA tile. I400ToARGBRow_SVE_SC(src_y, dst_argb, yuvconstants, width); } __arm_locally_streaming void I422ToARGBRow_SME( const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { // Streaming-SVE only, no use of ZA tile. I422ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); } __arm_locally_streaming void I422ToRGB24Row_SME( const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { I422ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); } __arm_locally_streaming void I422ToRGB565Row_SME( const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_rgb565, const struct YuvConstants* yuvconstants, int width) { I422ToRGB565Row_SVE_SC(src_y, src_u, src_v, dst_rgb565, yuvconstants, width); } __arm_locally_streaming void I422ToARGB1555Row_SME( const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_argb1555, const struct YuvConstants* yuvconstants, int width) { I422ToARGB1555Row_SVE_SC(src_y, src_u, src_v, dst_argb1555, yuvconstants, width); } __arm_locally_streaming void I422ToARGB4444Row_SME( const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_argb4444, const struct YuvConstants* yuvconstants, int width) { I422ToARGB4444Row_SVE_SC(src_y, src_u, src_v, dst_argb4444, yuvconstants, width); } __arm_locally_streaming void I422ToRGBARow_SME( const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { I422ToRGBARow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); } __arm_locally_streaming void I422ToAR30Row_SME( const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { I422ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); } __arm_locally_streaming void I422AlphaToARGBRow_SME( const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, const uint8_t* src_a, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { I422AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, width); } __arm_locally_streaming void I444AlphaToARGBRow_SME( const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, const uint8_t* src_a, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { I444AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, width); } __arm_locally_streaming void NV12ToARGBRow_SME( const uint8_t* src_y, const uint8_t* src_uv, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { NV12ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width); } __arm_locally_streaming void NV21ToARGBRow_SME( const uint8_t* src_y, const uint8_t* src_vu, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { NV21ToARGBRow_SVE_SC(src_y, src_vu, dst_argb, yuvconstants, width); } __arm_locally_streaming void NV12ToRGB24Row_SME( const uint8_t* src_y, const uint8_t* src_uv, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { NV12ToRGB24Row_SVE_SC(src_y, src_uv, dst_rgb24, yuvconstants, width); } __arm_locally_streaming void NV21ToRGB24Row_SME( const uint8_t* src_y, const uint8_t* src_vu, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width) { NV21ToRGB24Row_SVE_SC(src_y, src_vu, dst_rgb24, yuvconstants, width); } __arm_locally_streaming void YUY2ToARGBRow_SME( const uint8_t* src_yuy2, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { YUY2ToARGBRow_SVE_SC(src_yuy2, dst_argb, yuvconstants, width); } __arm_locally_streaming void UYVYToARGBRow_SME( const uint8_t* src_uyvy, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { UYVYToARGBRow_SVE_SC(src_uyvy, dst_argb, yuvconstants, width); } __arm_locally_streaming void I210ToARGBRow_SME( const uint16_t* src_y, const uint16_t* src_u, const uint16_t* src_v, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { I210ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); } __arm_locally_streaming void I210AlphaToARGBRow_SME( const uint16_t* src_y, const uint16_t* src_u, const uint16_t* src_v, const uint16_t* src_a, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { I210AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, width); } __arm_locally_streaming void I210ToAR30Row_SME( const uint16_t* src_y, const uint16_t* src_u, const uint16_t* src_v, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width) { I210ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width); } __arm_locally_streaming void P210ToARGBRow_SME( const uint16_t* src_y, const uint16_t* src_uv, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { P210ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width); } __arm_locally_streaming void P210ToAR30Row_SME( const uint16_t* src_y, const uint16_t* src_uv, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width) { P210ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width); } __arm_locally_streaming void I410ToARGBRow_SME( const uint16_t* src_y, const uint16_t* src_u, const uint16_t* src_v, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { I410ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); } __arm_locally_streaming void I410AlphaToARGBRow_SME( const uint16_t* src_y, const uint16_t* src_u, const uint16_t* src_v, const uint16_t* src_a, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { I410AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, width); } __arm_locally_streaming void I410ToAR30Row_SME( const uint16_t* src_y, const uint16_t* src_u, const uint16_t* src_v, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width) { I410ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width); } __arm_locally_streaming void P410ToARGBRow_SME( const uint16_t* src_y, const uint16_t* src_uv, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { P410ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width); } __arm_locally_streaming void P410ToAR30Row_SME( const uint16_t* src_y, const uint16_t* src_uv, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width) { P410ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width); } __arm_locally_streaming void I212ToAR30Row_SME( const uint16_t* src_y, const uint16_t* src_u, const uint16_t* src_v, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width) { I212ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width); } __arm_locally_streaming void I212ToARGBRow_SME( const uint16_t* src_y, const uint16_t* src_u, const uint16_t* src_v, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { I212ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); } __arm_locally_streaming void MultiplyRow_16_SME(const uint16_t* src_y, uint16_t* dst_y, int scale, int width) { // Streaming-SVE only, no use of ZA tile. int vl; asm volatile( "cnth %x[vl] \n" "mov z0.h, %w[scale] \n" "subs %w[width], %w[width], %w[vl] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p0.h \n" "1: \n" "ld1h {z1.h}, p0/z, [%[src_y]] \n" "incb %[src_y] \n" "mul z1.h, z0.h, z1.h \n" "subs %w[width], %w[width], %w[vl] \n" "st1h {z1.h}, p0, [%[dst_y]] \n" "incb %[dst_y] \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. "whilelt p0.h, wzr, %w[width] \n" "ld1h {z1.h}, p0/z, [%[src_y]] \n" "mul z1.h, z0.h, z1.h \n" "st1h {z1.h}, p0, [%[dst_y]] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] [dst_y] "+r"(dst_y), // %[dst_y] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : [scale] "r"(scale) // %[scale] : "memory", "cc", "z0", "z1", "p0"); } __arm_locally_streaming void ARGBMultiplyRow_SME(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, int width) { // Streaming-SVE only, no use of ZA tile. width *= 4; int vl; asm volatile( "cntb %x[vl] \n" "subs %w[width], %w[width], %w[vl] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p0.b \n" "1: \n" "ld1b {z0.b}, p0/z, [%[src_argb]] \n" "ld1b {z1.b}, p0/z, [%[src_argb1]] \n" "incb %[src_argb] \n" "incb %[src_argb1] \n" "umullb z2.h, z0.b, z1.b \n" "umullt z1.h, z0.b, z1.b \n" "rshrnb z0.b, z2.h, #8 \n" "rshrnt z0.b, z1.h, #8 \n" "subs %w[width], %w[width], %w[vl] \n" "st1b {z0.b}, p0, [%[dst_argb]] \n" "incb %[dst_argb] \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. "whilelt p0.b, wzr, %w[width] \n" "ld1b {z0.b}, p0/z, [%[src_argb]] \n" "ld1b {z1.b}, p0/z, [%[src_argb1]] \n" "umullb z2.h, z0.b, z1.b \n" "umullt z1.h, z0.b, z1.b \n" "rshrnb z0.b, z2.h, #8 \n" "rshrnt z0.b, z1.h, #8 \n" "st1b {z0.b}, p0, [%[dst_argb]] \n" "99: \n" : [src_argb] "+r"(src_argb), // %[src_argb] [src_argb1] "+r"(src_argb1), // %[src_argb1] [dst_argb] "+r"(dst_argb), // %[dst_argb] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "z2", "p0", "p1"); } __arm_locally_streaming void MergeUVRow_SME(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width) { // Streaming-SVE only, no use of ZA tile. int vl; asm volatile( "cntb %x[vl] \n" "subs %w[width], %w[width], %w[vl] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p0.b \n" "1: \n" "ld1b {z1.b}, p0/z, [%[src_u]] \n" "ld1b {z2.b}, p0/z, [%[src_v]] \n" "incb %[src_u] \n" "incb %[src_v] \n" "subs %w[width], %w[width], %w[vl] \n" "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" "incb %[dst_uv], all, mul #2 \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. "whilelt p0.b, wzr, %w[width] \n" "ld1b {z1.b}, p0/z, [%[src_u]] \n" "ld1b {z2.b}, p0/z, [%[src_v]] \n" "subs %w[width], %w[width], %w[vl] \n" "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" "99: \n" : [src_u] "+r"(src_u), // %[src_u] [src_v] "+r"(src_v), // %[src_v] [dst_uv] "+r"(dst_uv), // %[dst_uv] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "z1", "z2", "p0"); } __arm_locally_streaming void MergeUVRow_16_SME(const uint16_t* src_u, const uint16_t* src_v, uint16_t* dst_uv, int depth, int width) { int shift = 16 - depth; // Streaming-SVE only, no use of ZA tile. int vl; asm volatile( "cnth %x[vl] \n" "mov z0.h, %w[shift] \n" "subs %w[width], %w[width], %w[vl] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p0.h \n" "1: \n" "ld1h {z1.h}, p0/z, [%[src_u]] \n" "ld1h {z2.h}, p0/z, [%[src_v]] \n" "incb %[src_u] \n" "incb %[src_v] \n" "lsl z1.h, p0/m, z1.h, z0.h \n" "lsl z2.h, p0/m, z2.h, z0.h \n" "subs %w[width], %w[width], %w[vl] \n" "st2h {z1.h, z2.h}, p0, [%[dst_uv]] \n" "incb %[dst_uv], all, mul #2 \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. "whilelt p0.h, wzr, %w[width] \n" "ld1h {z1.h}, p0/z, [%[src_u]] \n" "ld1h {z2.h}, p0/z, [%[src_v]] \n" "lsl z1.h, p0/m, z1.h, z0.h \n" "lsl z2.h, p0/m, z2.h, z0.h \n" "subs %w[width], %w[width], %w[vl] \n" "st2h {z1.h, z2.h}, p0, [%[dst_uv]] \n" "99: \n" : [src_u] "+r"(src_u), // %[src_u] [src_v] "+r"(src_v), // %[src_v] [dst_uv] "+r"(dst_uv), // %[dst_uv] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : [shift] "r"(shift) // %[shift] : "memory", "cc", "z0", "z1", "z2", "p0"); } // Use scale to convert lsb formats to msb, depending how many bits there are: // 32768 = 9 bits = shr 1 // 16384 = 10 bits = shr 2 // 4096 = 12 bits = shr 4 // 256 = 16 bits = shr 8 __arm_locally_streaming void Convert16To8Row_SME(const uint16_t* src_y, uint8_t* dst_y, int scale, int width) { // 15 - clz(scale), + 8 to shift result into the high half of the lane to // saturate, then we can just use UZP2 to narrow rather than a pair of // saturating narrow instructions. const int shift = 23 - __builtin_clz((int32_t)scale); int vl; asm volatile( "cntb %x[vl] \n" "dup z0.h, %w[shift] \n" "subs %w[width], %w[width], %w[vl] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p0.b \n" "1: \n" "ld1h {z1.h}, p0/z, [%[src_y]] \n" "ld1h {z2.h}, p0/z, [%[src_y], #1, mul vl] \n" "incb %[src_y], all, mul #2 \n" "uqshl z1.h, p0/m, z1.h, z0.h \n" "uqshl z2.h, p0/m, z2.h, z0.h \n" "subs %w[width], %w[width], %w[vl] \n" "uzp2 z1.b, z1.b, z2.b \n" "st1b {z1.b}, p0, [%[dst_y]] \n" "incb %[dst_y] \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. // We need separate predicates for the load and store instructions since // they are operating on different element sizes (.b vs .h). "cnth %x[vl] \n" "whilelt p0.h, wzr, %w[width] \n" "whilelt p1.h, %w[vl], %w[width] \n" "whilelt p2.b, wzr, %w[width] \n" "ld1h {z1.h}, p0/z, [%[src_y]] \n" "ld1h {z2.h}, p1/z, [%[src_y], #1, mul vl] \n" "uqshl z1.h, p0/m, z1.h, z0.h \n" "uqshl z2.h, p1/m, z2.h, z0.h \n" "uzp2 z1.b, z1.b, z2.b \n" "st1b {z1.b}, p2, [%[dst_y]] \n" "99: \n" : [src_y] "+r"(src_y), // %[src_y] [dst_y] "+r"(dst_y), // %[dst_y] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : [shift] "r"(shift) // %[shift] : "cc", "memory", "z0", "z1", "z2", "p0", "p1", "p2"); } __arm_locally_streaming void CopyRow_SME(const uint8_t* src, uint8_t* dst, int width) { // Streaming-SVE only, no use of ZA tile. int vl; asm volatile( "cntb %x[vl] \n" "subs %w[width], %w[width], %w[vl] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p0.b \n" "1: \n" "ld1b {z0.b}, p0/z, [%[src]] \n" "incb %[src] \n" "subs %w[width], %w[width], %w[vl] \n" "st1b {z0.b}, p0, [%[dst]] \n" "incb %[dst] \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. "whilelt p0.b, wzr, %w[width] \n" "ld1b {z0.b}, p0/z, [%[src]] \n" "st1b {z0.b}, p0, [%[dst]] \n" "99: \n" : [src] "+r"(src), // %[src] [dst] "+r"(dst), // %[dst] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : : "memory", "cc", "z0", "p0"); } __arm_locally_streaming static void HalfRow_SME(uint8_t* dst_ptr, const uint8_t* src_ptr, ptrdiff_t src_stride, int width) { const uint8_t* src_ptr1 = src_ptr + src_stride; int vl; asm volatile( "cntb %x[vl] \n" "subs %w[width], %w[width], %w[vl] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p0.b \n" "1: \n" "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" "incb %[src_ptr] \n" "incb %[src_ptr1] \n" "urhadd z2.b, p0/m, z2.b, z3.b \n" "subs %w[width], %w[width], %w[vl] \n" "st1b {z2.b}, p0, [%[dst_ptr]] \n" "incb %[dst_ptr] \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. "whilelt p0.b, wzr, %w[width] \n" "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" "urhadd z2.b, p0/m, z2.b, z3.b \n" "subs %w[width], %w[width], %w[vl] \n" "st1b {z2.b}, p0, [%[dst_ptr]] \n" "99: \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : : "cc", "memory", "z0", "z1", "z2", "z3", "p0"); } __arm_locally_streaming void InterpolateRow_SME(uint8_t* dst_ptr, const uint8_t* src_ptr, ptrdiff_t src_stride, int width, int source_y_fraction) { int y1_fraction = source_y_fraction; int y0_fraction = 256 - y1_fraction; const uint8_t* src_ptr1 = src_ptr + src_stride; if (y0_fraction == 0) { CopyRow_SME(src_ptr1, dst_ptr, width); return; } if (y0_fraction == 128) { HalfRow_SME(dst_ptr, src_ptr, src_stride, width); return; } if (y0_fraction == 256) { CopyRow_SME(src_ptr, dst_ptr, width); return; } int vl; asm volatile( "cntb %x[vl] \n" "dup z0.b, %w[y0_fraction] \n" "dup z1.b, %w[y1_fraction] \n" "subs %w[width], %w[width], %w[vl] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p0.b \n" "1: \n" "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" "incb %[src_ptr] \n" "incb %[src_ptr1] \n" "umullb z4.h, z2.b, z0.b \n" "umullt z2.h, z2.b, z0.b \n" "subs %w[width], %w[width], %w[vl] \n" "umlalb z4.h, z3.b, z1.b \n" "umlalt z2.h, z3.b, z1.b \n" "rshrnb z3.b, z4.h, #8 \n" "rshrnt z3.b, z2.h, #8 \n" "st1b {z3.b}, p0, [%[dst_ptr]] \n" "incb %[dst_ptr] \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. "whilelt p0.b, wzr, %w[width] \n" "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" "umullb z4.h, z2.b, z0.b \n" "umullt z2.h, z2.b, z0.b \n" "umlalb z4.h, z3.b, z1.b \n" "umlalt z2.h, z3.b, z1.b \n" "rshrnb z3.b, z4.h, #8 \n" "rshrnt z3.b, z2.h, #8 \n" "st1b {z3.b}, p0, [%[dst_ptr]] \n" "99: \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] [y1_fraction] "r"(y1_fraction) // %[y1_fraction] : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0"); } __arm_locally_streaming static void HalfRow_16_SME(uint16_t* dst_ptr, const uint16_t* src_ptr, ptrdiff_t src_stride, int width) { const uint16_t* src_ptr1 = src_ptr + src_stride; int vl; asm volatile( "cnth %x[vl] \n" "subs %w[width], %w[width], %w[vl] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p0.h \n" "1: \n" "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" "incb %[src_ptr] \n" "incb %[src_ptr1] \n" "urhadd z2.h, p0/m, z2.h, z3.h \n" "subs %w[width], %w[width], %w[vl] \n" "st1h {z2.h}, p0, [%[dst_ptr]] \n" "incb %[dst_ptr] \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. "whilelt p0.h, wzr, %w[width] \n" "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" "urhadd z2.h, p0/m, z2.h, z3.h \n" "st1h {z2.h}, p0, [%[dst_ptr]] \n" "99: \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : : "cc", "memory", "z0", "z1", "z2", "z3", "p0"); } __arm_locally_streaming void InterpolateRow_16_SME(uint16_t* dst_ptr, const uint16_t* src_ptr, ptrdiff_t src_stride, int width, int source_y_fraction) { int y1_fraction = source_y_fraction; int y0_fraction = 256 - y1_fraction; const uint16_t* src_ptr1 = src_ptr + src_stride; if (y0_fraction == 0) { CopyRow_SME((const uint8_t*)src_ptr1, (uint8_t*)dst_ptr, width * sizeof(uint16_t)); return; } if (y0_fraction == 128) { HalfRow_16_SME(dst_ptr, src_ptr, src_stride, width); return; } if (y0_fraction == 256) { CopyRow_SME((const uint8_t*)src_ptr, (uint8_t*)dst_ptr, width * sizeof(uint16_t)); return; } int vl; asm volatile( "cnth %x[vl] \n" "subs %w[width], %w[width], %w[vl] \n" "dup z0.h, %w[y0_fraction] \n" "dup z1.h, %w[y1_fraction] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p0.h \n" "1: \n" "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" "incb %[src_ptr] \n" "incb %[src_ptr1] \n" "umullb z4.s, z2.h, z0.h \n" "umullt z2.s, z2.h, z0.h \n" "subs %w[width], %w[width], %w[vl] \n" "umlalb z4.s, z3.h, z1.h \n" "umlalt z2.s, z3.h, z1.h \n" "rshrnb z3.h, z4.s, #8 \n" "rshrnt z3.h, z2.s, #8 \n" "st1h {z3.h}, p0, [%[dst_ptr]] \n" "incb %[dst_ptr] \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. "whilelt p0.h, wzr, %w[width] \n" "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" "umullb z4.s, z2.h, z0.h \n" "umullt z2.s, z2.h, z0.h \n" "umlalb z4.s, z3.h, z1.h \n" "umlalt z2.s, z3.h, z1.h \n" "rshrnb z3.h, z4.s, #8 \n" "rshrnt z3.h, z2.s, #8 \n" "st1h {z3.h}, p0, [%[dst_ptr]] \n" "99: \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] [y1_fraction] "r"(y1_fraction) // %[y1_fraction] : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0"); } __arm_locally_streaming static void HalfRow_16To8_SME(uint8_t* dst_ptr, const uint16_t* src_ptr, ptrdiff_t src_stride, int scale, int width) { const uint16_t* src_ptr1 = src_ptr + src_stride; // 15 - clz(scale), + 8 to shift result into the high half of the lane to // saturate, then we can just use UZP2 to narrow rather than a pair of // saturating narrow instructions. const int shift = 23 - __builtin_clz((int32_t)scale); int vl; asm volatile( "cnth %x[vl] \n" "dup z31.h, %w[shift] \n" "subs %w[width], %w[width], %w[vl] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p0.h \n" "1: \n" "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" "incb %[src_ptr] \n" "incb %[src_ptr1] \n" "urhadd z2.h, p0/m, z2.h, z3.h \n" "subs %w[width], %w[width], %w[vl] \n" "uqshl z2.h, p0/m, z2.h, z31.h \n" "shrnb z2.b, z2.h, #8 \n" "st1b {z2.h}, p0, [%[dst_ptr]] \n" "inch %[dst_ptr] \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. "whilelt p0.h, wzr, %w[width] \n" "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" "urhadd z2.h, p0/m, z2.h, z3.h \n" "uqshl z2.h, p0/m, z2.h, z31.h \n" "shrnb z2.b, z2.h, #8 \n" "st1b {z2.h}, p0, [%[dst_ptr]] \n" "99: \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : [shift] "r"(shift) // %[shift] : "cc", "memory", "z0", "z1", "z2", "z3", "z31", "p0"); } // Use scale to convert lsb formats to msb, depending how many bits there are: // 32768 = 9 bits // 16384 = 10 bits // 4096 = 12 bits // 256 = 16 bits // TODO(fbarchard): change scale to bits __arm_locally_streaming void InterpolateRow_16To8_SME(uint8_t* dst_ptr, const uint16_t* src_ptr, ptrdiff_t src_stride, int scale, int width, int source_y_fraction) { const int y1_fraction = source_y_fraction; const int y0_fraction = 256 - y1_fraction; const uint16_t* src_ptr1 = src_ptr + src_stride; // y0_fraction == 0 is never called here. if (y0_fraction == 128) { HalfRow_16To8_SME(dst_ptr, src_ptr, src_stride, scale, width); return; } if (y0_fraction == 256) { Convert16To8Row_SME(src_ptr, dst_ptr, scale, width); return; } // 15 - clz(scale), + 8 to shift result into the high half of the lane to // saturate, then we can just use UZP2 to narrow rather than a pair of // saturating narrow instructions. const int shift = 23 - __builtin_clz((int32_t)scale); int vl; asm volatile( "cnth %x[vl] \n" "dup z31.h, %w[shift] \n" "dup z0.h, %w[y0_fraction] \n" "dup z1.h, %w[y1_fraction] \n" "subs %w[width], %w[width], %w[vl] \n" "b.lt 2f \n" // Run bulk of computation with an all-true predicate to avoid predicate // generation overhead. "ptrue p0.h \n" "1: \n" "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" "incb %[src_ptr] \n" "incb %[src_ptr1] \n" "umullb z4.s, z2.h, z0.h \n" "umullt z2.s, z2.h, z0.h \n" "subs %w[width], %w[width], %w[vl] \n" "umlalb z4.s, z3.h, z1.h \n" "umlalt z2.s, z3.h, z1.h \n" "rshrnb z3.h, z4.s, #8 \n" "rshrnt z3.h, z2.s, #8 \n" "uqshl z3.h, p0/m, z3.h, z31.h \n" "shrnb z3.b, z3.h, #8 \n" "st1b {z3.h}, p0, [%[dst_ptr]] \n" "inch %[dst_ptr] \n" "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl] \n" "b.eq 99f \n" // Calculate a predicate for the final iteration to deal with the tail. "whilelt p0.h, wzr, %w[width] \n" "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" "umullb z4.s, z2.h, z0.h \n" "umullt z2.s, z2.h, z0.h \n" "umlalb z4.s, z3.h, z1.h \n" "umlalt z2.s, z3.h, z1.h \n" "rshrnb z3.h, z4.s, #8 \n" "rshrnt z3.h, z2.s, #8 \n" "uqshl z3.h, p0/m, z3.h, z31.h \n" "shrnb z3.b, z3.h, #8 \n" "st1b {z3.h}, p0, [%[dst_ptr]] \n" "99: \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] [y1_fraction] "r"(y1_fraction), // %[y1_fraction] [shift] "r"(shift) // %[shift] : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z31", "p0"); } __arm_locally_streaming void Convert8To8Row_SME(const uint8_t* src_y, uint8_t* dst_y, int scale, int bias, int width) { Convert8To8Row_SVE_SC(src_y, dst_y, scale, bias, width); } #define CONVERT8TO16_SVE \ "ld1b {z0.h}, p0/z, [%[src]] \n" \ "ld1b {z1.h}, p1/z, [%[src], #1, mul vl] \n" \ "incb %[src] \n" \ "subs %w[width], %w[width], %w[vl], lsl #1 \n" \ "trn1 z0.b, z0.b, z0.b \n" \ "trn1 z1.b, z1.b, z1.b \n" \ "lsr z0.h, p0/m, z0.h, z2.h \n" \ "lsr z1.h, p1/m, z1.h, z2.h \n" \ "prfm pldl1keep, [%[src], 448] \n" \ "st1h {z0.h}, p0, [%[dst]] \n" \ "st1h {z1.h}, p1, [%[dst], #1, mul vl] \n" \ "incb %[dst], all, mul #2 \n" __arm_locally_streaming void Convert8To16Row_SME(const uint8_t* src_y, uint16_t* dst_y, int scale, int width) { // (src * 0x0101 * scale) >> 16. // Since scale is a power of two, compute the shift to use to avoid needing // to widen to int32. const int shift = __builtin_clz(scale) - 15; uint64_t vl; asm volatile( "dup z2.h, %w[shift] \n" "cnth %[vl] \n" "subs %w[width], %w[width], %w[vl], lsl #1 \n" "b.lt 2f \n" // Run bulk of computation with all-true predicates to avoid predicate // generation overhead. "ptrue p0.h \n" "ptrue p1.h \n" "1: \n" // CONVERT8TO16_SVE "b.ge 1b \n" "2: \n" "adds %w[width], %w[width], %w[vl], lsl #1 \n" "b.eq 99f \n" // Calculate predicates for the final iteration to deal with the tail. "whilelt p0.h, wzr, %w[width] \n" "whilelt p1.h, %w[vl], %w[width] \n" // CONVERT8TO16_SVE "99: \n" : [src] "+r"(src_y), // %[src] [dst] "+r"(dst_y), // %[dst] [width] "+r"(width), // %[width] [vl] "=&r"(vl) // %[vl] : [shift] "r"(shift) // %[shift] : "cc", "memory", "z0", "z1", "z2", "p0", "p1"); } __arm_locally_streaming void ARGBToUVRow_SME(const uint8_t* src_argb, int src_stride_argb, uint8_t* dst_u, uint8_t* dst_v, int width) { ARGBToUVMatrixRow_SVE_SC(src_argb, src_stride_argb, dst_u, dst_v, width, kARGBToUVCoefficients); } __arm_locally_streaming void ARGBToUVJRow_SME(const uint8_t* src_argb, int src_stride_argb, uint8_t* dst_u, uint8_t* dst_v, int width) { ARGBToUVMatrixRow_SVE_SC(src_argb, src_stride_argb, dst_u, dst_v, width, kARGBToUVJCoefficients); } __arm_locally_streaming void ABGRToUVJRow_SME(const uint8_t* src_abgr, int src_stride_abgr, uint8_t* dst_uj, uint8_t* dst_vj, int width) { ARGBToUVMatrixRow_SVE_SC(src_abgr, src_stride_abgr, dst_uj, dst_vj, width, kABGRToUVJCoefficients); } __arm_locally_streaming void BGRAToUVRow_SME(const uint8_t* src_bgra, int src_stride_bgra, uint8_t* dst_u, uint8_t* dst_v, int width) { ARGBToUVMatrixRow_SVE_SC(src_bgra, src_stride_bgra, dst_u, dst_v, width, kBGRAToUVCoefficients); } __arm_locally_streaming void ABGRToUVRow_SME(const uint8_t* src_abgr, int src_stride_abgr, uint8_t* dst_u, uint8_t* dst_v, int width) { ARGBToUVMatrixRow_SVE_SC(src_abgr, src_stride_abgr, dst_u, dst_v, width, kABGRToUVCoefficients); } __arm_locally_streaming void RGBAToUVRow_SME(const uint8_t* src_rgba, int src_stride_rgba, uint8_t* dst_u, uint8_t* dst_v, int width) { ARGBToUVMatrixRow_SVE_SC(src_rgba, src_stride_rgba, dst_u, dst_v, width, kRGBAToUVCoefficients); } #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && // defined(__aarch64__) #ifdef __cplusplus } // extern "C" } // namespace libyuv #endif