/* * Copyright 2014 The LibYuv Project Authors. All rights reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "libyuv/row.h" #include "libyuv/scale.h" #include "libyuv/scale_row.h" #ifdef __cplusplus namespace libyuv { extern "C" { #endif // This module is for GCC Neon armv8 64 bit. #if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) // Read 32x1 throw away even pixels, and write 16x1. void ScaleRowDown2_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { (void)src_stride; asm volatile( "1: \n" // load even pixels into v0, odd into v1 "ld2 {v0.16b,v1.16b}, [%0], #32 \n" "subs %w2, %w2, #16 \n" // 16 processed per loop "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "st1 {v1.16b}, [%1], #16 \n" // store odd pixels "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(dst), // %1 "+r"(dst_width) // %2 : : "memory", "cc", "v0", "v1" // Clobber List ); } // Read 32x1 average down and write 16x1. void ScaleRowDown2Linear_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { (void)src_stride; asm volatile( "1: \n" // load even pixels into v0, odd into v1 "ld2 {v0.16b,v1.16b}, [%0], #32 \n" "subs %w2, %w2, #16 \n" // 16 processed per loop "urhadd v0.16b, v0.16b, v1.16b \n" // rounding half add "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "st1 {v0.16b}, [%1], #16 \n" "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(dst), // %1 "+r"(dst_width) // %2 : : "memory", "cc", "v0", "v1" // Clobber List ); } // Read 32x2 average down and write 16x1. void ScaleRowDown2Box_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { asm volatile( // change the stride to row 2 pointer "add %1, %1, %0 \n" "1: \n" "ld1 {v0.16b, v1.16b}, [%0], #32 \n" // load row 1 and post inc "ld1 {v2.16b, v3.16b}, [%1], #32 \n" // load row 2 and post inc "subs %w3, %w3, #16 \n" // 16 processed per loop "uaddlp v0.8h, v0.16b \n" // row 1 add adjacent "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "uaddlp v1.8h, v1.16b \n" "prfm pldl1keep, [%1, 448] \n" "uadalp v0.8h, v2.16b \n" // += row 2 add adjacent "uadalp v1.8h, v3.16b \n" "rshrn v0.8b, v0.8h, #2 \n" // round and pack "rshrn2 v0.16b, v1.8h, #2 \n" "st1 {v0.16b}, [%2], #16 \n" "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_stride), // %1 "+r"(dst), // %2 "+r"(dst_width) // %3 : : "memory", "cc", "v0", "v1", "v2", "v3" // Clobber List ); } void ScaleRowDown4_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { (void)src_stride; asm volatile( "1: \n" "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // src line 0 "subs %w2, %w2, #16 \n" // 16 processed per loop "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "st1 {v2.16b}, [%1], #16 \n" "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(dst_ptr), // %1 "+r"(dst_width) // %2 : : "memory", "cc", "v0", "v1", "v2", "v3"); } void ScaleRowDown4Box_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { const uint8_t* src_ptr1 = src_ptr + src_stride; const uint8_t* src_ptr2 = src_ptr + src_stride * 2; const uint8_t* src_ptr3 = src_ptr + src_stride * 3; asm volatile( "1: \n" "ldp q0, q4, [%0], #32 \n" // load up 16x8 "ldp q1, q5, [%2], #32 \n" "ldp q2, q6, [%3], #32 \n" "ldp q3, q7, [%4], #32 \n" "subs %w5, %w5, #8 \n" "uaddlp v0.8h, v0.16b \n" "uaddlp v4.8h, v4.16b \n" "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "uadalp v0.8h, v1.16b \n" "uadalp v4.8h, v5.16b \n" "prfm pldl1keep, [%2, 448] \n" "uadalp v0.8h, v2.16b \n" "uadalp v4.8h, v6.16b \n" "prfm pldl1keep, [%3, 448] \n" "uadalp v0.8h, v3.16b \n" "uadalp v4.8h, v7.16b \n" "prfm pldl1keep, [%4, 448] \n" "addp v0.8h, v0.8h, v4.8h \n" "rshrn v0.8b, v0.8h, #4 \n" // divide by 16 w/rounding "str d0, [%1], #8 \n" "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(dst_ptr), // %1 "+r"(src_ptr1), // %2 "+r"(src_ptr2), // %3 "+r"(src_ptr3), // %4 "+r"(dst_width) // %5 : : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); } static const uvec8 kShuf34_0 = { 0, 1, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 19, 20, }; static const uvec8 kShuf34_1 = { 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 19, 20, 21, 23, 24, 25, }; static const uvec8 kShuf34_2 = { 11, 12, 13, 15, 16, 17, 19, 20, 21, 23, 24, 25, 27, 28, 29, 31, }; // Down scale from 4 to 3 pixels. Point samples 64 pixels to 48 pixels. void ScaleRowDown34_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { (void)src_stride; asm volatile( "ld1 {v29.16b}, [%[kShuf34_0]] \n" "ld1 {v30.16b}, [%[kShuf34_1]] \n" "ld1 {v31.16b}, [%[kShuf34_2]] \n" "1: \n" "ld1 {v0.16b,v1.16b,v2.16b,v3.16b}, [%[src_ptr]], #64 \n" "subs %w[width], %w[width], #48 \n" "tbl v0.16b, {v0.16b, v1.16b}, v29.16b \n" "prfm pldl1keep, [%[src_ptr], 448] \n" "tbl v1.16b, {v1.16b, v2.16b}, v30.16b \n" "tbl v2.16b, {v2.16b, v3.16b}, v31.16b \n" "st1 {v0.16b,v1.16b,v2.16b}, [%[dst_ptr]], #48 \n" "b.gt 1b \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [width] "+r"(dst_width) // %[width] : [kShuf34_0] "r"(&kShuf34_0), // %[kShuf34_0] [kShuf34_1] "r"(&kShuf34_1), // %[kShuf34_1] [kShuf34_2] "r"(&kShuf34_2) // %[kShuf34_2] : "memory", "cc", "v0", "v1", "v2", "v3", "v29", "v30", "v31"); } void ScaleRowDown34_0_Box_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { asm volatile( "movi v24.16b, #3 \n" "add %3, %3, %0 \n" "1: \n" "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // src line 0 "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%3], #64 \n" // src line 1 "subs %w2, %w2, #48 \n" // filter src line 0 with src line 1 // expand chars to shorts to allow for room // when adding lines together "ushll v16.8h, v4.8b, #0 \n" "ushll v17.8h, v5.8b, #0 \n" "ushll v18.8h, v6.8b, #0 \n" "ushll v19.8h, v7.8b, #0 \n" "ushll2 v20.8h, v4.16b, #0 \n" "ushll2 v21.8h, v5.16b, #0 \n" "ushll2 v22.8h, v6.16b, #0 \n" "ushll2 v23.8h, v7.16b, #0 \n" // 3 * line_0 + line_1 "umlal v16.8h, v0.8b, v24.8b \n" "umlal v17.8h, v1.8b, v24.8b \n" "umlal v18.8h, v2.8b, v24.8b \n" "umlal v19.8h, v3.8b, v24.8b \n" "umlal2 v20.8h, v0.16b, v24.16b \n" "umlal2 v21.8h, v1.16b, v24.16b \n" "umlal2 v22.8h, v2.16b, v24.16b \n" "umlal2 v23.8h, v3.16b, v24.16b \n" "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead // (3 * line_0 + line_1 + 2) >> 2 "uqrshrn v0.8b, v16.8h, #2 \n" "uqrshrn v1.8b, v17.8h, #2 \n" "uqrshrn v2.8b, v18.8h, #2 \n" "uqrshrn v3.8b, v19.8h, #2 \n" "uqrshrn2 v0.16b, v20.8h, #2 \n" "uqrshrn2 v1.16b, v21.8h, #2 \n" "uqrshrn2 v2.16b, v22.8h, #2 \n" "uqrshrn2 v3.16b, v23.8h, #2 \n" "prfm pldl1keep, [%3, 448] \n" // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2 "ushll v16.8h, v1.8b, #0 \n" "ushll2 v17.8h, v1.16b, #0 \n" "umlal v16.8h, v0.8b, v24.8b \n" "umlal2 v17.8h, v0.16b, v24.16b \n" "uqrshrn v0.8b, v16.8h, #2 \n" "uqrshrn2 v0.16b, v17.8h, #2 \n" // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1 "urhadd v1.16b, v1.16b, v2.16b \n" // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2 "ushll v16.8h, v2.8b, #0 \n" "ushll2 v17.8h, v2.16b, #0 \n" "umlal v16.8h, v3.8b, v24.8b \n" "umlal2 v17.8h, v3.16b, v24.16b \n" "uqrshrn v2.8b, v16.8h, #2 \n" "uqrshrn2 v2.16b, v17.8h, #2 \n" "st3 {v0.16b,v1.16b,v2.16b}, [%1], #48 \n" "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(dst_ptr), // %1 "+r"(dst_width), // %2 "+r"(src_stride) // %3 : : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24"); } void ScaleRowDown34_1_Box_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { asm volatile( "movi v20.16b, #3 \n" "add %3, %3, %0 \n" "1: \n" "ld4 {v0.16b,v1.16b,v2.16b,v3.16b}, [%0], #64 \n" // src line 0 "ld4 {v4.16b,v5.16b,v6.16b,v7.16b}, [%3], #64 \n" // src line 1 "subs %w2, %w2, #48 \n" // average src line 0 with src line 1 "urhadd v0.16b, v0.16b, v4.16b \n" "urhadd v1.16b, v1.16b, v5.16b \n" "urhadd v2.16b, v2.16b, v6.16b \n" "urhadd v3.16b, v3.16b, v7.16b \n" "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead // a0 = (src[0] * 3 + s[1] * 1 + 2) >> 2 "ushll v4.8h, v1.8b, #0 \n" "ushll2 v5.8h, v1.16b, #0 \n" "umlal v4.8h, v0.8b, v20.8b \n" "umlal2 v5.8h, v0.16b, v20.16b \n" "uqrshrn v0.8b, v4.8h, #2 \n" "uqrshrn2 v0.16b, v5.8h, #2 \n" "prfm pldl1keep, [%3, 448] \n" // a1 = (src[1] * 1 + s[2] * 1 + 1) >> 1 "urhadd v1.16b, v1.16b, v2.16b \n" // a2 = (src[2] * 1 + s[3] * 3 + 2) >> 2 "ushll v4.8h, v2.8b, #0 \n" "ushll2 v5.8h, v2.16b, #0 \n" "umlal v4.8h, v3.8b, v20.8b \n" "umlal2 v5.8h, v3.16b, v20.16b \n" "uqrshrn v2.8b, v4.8h, #2 \n" "uqrshrn2 v2.16b, v5.8h, #2 \n" "st3 {v0.16b,v1.16b,v2.16b}, [%1], #48 \n" "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(dst_ptr), // %1 "+r"(dst_width), // %2 "+r"(src_stride) // %3 : : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v20"); } static const uvec8 kShuf38 = {0, 3, 6, 8, 11, 14, 16, 19, 22, 24, 27, 30, 0, 0, 0, 0}; static const vec16 kMult38_Div664 = { 65536 / 12, 65536 / 12, 65536 / 8, 65536 / 12, 65536 / 12, 65536 / 8, 0, 0}; static const vec16 kMult38_Div996 = {65536 / 18, 65536 / 18, 65536 / 12, 65536 / 18, 65536 / 18, 65536 / 12, 0, 0}; // 32 -> 12 void ScaleRowDown38_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { (void)src_stride; asm volatile( "ld1 {v3.16b}, [%[kShuf38]] \n" "subs %w[width], %w[width], #12 \n" "b.eq 2f \n" "1: \n" "ldp q0, q1, [%[src_ptr]], #32 \n" "subs %w[width], %w[width], #12 \n" "tbl v2.16b, {v0.16b, v1.16b}, v3.16b \n" "prfm pldl1keep, [%[src_ptr], 448] \n" // prefetch 7 lines ahead "str q2, [%[dst_ptr]] \n" "add %[dst_ptr], %[dst_ptr], #12 \n" "b.gt 1b \n" // Store exactly 12 bytes on the final iteration to avoid writing past // the end of the array. "2: \n" "ldp q0, q1, [%[src_ptr]] \n" "tbl v2.16b, {v0.16b, v1.16b}, v3.16b \n" "st1 {v2.8b}, [%[dst_ptr]], #8 \n" "st1 {v2.s}[2], [%[dst_ptr]] \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [width] "+r"(dst_width) // %[width] : [kShuf38] "r"(&kShuf38) // %[kShuf38] : "memory", "cc", "v0", "v1", "v2", "v3"); } static const uvec8 kScaleRowDown38_3_BoxIndices1[] = { 0, 1, 6, 7, 12, 13, 16, 17, 22, 23, 28, 29, 255, 255, 255, 255}; static const uvec8 kScaleRowDown38_3_BoxIndices2[] = { 2, 3, 8, 9, 14, 15, 18, 19, 24, 25, 30, 31, 255, 255, 255, 255}; static const uvec8 kScaleRowDown38_3_BoxIndices3[] = { 4, 5, 10, 11, 255, 255, 20, 21, 26, 27, 255, 255, 255, 255, 255, 255}; static const uvec8 kScaleRowDown38_NarrowIndices[] = { 0, 2, 4, 6, 8, 10, 16, 18, 20, 22, 24, 26, 255, 255, 255, 255}; void ScaleRowDown38_3_Box_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { const uint8_t* src_ptr1 = src_ptr + src_stride; const uint8_t* src_ptr2 = src_ptr + src_stride * 2; asm volatile( "ld1 {v27.16b}, [%[tblArray1]] \n" "ld1 {v28.16b}, [%[tblArray2]] \n" "ld1 {v29.16b}, [%[tblArray3]] \n" "ld1 {v31.16b}, [%[tblArray4]] \n" "ld1 {v30.16b}, [%[div996]] \n" "1: \n" "ldp q20, q0, [%[src_ptr]], #32 \n" "ldp q21, q1, [%[src_ptr1]], #32 \n" "ldp q22, q2, [%[src_ptr2]], #32 \n" "subs %w[width], %w[width], #12 \n" // Add across strided rows first. "uaddl v23.8h, v20.8b, v21.8b \n" "uaddl v3.8h, v0.8b, v1.8b \n" "uaddl2 v24.8h, v20.16b, v21.16b \n" "uaddl2 v4.8h, v0.16b, v1.16b \n" "uaddw v23.8h, v23.8h, v22.8b \n" "uaddw v3.8h, v3.8h, v2.8b \n" "uaddw2 v24.8h, v24.8h, v22.16b \n" // abcdefgh ... "uaddw2 v4.8h, v4.8h, v2.16b \n" // Permute groups of {three,three,two} into separate vectors to sum. "tbl v20.16b, {v23.16b, v24.16b}, v27.16b \n" // a d g ... "tbl v0.16b, {v3.16b, v4.16b}, v27.16b \n" "tbl v21.16b, {v23.16b, v24.16b}, v28.16b \n" // b e h ... "tbl v1.16b, {v3.16b, v4.16b}, v28.16b \n" "tbl v22.16b, {v23.16b, v24.16b}, v29.16b \n" // c f 0... "tbl v2.16b, {v3.16b, v4.16b}, v29.16b \n" "add v23.8h, v20.8h, v21.8h \n" "add v3.8h, v0.8h, v1.8h \n" "add v24.8h, v23.8h, v22.8h \n" // a+b+c d+e+f g+h "add v4.8h, v3.8h, v2.8h \n" "sqrdmulh v24.8h, v24.8h, v30.8h \n" // v /= {9,9,6} "sqrdmulh v25.8h, v4.8h, v30.8h \n" "tbl v21.16b, {v24.16b, v25.16b}, v31.16b \n" // Narrow. "st1 {v21.d}[0], [%[dst_ptr]], #8 \n" "st1 {v21.s}[2], [%[dst_ptr]], #4 \n" "b.gt 1b \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] [src_ptr2] "+r"(src_ptr2), // %[src_ptr2] [width] "+r"(dst_width) // %[width] : [div996] "r"(&kMult38_Div996), // %[div996] [tblArray1] "r"(kScaleRowDown38_3_BoxIndices1), // %[tblArray1] [tblArray2] "r"(kScaleRowDown38_3_BoxIndices2), // %[tblArray2] [tblArray3] "r"(kScaleRowDown38_3_BoxIndices3), // %[tblArray3] [tblArray4] "r"(kScaleRowDown38_NarrowIndices) // %[tblArray4] : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v20", "v21", "22", "23", "24", "v27", "v28", "v29", "v30", "v31"); } static const uvec8 kScaleRowDown38_2_BoxIndices1[] = { 0, 1, 3, 4, 6, 7, 8, 9, 11, 12, 14, 15, 255, 255, 255, 255}; static const uvec8 kScaleRowDown38_2_BoxIndices2[] = { 2, 18, 5, 21, 255, 255, 10, 26, 13, 29, 255, 255, 255, 255, 255, 255}; void ScaleRowDown38_2_Box_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { const uint8_t* src_ptr1 = src_ptr + src_stride; asm volatile( "ld1 {v28.16b}, [%[tblArray1]] \n" "ld1 {v29.16b}, [%[tblArray2]] \n" "ld1 {v31.16b}, [%[tblArray3]] \n" "ld1 {v30.8h}, [%[div664]] \n" "1: \n" "ldp q20, q0, [%[src_ptr]], #32 \n" // abcdefgh ... "ldp q21, q1, [%[src_ptr1]], #32 \n" // ijklmnop ... "subs %w[width], %w[width], #12 \n" // Permute into groups of six values (three pairs) to be summed. "tbl v22.16b, {v20.16b}, v28.16b \n" // abdegh ... "tbl v2.16b, {v0.16b}, v28.16b \n" "tbl v23.16b, {v21.16b}, v28.16b \n" // ijlmop ... "tbl v3.16b, {v1.16b}, v28.16b \n" "tbl v24.16b, {v20.16b, v21.16b}, v29.16b \n" // ckfn00 ... "tbl v4.16b, {v0.16b, v1.16b}, v29.16b \n" "uaddlp v22.8h, v22.16b \n" // a+b d+e g+h ... "uaddlp v2.8h, v2.16b \n" "uaddlp v23.8h, v23.16b \n" // i+j l+m o+p ... "uaddlp v3.8h, v3.16b \n" "uaddlp v24.8h, v24.16b \n" // c+k f+n 0 ... "uaddlp v4.8h, v4.16b \n" "add v20.8h, v22.8h, v23.8h \n" "add v0.8h, v2.8h, v3.8h \n" "add v21.8h, v20.8h, v24.8h \n" // a+b+i+j+c+k ... "add v1.8h, v0.8h, v4.8h \n" "sqrdmulh v21.8h, v21.8h, v30.8h \n" // v /= {6,6,4} "sqrdmulh v22.8h, v1.8h, v30.8h \n" "tbl v21.16b, {v21.16b, v22.16b}, v31.16b \n" // Narrow. "st1 {v21.d}[0], [%[dst_ptr]], #8 \n" "st1 {v21.s}[2], [%[dst_ptr]], #4 \n" "b.gt 1b \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] [width] "+r"(dst_width) // %[width] : [div664] "r"(&kMult38_Div664), // %[div664] [tblArray1] "r"(kScaleRowDown38_2_BoxIndices1), // %[tblArray1] [tblArray2] "r"(kScaleRowDown38_2_BoxIndices2), // %[tblArray2] [tblArray3] "r"(kScaleRowDown38_NarrowIndices) // %[tblArray3] : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v20", "v21", "v22", "v23", "v24", "v28", "v29", "v30", "v31"); } void ScaleRowUp2_Linear_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width) { const uint8_t* src_temp = src_ptr + 1; asm volatile( "movi v31.16b, #3 \n" "1: \n" "ldr q0, [%0], #16 \n" // 0123456789abcdef "ldr q1, [%1], #16 \n" // 123456789abcdefg "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "ushll v2.8h, v0.8b, #0 \n" // 01234567 (16b) "ushll v3.8h, v1.8b, #0 \n" // 12345678 (16b) "ushll2 v4.8h, v0.16b, #0 \n" // 89abcdef (16b) "ushll2 v5.8h, v1.16b, #0 \n" // 9abcdefg (16b) "umlal v2.8h, v1.8b, v31.8b \n" // 3*near+far (odd) "umlal v3.8h, v0.8b, v31.8b \n" // 3*near+far (even) "umlal2 v4.8h, v1.16b, v31.16b \n" // 3*near+far (odd) "umlal2 v5.8h, v0.16b, v31.16b \n" // 3*near+far (even) "rshrn v2.8b, v2.8h, #2 \n" // 3/4*near+1/4*far (odd) "rshrn v1.8b, v3.8h, #2 \n" // 3/4*near+1/4*far (even) "rshrn2 v2.16b, v4.8h, #2 \n" // 3/4*near+1/4*far (odd) "rshrn2 v1.16b, v5.8h, #2 \n" // 3/4*near+1/4*far (even) "st2 {v1.16b, v2.16b}, [%2], #32 \n" "subs %w3, %w3, #32 \n" "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_temp), // %1 "+r"(dst_ptr), // %2 "+r"(dst_width) // %3 : : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v31" // Clobber List ); } void ScaleRowUp2_Bilinear_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) { const uint8_t* src_ptr1 = src_ptr + src_stride; uint8_t* dst_ptr1 = dst_ptr + dst_stride; const uint8_t* src_temp = src_ptr + 1; const uint8_t* src_temp1 = src_ptr1 + 1; asm volatile( "movi v31.8b, #3 \n" "movi v30.8h, #3 \n" "1: \n" "ldr d0, [%0], #8 \n" // 01234567 "ldr d1, [%2], #8 \n" // 12345678 "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "ushll v2.8h, v0.8b, #0 \n" // 01234567 (16b) "ushll v3.8h, v1.8b, #0 \n" // 12345678 (16b) "umlal v2.8h, v1.8b, v31.8b \n" // 3*near+far (1, odd) "umlal v3.8h, v0.8b, v31.8b \n" // 3*near+far (1, even) "ldr d0, [%1], #8 \n" "ldr d1, [%3], #8 \n" "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead "ushll v4.8h, v0.8b, #0 \n" // 01234567 (16b) "ushll v5.8h, v1.8b, #0 \n" // 12345678 (16b) "umlal v4.8h, v1.8b, v31.8b \n" // 3*near+far (2, odd) "umlal v5.8h, v0.8b, v31.8b \n" // 3*near+far (2, even) "mov v0.16b, v4.16b \n" "mov v1.16b, v5.16b \n" "mla v4.8h, v2.8h, v30.8h \n" // 9 3 3 1 (1, odd) "mla v5.8h, v3.8h, v30.8h \n" // 9 3 3 1 (1, even) "mla v2.8h, v0.8h, v30.8h \n" // 9 3 3 1 (2, odd) "mla v3.8h, v1.8h, v30.8h \n" // 9 3 3 1 (2, even) "rshrn v2.8b, v2.8h, #4 \n" // 2, odd "rshrn v1.8b, v3.8h, #4 \n" // 2, even "rshrn v4.8b, v4.8h, #4 \n" // 1, odd "rshrn v3.8b, v5.8h, #4 \n" // 1, even "st2 {v1.8b, v2.8b}, [%5], #16 \n" // store 1 "st2 {v3.8b, v4.8b}, [%4], #16 \n" // store 2 "subs %w6, %w6, #16 \n" // 8 sample -> 16 sample "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_ptr1), // %1 "+r"(src_temp), // %2 "+r"(src_temp1), // %3 "+r"(dst_ptr), // %4 "+r"(dst_ptr1), // %5 "+r"(dst_width) // %6 : : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v30", "v31" // Clobber List ); } void ScaleRowUp2_Linear_12_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, int dst_width) { const uint16_t* src_temp = src_ptr + 1; asm volatile( "movi v31.8h, #3 \n" "1: \n" "ld1 {v0.8h}, [%0], #16 \n" // 01234567 (16b) "ld1 {v1.8h}, [%1], #16 \n" // 12345678 (16b) "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "mov v2.16b, v0.16b \n" "mla v0.8h, v1.8h, v31.8h \n" // 3*near+far (odd) "mla v1.8h, v2.8h, v31.8h \n" // 3*near+far (even) "urshr v2.8h, v0.8h, #2 \n" // 3/4*near+1/4*far (odd) "urshr v1.8h, v1.8h, #2 \n" // 3/4*near+1/4*far (even) "st2 {v1.8h, v2.8h}, [%2], #32 \n" // store "subs %w3, %w3, #16 \n" // 8 sample -> 16 sample "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_temp), // %1 "+r"(dst_ptr), // %2 "+r"(dst_width) // %3 : : "memory", "cc", "v0", "v1", "v2", "v31" // Clobber List ); } void ScaleRowUp2_Bilinear_12_NEON(const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) { const uint16_t* src_ptr1 = src_ptr + src_stride; uint16_t* dst_ptr1 = dst_ptr + dst_stride; const uint16_t* src_temp = src_ptr + 1; const uint16_t* src_temp1 = src_ptr1 + 1; asm volatile( "movi v31.8h, #3 \n" "1: \n" "ld1 {v2.8h}, [%0], #16 \n" // 01234567 (16b) "ld1 {v3.8h}, [%2], #16 \n" // 12345678 (16b) "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "mov v0.16b, v2.16b \n" "mla v2.8h, v3.8h, v31.8h \n" // 3*near+far (odd) "mla v3.8h, v0.8h, v31.8h \n" // 3*near+far (even) "ld1 {v4.8h}, [%1], #16 \n" // 01234567 (16b) "ld1 {v5.8h}, [%3], #16 \n" // 12345678 (16b) "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead "mov v0.16b, v4.16b \n" "mla v4.8h, v5.8h, v31.8h \n" // 3*near+far (odd) "mla v5.8h, v0.8h, v31.8h \n" // 3*near+far (even) "mov v0.16b, v4.16b \n" "mov v1.16b, v5.16b \n" "mla v4.8h, v2.8h, v31.8h \n" // 9 3 3 1 (1, odd) "mla v5.8h, v3.8h, v31.8h \n" // 9 3 3 1 (1, even) "mla v2.8h, v0.8h, v31.8h \n" // 9 3 3 1 (2, odd) "mla v3.8h, v1.8h, v31.8h \n" // 9 3 3 1 (2, even) "urshr v2.8h, v2.8h, #4 \n" // 2, odd "urshr v1.8h, v3.8h, #4 \n" // 2, even "urshr v4.8h, v4.8h, #4 \n" // 1, odd "urshr v3.8h, v5.8h, #4 \n" // 1, even "st2 {v3.8h, v4.8h}, [%4], #32 \n" // store 1 "st2 {v1.8h, v2.8h}, [%5], #32 \n" // store 2 "subs %w6, %w6, #16 \n" // 8 sample -> 16 sample "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_ptr1), // %1 "+r"(src_temp), // %2 "+r"(src_temp1), // %3 "+r"(dst_ptr), // %4 "+r"(dst_ptr1), // %5 "+r"(dst_width) // %6 : : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v31" // Clobber List ); } void ScaleRowUp2_Linear_16_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, int dst_width) { const uint16_t* src_temp = src_ptr + 1; asm volatile( "movi v31.8h, #3 \n" "1: \n" "ld1 {v0.8h}, [%0], #16 \n" // 01234567 (16b) "ld1 {v1.8h}, [%1], #16 \n" // 12345678 (16b) "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "ushll v2.4s, v0.4h, #0 \n" // 0123 (32b) "ushll2 v3.4s, v0.8h, #0 \n" // 4567 (32b) "ushll v4.4s, v1.4h, #0 \n" // 1234 (32b) "ushll2 v5.4s, v1.8h, #0 \n" // 5678 (32b) "umlal v2.4s, v1.4h, v31.4h \n" // 3*near+far (1, odd) "umlal2 v3.4s, v1.8h, v31.8h \n" // 3*near+far (2, odd) "umlal v4.4s, v0.4h, v31.4h \n" // 3*near+far (1, even) "umlal2 v5.4s, v0.8h, v31.8h \n" // 3*near+far (2, even) "rshrn v0.4h, v4.4s, #2 \n" // 3/4*near+1/4*far "rshrn2 v0.8h, v5.4s, #2 \n" // 3/4*near+1/4*far (even) "rshrn v1.4h, v2.4s, #2 \n" // 3/4*near+1/4*far "rshrn2 v1.8h, v3.4s, #2 \n" // 3/4*near+1/4*far (odd) "st2 {v0.8h, v1.8h}, [%2], #32 \n" // store "subs %w3, %w3, #16 \n" // 8 sample -> 16 sample "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_temp), // %1 "+r"(dst_ptr), // %2 "+r"(dst_width) // %3 : : "memory", "cc", "v0", "v1", "v2", "v31" // Clobber List ); } void ScaleRowUp2_Bilinear_16_NEON(const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) { const uint16_t* src_ptr1 = src_ptr + src_stride; uint16_t* dst_ptr1 = dst_ptr + dst_stride; const uint16_t* src_temp = src_ptr + 1; const uint16_t* src_temp1 = src_ptr1 + 1; asm volatile( "movi v31.4h, #3 \n" "movi v30.4s, #3 \n" "1: \n" "ldr d0, [%0], #8 \n" // 0123 (16b) "ldr d1, [%2], #8 \n" // 1234 (16b) "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "ushll v2.4s, v0.4h, #0 \n" // 0123 (32b) "ushll v3.4s, v1.4h, #0 \n" // 1234 (32b) "umlal v2.4s, v1.4h, v31.4h \n" // 3*near+far (1, odd) "umlal v3.4s, v0.4h, v31.4h \n" // 3*near+far (1, even) "ldr d0, [%1], #8 \n" // 0123 (16b) "ldr d1, [%3], #8 \n" // 1234 (16b) "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead "ushll v4.4s, v0.4h, #0 \n" // 0123 (32b) "ushll v5.4s, v1.4h, #0 \n" // 1234 (32b) "umlal v4.4s, v1.4h, v31.4h \n" // 3*near+far (2, odd) "umlal v5.4s, v0.4h, v31.4h \n" // 3*near+far (2, even) "mov v0.16b, v4.16b \n" "mov v1.16b, v5.16b \n" "mla v4.4s, v2.4s, v30.4s \n" // 9 3 3 1 (1, odd) "mla v5.4s, v3.4s, v30.4s \n" // 9 3 3 1 (1, even) "mla v2.4s, v0.4s, v30.4s \n" // 9 3 3 1 (2, odd) "mla v3.4s, v1.4s, v30.4s \n" // 9 3 3 1 (2, even) "rshrn v1.4h, v4.4s, #4 \n" // 3/4*near+1/4*far "rshrn v0.4h, v5.4s, #4 \n" // 3/4*near+1/4*far "rshrn v5.4h, v2.4s, #4 \n" // 3/4*near+1/4*far "rshrn v4.4h, v3.4s, #4 \n" // 3/4*near+1/4*far "st2 {v0.4h, v1.4h}, [%4], #16 \n" // store 1 "st2 {v4.4h, v5.4h}, [%5], #16 \n" // store 2 "subs %w6, %w6, #8 \n" // 4 sample -> 8 sample "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_ptr1), // %1 "+r"(src_temp), // %2 "+r"(src_temp1), // %3 "+r"(dst_ptr), // %4 "+r"(dst_ptr1), // %5 "+r"(dst_width) // %6 : : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v30", "v31" // Clobber List ); } void ScaleUVRowUp2_Linear_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width) { const uint8_t* src_temp = src_ptr + 2; asm volatile( "movi v31.8b, #3 \n" "1: \n" "ldr d0, [%0], #8 \n" // 00112233 (1u1v) "ldr d1, [%1], #8 \n" // 11223344 (1u1v) "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "ushll v2.8h, v0.8b, #0 \n" // 00112233 (1u1v, 16b) "ushll v3.8h, v1.8b, #0 \n" // 11223344 (1u1v, 16b) "umlal v2.8h, v1.8b, v31.8b \n" // 3*near+far (odd) "umlal v3.8h, v0.8b, v31.8b \n" // 3*near+far (even) "rshrn v2.8b, v2.8h, #2 \n" // 3/4*near+1/4*far (odd) "rshrn v1.8b, v3.8h, #2 \n" // 3/4*near+1/4*far (even) "st2 {v1.4h, v2.4h}, [%2], #16 \n" // store "subs %w3, %w3, #8 \n" // 4 uv -> 8 uv "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_temp), // %1 "+r"(dst_ptr), // %2 "+r"(dst_width) // %3 : : "memory", "cc", "v0", "v1", "v2", "v3", "v31" // Clobber List ); } void ScaleUVRowUp2_Bilinear_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) { const uint8_t* src_ptr1 = src_ptr + src_stride; uint8_t* dst_ptr1 = dst_ptr + dst_stride; const uint8_t* src_temp = src_ptr + 2; const uint8_t* src_temp1 = src_ptr1 + 2; asm volatile( "movi v31.8b, #3 \n" "movi v30.8h, #3 \n" "1: \n" "ldr d0, [%0], #8 \n" "ldr d1, [%2], #8 \n" "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "ushll v2.8h, v0.8b, #0 \n" "ushll v3.8h, v1.8b, #0 \n" "umlal v2.8h, v1.8b, v31.8b \n" // 3*near+far (1, odd) "umlal v3.8h, v0.8b, v31.8b \n" // 3*near+far (1, even) "ldr d0, [%1], #8 \n" "ldr d1, [%3], #8 \n" "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead "ushll v4.8h, v0.8b, #0 \n" "ushll v5.8h, v1.8b, #0 \n" "umlal v4.8h, v1.8b, v31.8b \n" // 3*near+far (2, odd) "umlal v5.8h, v0.8b, v31.8b \n" // 3*near+far (2, even) "mov v0.16b, v4.16b \n" "mov v1.16b, v5.16b \n" "mla v4.8h, v2.8h, v30.8h \n" // 9 3 3 1 (1, odd) "mla v5.8h, v3.8h, v30.8h \n" // 9 3 3 1 (1, even) "mla v2.8h, v0.8h, v30.8h \n" // 9 3 3 1 (2, odd) "mla v3.8h, v1.8h, v30.8h \n" // 9 3 3 1 (2, even) "rshrn v2.8b, v2.8h, #4 \n" // 2, odd "rshrn v1.8b, v3.8h, #4 \n" // 2, even "rshrn v4.8b, v4.8h, #4 \n" // 1, odd "rshrn v3.8b, v5.8h, #4 \n" // 1, even "st2 {v1.4h, v2.4h}, [%5], #16 \n" // store 2 "st2 {v3.4h, v4.4h}, [%4], #16 \n" // store 1 "subs %w6, %w6, #8 \n" // 4 uv -> 8 uv "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_ptr1), // %1 "+r"(src_temp), // %2 "+r"(src_temp1), // %3 "+r"(dst_ptr), // %4 "+r"(dst_ptr1), // %5 "+r"(dst_width) // %6 : : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v30", "v31" // Clobber List ); } void ScaleUVRowUp2_Linear_16_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, int dst_width) { const uint16_t* src_temp = src_ptr + 2; asm volatile( "movi v31.8h, #3 \n" "1: \n" "ld1 {v0.8h}, [%0], #16 \n" // 01234567 (16b) "ld1 {v1.8h}, [%1], #16 \n" // 12345678 (16b) "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "ushll v2.4s, v0.4h, #0 \n" // 0011 (1u1v, 32b) "ushll v3.4s, v1.4h, #0 \n" // 1122 (1u1v, 32b) "ushll2 v4.4s, v0.8h, #0 \n" // 2233 (1u1v, 32b) "ushll2 v5.4s, v1.8h, #0 \n" // 3344 (1u1v, 32b) "umlal v2.4s, v1.4h, v31.4h \n" // 3*near+far (odd) "umlal v3.4s, v0.4h, v31.4h \n" // 3*near+far (even) "umlal2 v4.4s, v1.8h, v31.8h \n" // 3*near+far (odd) "umlal2 v5.4s, v0.8h, v31.8h \n" // 3*near+far (even) "rshrn v2.4h, v2.4s, #2 \n" // 3/4*near+1/4*far (odd) "rshrn v1.4h, v3.4s, #2 \n" // 3/4*near+1/4*far (even) "rshrn v4.4h, v4.4s, #2 \n" // 3/4*near+1/4*far (odd) "rshrn v3.4h, v5.4s, #2 \n" // 3/4*near+1/4*far (even) "st2 {v1.2s, v2.2s}, [%2], #16 \n" // store "st2 {v3.2s, v4.2s}, [%2], #16 \n" // store "subs %w3, %w3, #8 \n" // 4 uv -> 8 uv "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_temp), // %1 "+r"(dst_ptr), // %2 "+r"(dst_width) // %3 : : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v31" // Clobber List ); } void ScaleUVRowUp2_Bilinear_16_NEON(const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) { const uint16_t* src_ptr1 = src_ptr + src_stride; uint16_t* dst_ptr1 = dst_ptr + dst_stride; const uint16_t* src_temp = src_ptr + 2; const uint16_t* src_temp1 = src_ptr1 + 2; asm volatile( "movi v31.4h, #3 \n" "movi v30.4s, #3 \n" "1: \n" "ldr d0, [%0], #8 \n" "ldr d1, [%2], #8 \n" "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "ushll v2.4s, v0.4h, #0 \n" // 0011 (1u1v, 32b) "ushll v3.4s, v1.4h, #0 \n" // 1122 (1u1v, 32b) "umlal v2.4s, v1.4h, v31.4h \n" // 3*near+far (1, odd) "umlal v3.4s, v0.4h, v31.4h \n" // 3*near+far (1, even) "ldr d0, [%1], #8 \n" "ldr d1, [%3], #8 \n" "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead "ushll v4.4s, v0.4h, #0 \n" // 0011 (1u1v, 32b) "ushll v5.4s, v1.4h, #0 \n" // 1122 (1u1v, 32b) "umlal v4.4s, v1.4h, v31.4h \n" // 3*near+far (2, odd) "umlal v5.4s, v0.4h, v31.4h \n" // 3*near+far (2, even) "mov v0.16b, v4.16b \n" "mov v1.16b, v5.16b \n" "mla v4.4s, v2.4s, v30.4s \n" // 9 3 3 1 (1, odd) "mla v5.4s, v3.4s, v30.4s \n" // 9 3 3 1 (1, even) "mla v2.4s, v0.4s, v30.4s \n" // 9 3 3 1 (2, odd) "mla v3.4s, v1.4s, v30.4s \n" // 9 3 3 1 (2, even) "rshrn v1.4h, v2.4s, #4 \n" // 2, odd "rshrn v0.4h, v3.4s, #4 \n" // 2, even "rshrn v3.4h, v4.4s, #4 \n" // 1, odd "rshrn v2.4h, v5.4s, #4 \n" // 1, even "st2 {v0.2s, v1.2s}, [%5], #16 \n" // store 2 "st2 {v2.2s, v3.2s}, [%4], #16 \n" // store 1 "subs %w6, %w6, #4 \n" // 2 uv -> 4 uv "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_ptr1), // %1 "+r"(src_temp), // %2 "+r"(src_temp1), // %3 "+r"(dst_ptr), // %4 "+r"(dst_ptr1), // %5 "+r"(dst_width) // %6 : : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v30", "v31" // Clobber List ); } // Add a row of bytes to a row of shorts. Used for box filter. // Reads 16 bytes and accumulates to 16 shorts at a time. void ScaleAddRow_NEON(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) { asm volatile( "1: \n" "ld1 {v1.8h, v2.8h}, [%1] \n" // load accumulator "ld1 {v0.16b}, [%0], #16 \n" // load 16 bytes "uaddw2 v2.8h, v2.8h, v0.16b \n" // add "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "uaddw v1.8h, v1.8h, v0.8b \n" "st1 {v1.8h, v2.8h}, [%1], #32 \n" // store accumulator "subs %w2, %w2, #16 \n" // 16 processed per loop "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(dst_ptr), // %1 "+r"(src_width) // %2 : : "memory", "cc", "v0", "v1", "v2" // Clobber List ); } #define SCALE_FILTER_COLS_STEP_ADDR \ "lsr %[tmp_offset], %x[x], #16 \n" \ "add %[tmp_ptr], %[src_ptr], %[tmp_offset] \n" \ "add %x[x], %x[x], %x[dx] \n" // The Neon version mimics this formula (from scale_common.cc): // #define BLENDER(a, b, f) (uint8_t)((int)(a) + // ((((int)((f)) * ((int)(b) - (int)(a))) + 0x8000) >> 16)) void ScaleFilterCols_NEON(uint8_t* dst_ptr, const uint8_t* src_ptr, int dst_width, int x, int dx) { int dx_offset[4] = {0, 1, 2, 3}; int64_t tmp_offset; uint8_t* tmp_ptr; asm volatile( "dup v0.4s, %w[x] \n" "dup v1.4s, %w[dx] \n" "ld1 {v2.4s}, [%[dx_offset]] \n" // 0 1 2 3 "shl v3.4s, v1.4s, #2 \n" // 4 * dx "shl v22.4s, v1.4s, #3 \n" // 8 * dx "mul v1.4s, v1.4s, v2.4s \n" // x , x + 1 * dx, x + 2 * dx, x + 3 * dx "add v1.4s, v1.4s, v0.4s \n" // x + 4 * dx, x + 5 * dx, x + 6 * dx, x + 7 * dx "add v2.4s, v1.4s, v3.4s \n" "movi v0.8h, #0 \n" // truncate to uint16_t "trn1 v22.8h, v22.8h, v0.8h \n" "trn1 v20.8h, v1.8h, v0.8h \n" "trn1 v21.8h, v2.8h, v0.8h \n" "1: \n" SCALE_FILTER_COLS_STEP_ADDR "ldr h6, [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR "ld1 {v6.h}[1], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR "ld1 {v6.h}[2], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR "ld1 {v6.h}[3], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR "ld1 {v6.h}[4], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR "ld1 {v6.h}[5], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR "ld1 {v6.h}[6], [%[tmp_ptr]] \n" SCALE_FILTER_COLS_STEP_ADDR "ld1 {v6.h}[7], [%[tmp_ptr]] \n" "subs %w[width], %w[width], #8 \n" // 8 processed per loop "trn1 v4.16b, v6.16b, v0.16b \n" "trn2 v5.16b, v6.16b, v0.16b \n" "ssubl v16.4s, v5.4h, v4.4h \n" "ssubl2 v17.4s, v5.8h, v4.8h \n" "mul v16.4s, v16.4s, v20.4s \n" "mul v17.4s, v17.4s, v21.4s \n" "rshrn v6.4h, v16.4s, #16 \n" "rshrn2 v6.8h, v17.4s, #16 \n" "add v4.8h, v4.8h, v6.8h \n" "xtn v4.8b, v4.8h \n" "add v20.8h, v20.8h, v22.8h \n" "add v21.8h, v21.8h, v22.8h \n" "st1 {v4.8b}, [%[dst_ptr]], #8 \n" // store pixels "b.gt 1b \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] [width] "+r"(dst_width), // %[width] [x] "+r"(x), // %[x] [dx] "+r"(dx), // %[dx] [tmp_offset] "=&r"(tmp_offset), // %[tmp_offset] [tmp_ptr] "=&r"(tmp_ptr) // %[tmp_ptr] : [dx_offset] "r"(dx_offset) // %[dx_offset] : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v16", "v17", "v20", "v21", "v22"); } #undef SCALE_FILTER_COLS_STEP_ADDR void ScaleARGBRowDown2_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { (void)src_stride; asm volatile( "1: \n" "ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [%[src]], #64 \n" "subs %w[width], %w[width], #8 \n" "prfm pldl1keep, [%[src], 448] \n" "uzp2 v0.4s, v0.4s, v1.4s \n" "uzp2 v1.4s, v2.4s, v3.4s \n" "st1 {v0.4s, v1.4s}, [%[dst]], #32 \n" "b.gt 1b \n" : [src] "+r"(src_ptr), // %[src] [dst] "+r"(dst), // %[dst] [width] "+r"(dst_width) // %[width] : : "memory", "cc", "v0", "v1", "v2", "v3"); } void ScaleARGBRowDown2Linear_NEON(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { (void)src_stride; const uint8_t* src_argb1 = src_argb + 32; asm volatile( "1: \n" "ld2 {v0.4s, v1.4s}, [%[src]] \n" "add %[src], %[src], #64 \n" "ld2 {v2.4s, v3.4s}, [%[src1]] \n" "add %[src1], %[src1], #64 \n" "urhadd v0.16b, v0.16b, v1.16b \n" "urhadd v1.16b, v2.16b, v3.16b \n" "subs %w[width], %w[width], #8 \n" "st1 {v0.16b, v1.16b}, [%[dst]], #32 \n" "b.gt 1b \n" : [src] "+r"(src_argb), // %[src] [src1] "+r"(src_argb1), // %[src1] [dst] "+r"(dst_argb), // %[dst] [width] "+r"(dst_width) // %[width] : : "memory", "cc", "v0", "v1", "v2", "v3"); } void ScaleARGBRowDown2Box_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { const uint8_t* src_ptr1 = src_ptr + src_stride; asm volatile( "1: \n" "ld2 {v0.4s, v1.4s}, [%[src]], #32 \n" "ld2 {v20.4s, v21.4s}, [%[src1]], #32 \n" "uaddl v2.8h, v0.8b, v1.8b \n" "uaddl2 v3.8h, v0.16b, v1.16b \n" "uaddl v22.8h, v20.8b, v21.8b \n" "uaddl2 v23.8h, v20.16b, v21.16b \n" "add v0.8h, v2.8h, v22.8h \n" "add v1.8h, v3.8h, v23.8h \n" "rshrn v0.8b, v0.8h, #2 \n" "rshrn v1.8b, v1.8h, #2 \n" "subs %w[width], %w[width], #4 \n" "stp d0, d1, [%[dst]], #16 \n" "b.gt 1b \n" : [src] "+r"(src_ptr), [src1] "+r"(src_ptr1), [dst] "+r"(dst), [width] "+r"(dst_width) : : "memory", "cc", "v0", "v1", "v2", "v3", "v20", "v21", "v22", "v23"); } void ScaleARGBRowDownEven_NEON(const uint8_t* src_argb, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_argb, int dst_width) { const uint8_t* src_argb1 = src_argb + src_stepx * 4; const uint8_t* src_argb2 = src_argb + src_stepx * 8; const uint8_t* src_argb3 = src_argb + src_stepx * 12; int64_t i = 0; (void)src_stride; asm volatile( "1: \n" "ldr w10, [%[src], %[i]] \n" "ldr w11, [%[src1], %[i]] \n" "ldr w12, [%[src2], %[i]] \n" "ldr w13, [%[src3], %[i]] \n" "add %[i], %[i], %[step] \n" "subs %w[width], %w[width], #4 \n" "prfm pldl1keep, [%[src], 448] \n" "stp w10, w11, [%[dst]], #8 \n" "stp w12, w13, [%[dst]], #8 \n" "b.gt 1b \n" : [src] "+r"(src_argb), [src1] "+r"(src_argb1), [src2] "+r"(src_argb2), [src3] "+r"(src_argb3), [dst] "+r"(dst_argb), [width] "+r"(dst_width), [i] "+r"(i) : [step] "r"((int64_t)(src_stepx * 16)) : "memory", "cc", "w10", "w11", "w12", "w13"); } // Reads 4 pixels at a time. // Alignment requirement: src_argb 4 byte aligned. // TODO(Yang Zhang): Might be worth another optimization pass in future. // It could be upgraded to 8 pixels at a time to start with. void ScaleARGBRowDownEvenBox_NEON(const uint8_t* src_argb, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_argb, int dst_width) { asm volatile( "add %1, %1, %0 \n" "1: \n" "ld1 {v0.8b}, [%0], %4 \n" // Read 4 2x2 -> 2x1 "ld1 {v1.8b}, [%1], %4 \n" "ld1 {v2.8b}, [%0], %4 \n" "ld1 {v3.8b}, [%1], %4 \n" "ld1 {v4.8b}, [%0], %4 \n" "ld1 {v5.8b}, [%1], %4 \n" "ld1 {v6.8b}, [%0], %4 \n" "ld1 {v7.8b}, [%1], %4 \n" "uaddl v0.8h, v0.8b, v1.8b \n" "uaddl v2.8h, v2.8b, v3.8b \n" "uaddl v4.8h, v4.8b, v5.8b \n" "uaddl v6.8h, v6.8b, v7.8b \n" "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "zip1 v1.2d, v0.2d, v2.2d \n" "zip2 v2.2d, v0.2d, v2.2d \n" "zip1 v5.2d, v4.2d, v6.2d \n" "zip2 v6.2d, v4.2d, v6.2d \n" "prfm pldl1keep, [%1, 448] \n" "add v0.8h, v1.8h, v2.8h \n" // (a+b)_(c+d) "add v4.8h, v5.8h, v6.8h \n" // (e+f)_(g+h) "rshrn v0.8b, v0.8h, #2 \n" // first 2 pixels. "rshrn v1.8b, v4.8h, #2 \n" // next 2 pixels. "subs %w3, %w3, #4 \n" // 4 pixels per loop. "stp d0, d1, [%2], #16 \n" "b.gt 1b \n" : "+r"(src_argb), // %0 "+r"(src_stride), // %1 "+r"(dst_argb), // %2 "+r"(dst_width) // %3 : "r"((int64_t)(src_stepx * 4)) // %4 : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); } // TODO(Yang Zhang): Investigate less load instructions for // the x/dx stepping #define LOAD1_DATA32_LANE(vn, n) \ "lsr %5, %3, #16 \n" \ "add %6, %1, %5, lsl #2 \n" \ "add %3, %3, %4 \n" \ "ld1 {" #vn ".s}[" #n "], [%6] \n" void ScaleARGBCols_NEON(uint8_t* dst_argb, const uint8_t* src_argb, int dst_width, int x, int dx) { const uint8_t* src_tmp = src_argb; int64_t x64 = (int64_t)x; // NOLINT int64_t dx64 = (int64_t)dx; // NOLINT int64_t tmp64; asm volatile ( "1: \n" // clang-format off LOAD1_DATA32_LANE(v0, 0) LOAD1_DATA32_LANE(v0, 1) LOAD1_DATA32_LANE(v0, 2) LOAD1_DATA32_LANE(v0, 3) LOAD1_DATA32_LANE(v1, 0) LOAD1_DATA32_LANE(v1, 1) LOAD1_DATA32_LANE(v1, 2) LOAD1_DATA32_LANE(v1, 3) "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead // clang-format on "st1 {v0.4s, v1.4s}, [%0], #32 \n" // store pixels "subs %w2, %w2, #8 \n" // 8 processed per loop "b.gt 1b \n" : "+r"(dst_argb), // %0 "+r"(src_argb), // %1 "+r"(dst_width), // %2 "+r"(x64), // %3 "+r"(dx64), // %4 "=&r"(tmp64), // %5 "+r"(src_tmp) // %6 : : "memory", "cc", "v0", "v1"); } #undef LOAD1_DATA32_LANE static const uvec8 kScaleARGBFilterColsShuffleIndices = { 0, 0, 0, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6, }; #define SCALE_ARGB_FILTER_COLS_STEP_ADDR \ "lsr %5, %3, #16 \n" \ "add %6, %1, %5, lsl #2 \n" \ "add %3, %3, %4 \n" void ScaleARGBFilterCols_NEON(uint8_t* dst_argb, const uint8_t* src_argb, int dst_width, int x, int dx) { int dx_offset[4] = {0, 1, 2, 3}; int64_t tmp; const uint8_t* src_tmp = src_argb; int64_t x64 = (int64_t)x; int64_t dx64 = (int64_t)dx; asm volatile( "dup v0.4s, %w3 \n" "dup v1.4s, %w4 \n" "ld1 {v2.4s}, [%[kOffsets]] \n" "shl v6.4s, v1.4s, #2 \n" "mul v1.4s, v1.4s, v2.4s \n" "movi v3.16b, #0x7f \n" "add v5.4s, v1.4s, v0.4s \n" "ldr q18, [%[kIndices]] \n" "1: \n" // SCALE_ARGB_FILTER_COLS_STEP_ADDR "ldr d1, [%6] \n" // SCALE_ARGB_FILTER_COLS_STEP_ADDR "ldr d2, [%6] \n" "shrn v4.4h, v5.4s, #9 \n" // SCALE_ARGB_FILTER_COLS_STEP_ADDR "ld1 {v1.d}[1], [%6] \n" // SCALE_ARGB_FILTER_COLS_STEP_ADDR "ld1 {v2.d}[1], [%6] \n" "subs %w2, %w2, #4 \n" // 4 processed per loop "and v4.8b, v4.8b, v3.8b \n" "trn1 v0.4s, v1.4s, v2.4s \n" "tbl v4.16b, {v4.16b}, v18.16b \n" // f "trn2 v1.4s, v1.4s, v2.4s \n" "eor v7.16b, v4.16b, v3.16b \n" // 0x7f ^ f "umull v16.8h, v1.8b, v4.8b \n" "umull2 v17.8h, v1.16b, v4.16b \n" "umlal v16.8h, v0.8b, v7.8b \n" "umlal2 v17.8h, v0.16b, v7.16b \n" "prfm pldl1keep, [%1, 448] \n" // prefetch 7 lines ahead "shrn v0.8b, v16.8h, #7 \n" "shrn v1.8b, v17.8h, #7 \n" "add v5.4s, v5.4s, v6.4s \n" "stp d0, d1, [%0], #16 \n" // store pixels "b.gt 1b \n" : "+r"(dst_argb), // %0 "+r"(src_argb), // %1 "+r"(dst_width), // %2 "+r"(x64), // %3 "+r"(dx64), // %4 "=&r"(tmp), // %5 "+r"(src_tmp) // %6 : [kIndices] "r"(&kScaleARGBFilterColsShuffleIndices), // %[kIndices] [kOffsets] "r"(dx_offset) // %[kOffsets] : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16", "v17", "v18", "v19"); } #undef SCALE_ARGB_FILTER_COLS_STEP_ADDR void ScaleRowDown2_16_NEON(const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst, int dst_width) { (void)src_stride; asm volatile( "subs %w[dst_width], %w[dst_width], #32 \n" "b.lt 2f \n" "1: \n" "ldp q0, q1, [%[src_ptr]] \n" "ldp q2, q3, [%[src_ptr], #32] \n" "ldp q4, q5, [%[src_ptr], #64] \n" "ldp q6, q7, [%[src_ptr], #96] \n" "add %[src_ptr], %[src_ptr], #128 \n" "uzp2 v0.8h, v0.8h, v1.8h \n" "uzp2 v1.8h, v2.8h, v3.8h \n" "uzp2 v2.8h, v4.8h, v5.8h \n" "uzp2 v3.8h, v6.8h, v7.8h \n" "subs %w[dst_width], %w[dst_width], #32 \n" // 32 elems per // iteration. "stp q0, q1, [%[dst_ptr]] \n" "stp q2, q3, [%[dst_ptr], #32] \n" "add %[dst_ptr], %[dst_ptr], #64 \n" "b.ge 1b \n" "2: \n" "adds %w[dst_width], %w[dst_width], #32 \n" "b.eq 99f \n" "ldp q0, q1, [%[src_ptr]] \n" "ldp q2, q3, [%[src_ptr], #32] \n" "uzp2 v0.8h, v0.8h, v1.8h \n" "uzp2 v1.8h, v2.8h, v3.8h \n" "stp q0, q1, [%[dst_ptr]] \n" "99: \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [dst_ptr] "+r"(dst), // %[dst_ptr] [dst_width] "+r"(dst_width) // %[dst_width] : : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); } void ScaleRowDown2Linear_16_NEON(const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst, int dst_width) { (void)src_stride; asm volatile( "1: \n" "ld2 {v0.8h, v1.8h}, [%[src_ptr]], #32 \n" "ld2 {v2.8h, v3.8h}, [%[src_ptr]], #32 \n" "subs %w[dst_width], %w[dst_width], #16 \n" "urhadd v0.8h, v0.8h, v1.8h \n" "urhadd v1.8h, v2.8h, v3.8h \n" "prfm pldl1keep, [%[src_ptr], 448] \n" "stp q0, q1, [%[dst_ptr]], #32 \n" "b.gt 1b \n" : [src_ptr] "+r"(src_ptr), // %[src_ptr] [dst_ptr] "+r"(dst), // %[dst_ptr] [dst_width] "+r"(dst_width) // %[dst_width] : : "memory", "cc", "v0", "v1", "v2", "v3"); } // Read 16x2 average down and write 8x1. void ScaleRowDown2Box_16_NEON(const uint16_t* src_ptr, ptrdiff_t src_stride, uint16_t* dst, int dst_width) { asm volatile( // change the stride to row 2 pointer "add %1, %0, %1, lsl #1 \n" // ptr + stide * 2 "1: \n" "ld1 {v0.8h, v1.8h}, [%0], #32 \n" // load row 1 and post inc "ld1 {v2.8h, v3.8h}, [%1], #32 \n" // load row 2 and post inc "subs %w3, %w3, #8 \n" // 8 processed per loop "uaddlp v0.4s, v0.8h \n" // row 1 add adjacent "uaddlp v1.4s, v1.8h \n" "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "uadalp v0.4s, v2.8h \n" // +row 2 add adjacent "uadalp v1.4s, v3.8h \n" "prfm pldl1keep, [%1, 448] \n" "rshrn v0.4h, v0.4s, #2 \n" // round and pack "rshrn2 v0.8h, v1.4s, #2 \n" "st1 {v0.8h}, [%2], #16 \n" "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_stride), // %1 "+r"(dst), // %2 "+r"(dst_width) // %3 : : "memory", "cc", "v0", "v1", "v2", "v3" // Clobber List ); } void ScaleUVRowDown2_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { (void)src_stride; asm volatile( "1: \n" "ld2 {v0.8h,v1.8h}, [%0], #32 \n" // load 16 UV "subs %w2, %w2, #8 \n" // 8 processed per loop. "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "st1 {v1.8h}, [%1], #16 \n" // store 8 UV "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(dst), // %1 "+r"(dst_width) // %2 : : "memory", "cc", "v0", "v1"); } void ScaleUVRowDown2Linear_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { (void)src_stride; asm volatile( "1: \n" "ld2 {v0.8h,v1.8h}, [%0], #32 \n" // load 16 UV "subs %w2, %w2, #8 \n" // 8 processed per loop. "urhadd v0.16b, v0.16b, v1.16b \n" // rounding half add "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "st1 {v0.8h}, [%1], #16 \n" // store 8 UV "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(dst), // %1 "+r"(dst_width) // %2 : : "memory", "cc", "v0", "v1"); } void ScaleUVRowDown2Box_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { asm volatile( // change the stride to row 2 pointer "add %1, %1, %0 \n" "1: \n" "ld2 {v0.16b,v1.16b}, [%0], #32 \n" // load 16 UV "subs %w3, %w3, #8 \n" // 8 processed per loop. "uaddlp v0.8h, v0.16b \n" // U 16 bytes -> 8 shorts. "uaddlp v1.8h, v1.16b \n" // V 16 bytes -> 8 shorts. "ld2 {v16.16b,v17.16b}, [%1], #32 \n" // load 16 "uadalp v0.8h, v16.16b \n" // U 16 bytes -> 8 shorts. "uadalp v1.8h, v17.16b \n" // V 16 bytes -> 8 shorts. "prfm pldl1keep, [%0, 448] \n" // prefetch 7 lines ahead "rshrn v0.8b, v0.8h, #2 \n" // round and pack "prfm pldl1keep, [%1, 448] \n" "rshrn v1.8b, v1.8h, #2 \n" "st2 {v0.8b,v1.8b}, [%2], #16 \n" "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src_stride), // %1 "+r"(dst), // %2 "+r"(dst_width) // %3 : : "memory", "cc", "v0", "v1", "v16", "v17"); } // Reads 4 pixels at a time. void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, int src_stepx, // pixel step uint8_t* dst_ptr, int dst_width) { const uint8_t* src1_ptr = src_ptr + src_stepx * 2; const uint8_t* src2_ptr = src_ptr + src_stepx * 4; const uint8_t* src3_ptr = src_ptr + src_stepx * 6; (void)src_stride; asm volatile( "1: \n" "ld1 {v0.h}[0], [%0], %6 \n" "ld1 {v1.h}[0], [%1], %6 \n" "ld1 {v2.h}[0], [%2], %6 \n" "ld1 {v3.h}[0], [%3], %6 \n" "subs %w5, %w5, #4 \n" // 4 pixels per loop. "st4 {v0.h, v1.h, v2.h, v3.h}[0], [%4], #8 \n" "b.gt 1b \n" : "+r"(src_ptr), // %0 "+r"(src1_ptr), // %1 "+r"(src2_ptr), // %2 "+r"(src3_ptr), // %3 "+r"(dst_ptr), // %4 "+r"(dst_width) // %5 : "r"((int64_t)(src_stepx * 8)) // %6 : "memory", "cc", "v0", "v1", "v2", "v3"); } #endif // !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) #ifdef __cplusplus } // extern "C" } // namespace libyuv #endif