/* * Copyright (c) 2024, Alliance for Open Media. All rights reserved. * * This source code is subject to the terms of the BSD 2 Clause License and * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License * was not distributed with this source code in the LICENSE file, you can * obtain it at www.aomedia.org/license/software. If the Alliance for Open * Media Patent License 1.0 was not distributed with this source code in the * PATENTS file, you can obtain it at www.aomedia.org/license/patent. */ #ifndef AOM_AV1_COMMON_ARM_CONVOLVE_NEON_I8MM_H_ #define AOM_AV1_COMMON_ARM_CONVOLVE_NEON_I8MM_H_ #include #include #include "config/aom_config.h" #include "config/av1_rtcd.h" #include "aom/aom_integer.h" #include "aom_dsp/aom_dsp_common.h" #include "aom_dsp/arm/mem_neon.h" #include "aom_ports/mem.h" DECLARE_ALIGNED(16, extern const uint8_t, kMatMul6PermuteTbl[32]); DECLARE_ALIGNED(16, extern const uint8_t, kMatMul8PermuteTbl[32]); DECLARE_ALIGNED(16, extern const uint8_t, kFilterPermuteTbl[16]); static inline int16x4_t convolve12_4_2d_h(uint8x16_t samples[2], const int8x16_t filter[2], const uint8x16_t permute_tbl, int32x4_t horiz_const) { // Permute samples ready for matrix multiply. // { 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9 } // { 4, 5, 6, 7, 8, 9, 10, 11, 6, 7, 8, 9, 10, 11, 12, 13 } uint8x16_t perm_samples[2] = { vqtbl1q_u8(samples[0], permute_tbl), vqtbl1q_u8(samples[1], permute_tbl) }; // These instructions multiply a 2x8 matrix (samples) by an 8x2 matrix // (filter), destructively accumulating into the destination register. int32x4_t sum = vusmmlaq_s32(horiz_const, perm_samples[0], filter[0]); sum = vusmmlaq_s32(sum, perm_samples[1], filter[1]); // Narrow and re-pack. return vshrn_n_s32(sum, ROUND0_BITS); } static inline int16x8_t convolve12_8_2d_h(uint8x16_t samples[2], const int8x16_t filter[2], const uint8x16x2_t permute_tbl, const int32x4_t horiz_const) { /// Permute samples ready for matrix multiply. // { 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9 } // { 4, 5, 6, 7, 8, 9, 10, 11, 6, 7, 8, 9, 10, 11, 12, 13 } // { 6, 7, 8, 9, 10, 11, 12, 13, 8, 9, 10, 11, 12, 13, 14, 15 } // { 10, 11, 12, 13, 14, 15, 16, 17, 12, 13, 14, 15, 16, 17, 18, 19 } uint8x16_t perm_samples[4] = { vqtbl1q_u8(samples[0], permute_tbl.val[0]), vqtbl1q_u8(samples[0], permute_tbl.val[1]), vqtbl1q_u8(samples[1], permute_tbl.val[0]), vqtbl1q_u8(samples[1], permute_tbl.val[1]) }; // These instructions multiply a 2x8 matrix (samples) by an 8x2 matrix // (filter), destructively accumulating into the destination register. int32x4_t sum0123 = vusmmlaq_s32(horiz_const, perm_samples[0], filter[0]); int32x4_t sum4567 = vusmmlaq_s32(horiz_const, perm_samples[1], filter[0]); sum0123 = vusmmlaq_s32(sum0123, perm_samples[2], filter[1]); sum4567 = vusmmlaq_s32(sum4567, perm_samples[3], filter[1]); // Narrow and re-pack. return vcombine_s16(vshrn_n_s32(sum0123, ROUND0_BITS), vshrn_n_s32(sum4567, ROUND0_BITS)); } static inline void convolve_2d_sr_horiz_12tap_neon_i8mm( const uint8_t *src_ptr, int src_stride, int16_t *dst_ptr, const int dst_stride, int w, int h, const int16_t *x_filter_ptr) { // The no-op filter should never be used here. assert(x_filter_ptr[5] != 128); // Split 12-tap filter into two 6-tap filters, masking the top two elements. // { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0 } const int8x8_t mask = vcreate_s8(0x0000ffffffffffff); const int8x8_t filter_0 = vand_s8(vmovn_s16(vld1q_s16(x_filter_ptr)), mask); const int8x8_t filter_1 = vext_s8(vmovn_s16(vld1q_s16(x_filter_ptr + 4)), vdup_n_s8(0), 2); // Stagger each 6-tap filter to enable use of matrix multiply instructions. // { f0, f1, f2, f3, f4, f5, 0, 0, 0, f0, f1, f2, f3, f4, f5, 0 } const int8x16_t filter[2] = { vcombine_s8(filter_0, vext_s8(filter_0, filter_0, 7)), vcombine_s8(filter_1, vext_s8(filter_1, filter_1, 7)) }; // This shim of 1 << (ROUND0_BITS - 1) enables us to use non-rounding shifts // in convolution kernels - which are generally faster than rounding shifts on // modern CPUs. const int32x4_t horiz_const = vdupq_n_s32(1 << (ROUND0_BITS - 1)); if (w <= 4) { const uint8x16_t permute_tbl = vld1q_u8(kMatMul6PermuteTbl); do { uint8x16_t s0[2], s1[2], s2[2], s3[2]; load_u8_16x4(src_ptr, src_stride, &s0[0], &s1[0], &s2[0], &s3[0]); load_u8_16x4(src_ptr + 6, src_stride, &s0[1], &s1[1], &s2[1], &s3[1]); int16x4_t d0 = convolve12_4_2d_h(s0, filter, permute_tbl, horiz_const); int16x4_t d1 = convolve12_4_2d_h(s1, filter, permute_tbl, horiz_const); int16x4_t d2 = convolve12_4_2d_h(s2, filter, permute_tbl, horiz_const); int16x4_t d3 = convolve12_4_2d_h(s3, filter, permute_tbl, horiz_const); store_s16_4x4(dst_ptr, dst_stride, d0, d1, d2, d3); src_ptr += 4 * src_stride; dst_ptr += 4 * dst_stride; h -= 4; } while (h > 4); do { uint8x16_t s0[2]; s0[0] = vld1q_u8(src_ptr); s0[1] = vld1q_u8(src_ptr + 6); int16x4_t d0 = convolve12_4_2d_h(s0, filter, permute_tbl, horiz_const); vst1_s16(dst_ptr, d0); src_ptr += src_stride; dst_ptr += dst_stride; } while (--h != 0); } else { const uint8x16x2_t permute_tbl = vld1q_u8_x2(kMatMul6PermuteTbl); do { const uint8_t *s = src_ptr; int16_t *d = dst_ptr; int width = w; do { uint8x16_t s0[2], s1[2], s2[2], s3[2]; load_u8_16x4(s, src_stride, &s0[0], &s1[0], &s2[0], &s3[0]); load_u8_16x4(s + 6, src_stride, &s0[1], &s1[1], &s2[1], &s3[1]); int16x8_t d0 = convolve12_8_2d_h(s0, filter, permute_tbl, horiz_const); int16x8_t d1 = convolve12_8_2d_h(s1, filter, permute_tbl, horiz_const); int16x8_t d2 = convolve12_8_2d_h(s2, filter, permute_tbl, horiz_const); int16x8_t d3 = convolve12_8_2d_h(s3, filter, permute_tbl, horiz_const); store_s16_8x4(d, dst_stride, d0, d1, d2, d3); s += 8; d += 8; width -= 8; } while (width != 0); src_ptr += 4 * src_stride; dst_ptr += 4 * dst_stride; h -= 4; } while (h > 4); do { const uint8_t *s = src_ptr; int16_t *d = dst_ptr; int width = w; do { uint8x16_t s0[2]; s0[0] = vld1q_u8(s); s0[1] = vld1q_u8(s + 6); int16x8_t d0 = convolve12_8_2d_h(s0, filter, permute_tbl, horiz_const); vst1q_s16(d, d0); s += 8; d += 8; width -= 8; } while (width != 0); src_ptr += src_stride; dst_ptr += dst_stride; } while (--h != 0); } } #endif // AOM_AV1_COMMON_ARM_CONVOLVE_NEON_I8MM_H_