xref: /third_party/ffmpeg/libavcodec/loongarch/vp9_intra_lsx.c

/*
 * Copyright (c) 2021 Loongson Technology Corporation Limited
 * Contributed by Hao Chen <chenhao@loongson.cn>
 *
 * This file is part of FFmpeg.
 *
 * FFmpeg is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * FFmpeg is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include "libavcodec/vp9dsp.h"
#include "libavutil/loongarch/loongson_intrinsics.h"
#include "vp9dsp_loongarch.h"

#define LSX_ST_8(_dst0, _dst1, _dst2, _dst3, _dst4,   \
                 _dst5, _dst6, _dst7, _dst, _stride,  \
                 _stride2, _stride3, _stride4)        \
{                                                     \
    __lsx_vst(_dst0, _dst, 0);                        \
    __lsx_vstx(_dst1, _dst, _stride);                 \
    __lsx_vstx(_dst2, _dst, _stride2);                \
    __lsx_vstx(_dst3, _dst, _stride3);                \
    _dst += _stride4;                                 \
    __lsx_vst(_dst4, _dst, 0);                        \
    __lsx_vstx(_dst5, _dst, _stride);                 \
    __lsx_vstx(_dst6, _dst, _stride2);                \
    __lsx_vstx(_dst7, _dst, _stride3);                \
}

#define LSX_ST_8X16(_dst0, _dst1, _dst2, _dst3, _dst4,   \
                    _dst5, _dst6, _dst7, _dst, _stride)  \
{                                                        \
    __lsx_vst(_dst0, _dst, 0);                           \
    __lsx_vst(_dst0, _dst, 16);                          \
    _dst += _stride;                                     \
    __lsx_vst(_dst1, _dst, 0);                           \
    __lsx_vst(_dst1, _dst, 16);                          \
    _dst += _stride;                                     \
    __lsx_vst(_dst2, _dst, 0);                           \
    __lsx_vst(_dst2, _dst, 16);                          \
    _dst += _stride;                                     \
    __lsx_vst(_dst3, _dst, 0);                           \
    __lsx_vst(_dst3, _dst, 16);                          \
    _dst += _stride;                                     \
    __lsx_vst(_dst4, _dst, 0);                           \
    __lsx_vst(_dst4, _dst, 16);                          \
    _dst += _stride;                                     \
    __lsx_vst(_dst5, _dst, 0);                           \
    __lsx_vst(_dst5, _dst, 16);                          \
    _dst += _stride;                                     \
    __lsx_vst(_dst6, _dst, 0);                           \
    __lsx_vst(_dst6, _dst, 16);                          \
    _dst += _stride;                                     \
    __lsx_vst(_dst7, _dst, 0);                           \
    __lsx_vst(_dst7, _dst, 16);                          \
    _dst += _stride;                                     \
}

void ff_vert_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *left,
                       const uint8_t *src)
{
    __m128i src0;
    ptrdiff_t stride2 = dst_stride << 1;
    ptrdiff_t stride3 = stride2 + dst_stride;
    ptrdiff_t stride4 = stride2 << 1;
    src0 = __lsx_vld(src, 0);
    LSX_ST_8(src0, src0, src0, src0, src0, src0, src0, src0, dst,
             dst_stride, stride2, stride3, stride4);
    dst += stride4;
    LSX_ST_8(src0, src0, src0, src0, src0, src0, src0, src0, dst,
             dst_stride, stride2, stride3, stride4);
}

void ff_vert_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *left,
                       const uint8_t *src)
{
    uint32_t row;
    __m128i src0, src1;

    DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
    for (row = 32; row--;) {
        __lsx_vst(src0, dst, 0);
        __lsx_vst(src1, dst, 16);
        dst += dst_stride;
    }
}

void ff_hor_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src,
                      const uint8_t *top)
{
    __m128i src0, src1, src2, src3, src4, src5, src6, src7;
    __m128i src8, src9, src10, src11, src12, src13, src14, src15;
    ptrdiff_t stride2 = dst_stride << 1;
    ptrdiff_t stride3 = stride2 + dst_stride;
    ptrdiff_t stride4 = stride2 << 1;

    src15 = __lsx_vldrepl_b(src, 0);
    src14 = __lsx_vldrepl_b(src, 1);
    src13 = __lsx_vldrepl_b(src, 2);
    src12 = __lsx_vldrepl_b(src, 3);
    src11 = __lsx_vldrepl_b(src, 4);
    src10 = __lsx_vldrepl_b(src, 5);
    src9  = __lsx_vldrepl_b(src, 6);
    src8  = __lsx_vldrepl_b(src, 7);
    src7  = __lsx_vldrepl_b(src, 8);
    src6  = __lsx_vldrepl_b(src, 9);
    src5  = __lsx_vldrepl_b(src, 10);
    src4  = __lsx_vldrepl_b(src, 11);
    src3  = __lsx_vldrepl_b(src, 12);
    src2  = __lsx_vldrepl_b(src, 13);
    src1  = __lsx_vldrepl_b(src, 14);
    src0  = __lsx_vldrepl_b(src, 15);
    LSX_ST_8(src0, src1, src2, src3, src4, src5, src6, src7, dst,
             dst_stride, stride2, stride3, stride4);
    dst += stride4;
    LSX_ST_8(src8, src9, src10, src11, src12, src13, src14, src15, dst,
             dst_stride, stride2, stride3, stride4);
}

void ff_hor_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src,
                      const uint8_t *top)
{
    __m128i src0, src1, src2, src3, src4, src5, src6, src7;
    __m128i src8, src9, src10, src11, src12, src13, src14, src15;
    __m128i src16, src17, src18, src19, src20, src21, src22, src23;
    __m128i src24, src25, src26, src27, src28, src29, src30, src31;

    src31 = __lsx_vldrepl_b(src, 0);
    src30 = __lsx_vldrepl_b(src, 1);
    src29 = __lsx_vldrepl_b(src, 2);
    src28 = __lsx_vldrepl_b(src, 3);
    src27 = __lsx_vldrepl_b(src, 4);
    src26 = __lsx_vldrepl_b(src, 5);
    src25 = __lsx_vldrepl_b(src, 6);
    src24 = __lsx_vldrepl_b(src, 7);
    src23 = __lsx_vldrepl_b(src, 8);
    src22 = __lsx_vldrepl_b(src, 9);
    src21 = __lsx_vldrepl_b(src, 10);
    src20 = __lsx_vldrepl_b(src, 11);
    src19 = __lsx_vldrepl_b(src, 12);
    src18 = __lsx_vldrepl_b(src, 13);
    src17 = __lsx_vldrepl_b(src, 14);
    src16 = __lsx_vldrepl_b(src, 15);
    src15 = __lsx_vldrepl_b(src, 16);
    src14 = __lsx_vldrepl_b(src, 17);
    src13 = __lsx_vldrepl_b(src, 18);
    src12 = __lsx_vldrepl_b(src, 19);
    src11 = __lsx_vldrepl_b(src, 20);
    src10 = __lsx_vldrepl_b(src, 21);
    src9  = __lsx_vldrepl_b(src, 22);
    src8  = __lsx_vldrepl_b(src, 23);
    src7  = __lsx_vldrepl_b(src, 24);
    src6  = __lsx_vldrepl_b(src, 25);
    src5  = __lsx_vldrepl_b(src, 26);
    src4  = __lsx_vldrepl_b(src, 27);
    src3  = __lsx_vldrepl_b(src, 28);
    src2  = __lsx_vldrepl_b(src, 29);
    src1  = __lsx_vldrepl_b(src, 30);
    src0  = __lsx_vldrepl_b(src, 31);
    LSX_ST_8X16(src0, src1, src2, src3, src4, src5, src6, src7,
                dst, dst_stride);
    LSX_ST_8X16(src8, src9, src10, src11, src12, src13, src14, src15,
                dst, dst_stride);
    LSX_ST_8X16(src16, src17, src18, src19, src20, src21, src22, src23,
                dst, dst_stride);
    LSX_ST_8X16(src24, src25, src26, src27, src28, src29, src30, src31,
                dst, dst_stride);
}

void ff_dc_4x4_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left,
                   const uint8_t *src_top)
{
    __m128i tmp0, tmp1, dst0;

    tmp0 = __lsx_vldrepl_w(src_top, 0);
    tmp1 = __lsx_vldrepl_w(src_left, 0);
    dst0 = __lsx_vilvl_w(tmp1, tmp0);
    dst0 = __lsx_vhaddw_hu_bu(dst0, dst0);
    dst0 = __lsx_vhaddw_wu_hu(dst0, dst0);
    dst0 = __lsx_vhaddw_du_wu(dst0, dst0);
    dst0 = __lsx_vsrari_w(dst0, 3);
    dst0 = __lsx_vshuf4i_b(dst0, 0);
    __lsx_vstelm_w(dst0, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_w(dst0, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_w(dst0, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_w(dst0, dst, 0, 0);
}

#define INTRA_DC_TL_4X4(dir)                                            \
void ff_dc_##dir##_4x4_lsx(uint8_t *dst, ptrdiff_t dst_stride,          \
                          const uint8_t *left,                          \
                          const uint8_t *top)                           \
{                                                                       \
    __m128i tmp0, dst0;                                                 \
                                                                        \
    tmp0 = __lsx_vldrepl_w(dir, 0);                                     \
    dst0 = __lsx_vhaddw_hu_bu(tmp0, tmp0);                              \
    dst0 = __lsx_vhaddw_wu_hu(dst0, dst0);                              \
    dst0 = __lsx_vsrari_w(dst0, 2);                                     \
    dst0 = __lsx_vshuf4i_b(dst0, 0);                                    \
    __lsx_vstelm_w(dst0, dst, 0, 0);                                    \
    dst += dst_stride;                                                  \
    __lsx_vstelm_w(dst0, dst, 0, 0);                                    \
    dst += dst_stride;                                                  \
    __lsx_vstelm_w(dst0, dst, 0, 0);                                    \
    dst += dst_stride;                                                  \
    __lsx_vstelm_w(dst0, dst, 0, 0);                                    \
}
INTRA_DC_TL_4X4(top);
INTRA_DC_TL_4X4(left);

void ff_dc_8x8_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left,
                   const uint8_t *src_top)
{
    __m128i tmp0, tmp1, dst0;

    tmp0 = __lsx_vldrepl_d(src_top, 0);
    tmp1 = __lsx_vldrepl_d(src_left, 0);
    dst0 = __lsx_vilvl_d(tmp1, tmp0);
    dst0 = __lsx_vhaddw_hu_bu(dst0, dst0);
    dst0 = __lsx_vhaddw_wu_hu(dst0, dst0);
    dst0 = __lsx_vhaddw_du_wu(dst0, dst0);
    dst0 = __lsx_vhaddw_qu_du(dst0, dst0);
    dst0 = __lsx_vsrari_w(dst0, 4);
    dst0 = __lsx_vreplvei_b(dst0, 0);
    __lsx_vstelm_d(dst0, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_d(dst0, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_d(dst0, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_d(dst0, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_d(dst0, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_d(dst0, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_d(dst0, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_d(dst0, dst, 0, 0);
}

#define INTRA_DC_TL_8X8(dir)                                                  \
void ff_dc_##dir##_8x8_lsx(uint8_t *dst, ptrdiff_t dst_stride,                \
                           const uint8_t *left,                               \
                           const uint8_t *top)                                \
{                                                                             \
    __m128i tmp0, dst0;                                                       \
                                                                              \
    tmp0 = __lsx_vldrepl_d(dir, 0);                                           \
    dst0 = __lsx_vhaddw_hu_bu(tmp0, tmp0);                                    \
    dst0 = __lsx_vhaddw_wu_hu(dst0, dst0);                                    \
    dst0 = __lsx_vhaddw_du_wu(dst0, dst0);                                    \
    dst0 = __lsx_vsrari_w(dst0, 3);                                           \
    dst0 = __lsx_vreplvei_b(dst0, 0);                                         \
    __lsx_vstelm_d(dst0, dst, 0, 0);                                          \
    dst += dst_stride;                                                        \
    __lsx_vstelm_d(dst0, dst, 0, 0);                                          \
    dst += dst_stride;                                                        \
    __lsx_vstelm_d(dst0, dst, 0, 0);                                          \
    dst += dst_stride;                                                        \
    __lsx_vstelm_d(dst0, dst, 0, 0);                                          \
    dst += dst_stride;                                                        \
    __lsx_vstelm_d(dst0, dst, 0, 0);                                          \
    dst += dst_stride;                                                        \
    __lsx_vstelm_d(dst0, dst, 0, 0);                                          \
    dst += dst_stride;                                                        \
    __lsx_vstelm_d(dst0, dst, 0, 0);                                          \
    dst += dst_stride;                                                        \
    __lsx_vstelm_d(dst0, dst, 0, 0);                                          \
}

INTRA_DC_TL_8X8(top);
INTRA_DC_TL_8X8(left);

void ff_dc_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride,
                     const uint8_t *src_left, const uint8_t *src_top)
{
    __m128i tmp0, tmp1, dst0;
    ptrdiff_t stride2 = dst_stride << 1;
    ptrdiff_t stride3 = stride2 + dst_stride;
    ptrdiff_t stride4 = stride2 << 1;

    tmp0 = __lsx_vld(src_top, 0);
    tmp1 = __lsx_vld(src_left, 0);
    DUP2_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp0, tmp1);
    dst0 = __lsx_vadd_h(tmp0, tmp1);
    dst0 = __lsx_vhaddw_wu_hu(dst0, dst0);
    dst0 = __lsx_vhaddw_du_wu(dst0, dst0);
    dst0 = __lsx_vhaddw_qu_du(dst0, dst0);
    dst0 = __lsx_vsrari_w(dst0, 5);
    dst0 = __lsx_vreplvei_b(dst0, 0);
    LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst,
             dst_stride, stride2, stride3, stride4);
    dst += stride4;
    LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst,
             dst_stride, stride2, stride3, stride4);
}

#define INTRA_DC_TL_16X16(dir)                                                \
void ff_dc_##dir##_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride,              \
                             const uint8_t *left,                             \
                             const uint8_t *top)                              \
{                                                                             \
    __m128i tmp0, dst0;                                                       \
    ptrdiff_t stride2 = dst_stride << 1;                                      \
    ptrdiff_t stride3 = stride2 + dst_stride;                                 \
    ptrdiff_t stride4 = stride2 << 1;                                         \
                                                                              \
    tmp0 = __lsx_vld(dir, 0);                                                 \
    dst0 = __lsx_vhaddw_hu_bu(tmp0, tmp0);                                    \
    dst0 = __lsx_vhaddw_wu_hu(dst0, dst0);                                    \
    dst0 = __lsx_vhaddw_du_wu(dst0, dst0);                                    \
    dst0 = __lsx_vhaddw_qu_du(dst0, dst0);                                    \
    dst0 = __lsx_vsrari_w(dst0, 4);                                           \
    dst0 = __lsx_vreplvei_b(dst0, 0);                                         \
    LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst,             \
             dst_stride, stride2, stride3, stride4);                          \
    dst += stride4;                                                           \
    LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst,             \
             dst_stride, stride2, stride3, stride4);                          \
}

INTRA_DC_TL_16X16(top);
INTRA_DC_TL_16X16(left);

void ff_dc_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride,
                     const uint8_t *src_left, const uint8_t *src_top)
{
    __m128i tmp0, tmp1, tmp2, tmp3, dst0;

    DUP2_ARG2(__lsx_vld, src_top, 0, src_top, 16, tmp0, tmp1);
    DUP2_ARG2(__lsx_vld, src_left, 0, src_left, 16, tmp2, tmp3);
    DUP4_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp2, tmp2,
              tmp3, tmp3, tmp0, tmp1, tmp2, tmp3);
    DUP2_ARG2(__lsx_vadd_h, tmp0, tmp1, tmp2, tmp3, tmp0, tmp1);
    dst0 = __lsx_vadd_h(tmp0, tmp1);
    dst0 = __lsx_vhaddw_wu_hu(dst0, dst0);
    dst0 = __lsx_vhaddw_du_wu(dst0, dst0);
    dst0 = __lsx_vhaddw_qu_du(dst0, dst0);
    dst0 = __lsx_vsrari_w(dst0, 6);
    dst0 = __lsx_vreplvei_b(dst0, 0);
    LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0,
                dst, dst_stride);
    LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0,
                dst, dst_stride);
    LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0,
                dst, dst_stride);
    LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0,
                dst, dst_stride);
}

#define INTRA_DC_TL_32X32(dir)                                               \
void ff_dc_##dir##_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride,             \
                             const uint8_t *left,                            \
                             const uint8_t *top)                             \
{                                                                            \
    __m128i tmp0, tmp1, dst0;                                                \
                                                                             \
    DUP2_ARG2(__lsx_vld, dir, 0, dir, 16, tmp0, tmp1);                       \
    DUP2_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp0, tmp1);       \
    dst0 = __lsx_vadd_h(tmp0, tmp1);                                         \
    dst0 = __lsx_vhaddw_wu_hu(dst0, dst0);                                   \
    dst0 = __lsx_vhaddw_du_wu(dst0, dst0);                                   \
    dst0 = __lsx_vhaddw_qu_du(dst0, dst0);                                   \
    dst0 = __lsx_vsrari_w(dst0, 5);                                          \
    dst0 = __lsx_vreplvei_b(dst0, 0);                                        \
    LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0,              \
                dst, dst_stride);                                            \
    LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0,              \
                dst, dst_stride);                                            \
    LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0,              \
                dst, dst_stride);                                            \
    LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0,              \
                dst, dst_stride);                                            \
}

INTRA_DC_TL_32X32(top);
INTRA_DC_TL_32X32(left);

#define INTRA_PREDICT_VALDC_16X16_LSX(val)                             \
void ff_dc_##val##_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride,       \
                             const uint8_t *left, const uint8_t *top)  \
{                                                                      \
    __m128i out = __lsx_vldi(val);                                     \
    ptrdiff_t stride2 = dst_stride << 1;                               \
    ptrdiff_t stride3 = stride2 + dst_stride;                          \
    ptrdiff_t stride4 = stride2 << 1;                                  \
                                                                       \
    LSX_ST_8(out, out, out, out, out, out, out, out, dst,              \
             dst_stride, stride2, stride3, stride4);                   \
    dst += stride4;                                                    \
    LSX_ST_8(out, out, out, out, out, out, out, out, dst,              \
             dst_stride, stride2, stride3, stride4);                   \
}

INTRA_PREDICT_VALDC_16X16_LSX(127);
INTRA_PREDICT_VALDC_16X16_LSX(128);
INTRA_PREDICT_VALDC_16X16_LSX(129);

#define INTRA_PREDICT_VALDC_32X32_LSX(val)                               \
void ff_dc_##val##_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride,         \
                             const uint8_t *left, const uint8_t *top)    \
{                                                                        \
    __m128i out = __lsx_vldi(val);                                       \
                                                                         \
    LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\
    LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\
    LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\
    LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\
}

INTRA_PREDICT_VALDC_32X32_LSX(127);
INTRA_PREDICT_VALDC_32X32_LSX(128);
INTRA_PREDICT_VALDC_32X32_LSX(129);

void ff_tm_4x4_lsx(uint8_t *dst, ptrdiff_t dst_stride,
                   const uint8_t *src_left, const uint8_t *src_top_ptr)
{
    uint8_t top_left = src_top_ptr[-1];
    __m128i tmp0, tmp1, tmp2, tmp3, reg0, reg1;
    __m128i src0, src1, src2, src3;
    __m128i dst0, dst1, dst2, dst3;

    reg0 = __lsx_vreplgr2vr_h(top_left);
    reg1 = __lsx_vld(src_top_ptr, 0);
    DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left,
              3, tmp3, tmp2, tmp1, tmp0);
    DUP4_ARG2(__lsx_vilvl_b, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1,
              src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vhaddw_hu_bu, src0, src0, src1, src1, src2, src2, src3,
              src3, dst0, dst1, dst2, dst3);
    DUP4_ARG2(__lsx_vssub_hu, dst0, reg0, dst1, reg0, dst2, reg0, dst3, reg0,
              dst0, dst1, dst2, dst3);
    DUP4_ARG2(__lsx_vsat_hu, dst0, 7, dst1, 7, dst2, 7, dst3, 7,
              dst0, dst1, dst2, dst3);
    DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst0, dst1);
    __lsx_vstelm_w(dst0, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_w(dst0, dst, 0, 2);
    dst += dst_stride;
    __lsx_vstelm_w(dst1, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_w(dst1, dst, 0, 2);
}

void ff_tm_8x8_lsx(uint8_t *dst, ptrdiff_t dst_stride,
                   const uint8_t *src_left, const uint8_t *src_top_ptr)
{
    uint8_t top_left = src_top_ptr[-1];
    __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
    __m128i src0, src1, src2, src3, src4, src5, src6, src7;
    __m128i reg0, reg1;

    reg0 = __lsx_vreplgr2vr_h(top_left);
    reg1 = __lsx_vld(src_top_ptr, 0);
    DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left,
              3, tmp7, tmp6, tmp5, tmp4);
    DUP4_ARG2(__lsx_vldrepl_b, src_left, 4, src_left, 5, src_left, 6, src_left,
              7, tmp3, tmp2, tmp1, tmp0);
    DUP4_ARG2(__lsx_vilvl_b, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1,
              src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vilvl_b, tmp4, reg1, tmp5, reg1, tmp6, reg1, tmp7, reg1,
              src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vhaddw_hu_bu, src0, src0, src1, src1, src2, src2, src3,
              src3, src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vhaddw_hu_bu, src4, src4, src5, src5, src6, src6, src7,
              src7, src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0,
              src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0,
              src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7,
              src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7,
              src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vpickev_b, src1, src0, src3, src2, src5, src4, src7, src6,
              src0, src1, src2, src3);
    __lsx_vstelm_d(src0, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_d(src0, dst, 0, 1);
    dst += dst_stride;
    __lsx_vstelm_d(src1, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_d(src1, dst, 0, 1);
    dst += dst_stride;
    __lsx_vstelm_d(src2, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_d(src2, dst, 0, 1);
    dst += dst_stride;
    __lsx_vstelm_d(src3, dst, 0, 0);
    dst += dst_stride;
    __lsx_vstelm_d(src3, dst, 0, 1);
}

void ff_tm_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride,
                     const uint8_t *src_left, const uint8_t *src_top_ptr)
{
    uint8_t top_left = src_top_ptr[-1];
    __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
    __m128i tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15;
    __m128i src0, src1, src2, src3, src4, src5, src6, src7;
    __m128i reg0, reg1;
    ptrdiff_t stride2 = dst_stride << 1;
    ptrdiff_t stride3 = stride2 + dst_stride;
    ptrdiff_t stride4 = stride2 << 1;

    reg0 = __lsx_vreplgr2vr_h(top_left);
    reg1 = __lsx_vld(src_top_ptr, 0);
    DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left,
              3, tmp15, tmp14, tmp13, tmp12);
    DUP4_ARG2(__lsx_vldrepl_b, src_left, 4, src_left, 5, src_left, 6, src_left,
              7, tmp11, tmp10, tmp9, tmp8);
    DUP4_ARG2(__lsx_vldrepl_b, src_left, 8, src_left, 9, src_left, 10,
              src_left, 11, tmp7, tmp6, tmp5, tmp4);
    DUP4_ARG2(__lsx_vldrepl_b, src_left, 12, src_left, 13, src_left, 14,
              src_left, 15, tmp3, tmp2, tmp1, tmp0);
    DUP4_ARG2(__lsx_vaddwev_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3,
              reg1, src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vaddwod_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3,
              reg1, src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0,
              src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0,
              src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7,
              src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7,
              src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3,
              tmp0, tmp1, tmp2, tmp3);
    DUP4_ARG2(__lsx_vaddwev_h_bu, tmp4, reg1, tmp5, reg1, tmp6, reg1, tmp7,
              reg1, src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vaddwod_h_bu, tmp4, reg1, tmp5, reg1, tmp6, reg1, tmp7,
              reg1, src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0,
              src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0,
              src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7,
              src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7,
              src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3,
              tmp4, tmp5, tmp6, tmp7);
    DUP4_ARG2(__lsx_vaddwev_h_bu, tmp8, reg1, tmp9, reg1, tmp10, reg1, tmp11,
              reg1, src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vaddwod_h_bu, tmp8, reg1, tmp9, reg1, tmp10, reg1, tmp11,
              reg1, src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0,
              src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0,
              src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7,
              src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7,
              src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3,
              tmp8, tmp9, tmp10, tmp11);
    DUP4_ARG2(__lsx_vaddwev_h_bu, tmp12, reg1, tmp13, reg1, tmp14, reg1,
              tmp15, reg1, src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vaddwod_h_bu, tmp12, reg1, tmp13, reg1, tmp14, reg1,
              tmp15, reg1, src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0,
              src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0,
              src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7,
              src0, src1, src2, src3);
    DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7,
              src4, src5, src6, src7);
    DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3,
              tmp12, tmp13, tmp14, tmp15);
    LSX_ST_8(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, dst,
             dst_stride, stride2, stride3, stride4);
    dst += stride4;
    LSX_ST_8(tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15, dst,
             dst_stride, stride2, stride3, stride4);
}

void ff_tm_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride,
                     const uint8_t *src_left, const uint8_t *src_top_ptr)
{
    uint8_t top_left = src_top_ptr[-1];
    uint32_t loop_cnt;
    __m128i tmp0, tmp1, tmp2, tmp3, reg0, reg1, reg2;
    __m128i src0, src1, src2, src3, src4, src5, src6, src7;
    __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;

    reg0 = __lsx_vreplgr2vr_h(top_left);
    DUP2_ARG2(__lsx_vld, src_top_ptr, 0, src_top_ptr, 16, reg1, reg2);

    src_left += 28;
    for (loop_cnt = 8; loop_cnt--;) {
        DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2,
                  src_left, 3, tmp3, tmp2, tmp1, tmp0);
        src_left -= 4;
        DUP4_ARG2(__lsx_vaddwev_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1,
                  tmp3, reg1, src0, src1, src2, src3);
        DUP4_ARG2(__lsx_vaddwod_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1,
                  tmp3, reg1, src4, src5, src6, src7);
        DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3,
                  reg0, src0, src1, src2, src3);
        DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7,
                  reg0, src4, src5, src6, src7);
        DUP4_ARG2(__lsx_vaddwev_h_bu, tmp0, reg2, tmp1, reg2, tmp2, reg2,
                  tmp3, reg2, dst0, dst1, dst2, dst3);
        DUP4_ARG2(__lsx_vaddwod_h_bu, tmp0, reg2, tmp1, reg2, tmp2, reg2,
                  tmp3, reg2, dst4, dst5, dst6, dst7);
        DUP4_ARG2(__lsx_vssub_hu, dst0, reg0, dst1, reg0, dst2, reg0, dst3,
                  reg0, dst0, dst1, dst2, dst3);
        DUP4_ARG2(__lsx_vssub_hu, dst4, reg0, dst5, reg0, dst6, reg0, dst7,
                  reg0, dst4, dst5, dst6, dst7);
        DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7,
                  src0, src1, src2, src3);
        DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7,
                  src4, src5, src6, src7);
        DUP4_ARG2(__lsx_vsat_hu, dst0, 7, dst1, 7, dst2, 7, dst3, 7,
                  dst0, dst1, dst2, dst3);
        DUP4_ARG2(__lsx_vsat_hu, dst4, 7, dst5, 7, dst6, 7, dst7, 7,
                  dst4, dst5, dst6, dst7);
        DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7,
                  src3, src0, src1, src2, src3);
        DUP4_ARG2(__lsx_vpackev_b, dst4, dst0, dst5, dst1, dst6, dst2, dst7,
                  dst3, dst0, dst1, dst2, dst3);
        __lsx_vst(src0, dst, 0);
        __lsx_vst(dst0, dst, 16);
        dst += dst_stride;
        __lsx_vst(src1, dst, 0);
        __lsx_vst(dst1, dst, 16);
        dst += dst_stride;
        __lsx_vst(src2, dst, 0);
        __lsx_vst(dst2, dst, 16);
        dst += dst_stride;
        __lsx_vst(src3, dst, 0);
        __lsx_vst(dst3, dst, 16);
        dst += dst_stride;
    }
}