diff options
Diffstat (limited to 'ffmpeg1/libswscale/x86/scale.asm')
| -rw-r--r-- | ffmpeg1/libswscale/x86/scale.asm | 431 |
1 files changed, 431 insertions, 0 deletions
diff --git a/ffmpeg1/libswscale/x86/scale.asm b/ffmpeg1/libswscale/x86/scale.asm new file mode 100644 index 0000000..c6dafde --- /dev/null +++ b/ffmpeg1/libswscale/x86/scale.asm @@ -0,0 +1,431 @@ +;****************************************************************************** +;* x86-optimized horizontal line scaling functions +;* Copyright (c) 2011 Ronald S. Bultje <rsbultje@gmail.com> +;* +;* This file is part of Libav. +;* +;* Libav is free software; you can redistribute it and/or +;* modify it under the terms of the GNU Lesser General Public +;* License as published by the Free Software Foundation; either +;* version 2.1 of the License, or (at your option) any later version. +;* +;* Libav is distributed in the hope that it will be useful, +;* but WITHOUT ANY WARRANTY; without even the implied warranty of +;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;* Lesser General Public License for more details. +;* +;* You should have received a copy of the GNU Lesser General Public +;* License along with Libav; if not, write to the Free Software +;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +;****************************************************************************** + +%include "libavutil/x86/x86util.asm" + +SECTION_RODATA + +max_19bit_int: times 4 dd 0x7ffff +max_19bit_flt: times 4 dd 524287.0 +minshort: times 8 dw 0x8000 +unicoeff: times 4 dd 0x20000000 + +SECTION .text + +;----------------------------------------------------------------------------- +; horizontal line scaling +; +; void hscale<source_width>to<intermediate_nbits>_<filterSize>_<opt> +; (SwsContext *c, int{16,32}_t *dst, +; int dstW, const uint{8,16}_t *src, +; const int16_t *filter, +; const int32_t *filterPos, int filterSize); +; +; Scale one horizontal line. Input is either 8-bits width or 16-bits width +; ($source_width can be either 8, 9, 10 or 16, difference is whether we have to +; downscale before multiplying). Filter is 14-bits. Output is either 15bits +; (in int16_t) or 19bits (in int32_t), as given in $intermediate_nbits. Each +; output pixel is generated from $filterSize input pixels, the position of +; the first pixel is given in filterPos[nOutputPixel]. +;----------------------------------------------------------------------------- + +; SCALE_FUNC source_width, intermediate_nbits, filtersize, filtersuffix, n_args, n_xmm +%macro SCALE_FUNC 6 +%ifnidn %3, X +cglobal hscale%1to%2_%4, %5, 7, %6, pos0, dst, w, src, filter, fltpos, pos1 +%else +cglobal hscale%1to%2_%4, %5, 10, %6, pos0, dst, w, srcmem, filter, fltpos, fltsize +%endif +%if ARCH_X86_64 + movsxd wq, wd +%define mov32 movsxd +%else ; x86-32 +%define mov32 mov +%endif ; x86-64 +%if %2 == 19 +%if mmsize == 8 ; mmx + mova m2, [max_19bit_int] +%elif cpuflag(sse4) + mova m2, [max_19bit_int] +%else ; ssse3/sse2 + mova m2, [max_19bit_flt] +%endif ; mmx/sse2/ssse3/sse4 +%endif ; %2 == 19 +%if %1 == 16 + mova m6, [minshort] + mova m7, [unicoeff] +%elif %1 == 8 + pxor m3, m3 +%endif ; %1 == 8/16 + +%if %1 == 8 +%define movlh movd +%define movbh movh +%define srcmul 1 +%else ; %1 == 9-16 +%define movlh movq +%define movbh movu +%define srcmul 2 +%endif ; %1 == 8/9-16 + +%ifnidn %3, X + + ; setup loop +%if %3 == 8 + shl wq, 1 ; this allows *16 (i.e. now *8) in lea instructions for the 8-tap filter +%define wshr 1 +%else ; %3 == 4 +%define wshr 0 +%endif ; %3 == 8 + lea filterq, [filterq+wq*8] +%if %2 == 15 + lea dstq, [dstq+wq*(2>>wshr)] +%else ; %2 == 19 + lea dstq, [dstq+wq*(4>>wshr)] +%endif ; %2 == 15/19 + lea fltposq, [fltposq+wq*(4>>wshr)] + neg wq + +.loop: +%if %3 == 4 ; filterSize == 4 scaling + ; load 2x4 or 4x4 source pixels into m0/m1 + mov32 pos0q, dword [fltposq+wq*4+ 0] ; filterPos[0] + mov32 pos1q, dword [fltposq+wq*4+ 4] ; filterPos[1] + movlh m0, [srcq+pos0q*srcmul] ; src[filterPos[0] + {0,1,2,3}] +%if mmsize == 8 + movlh m1, [srcq+pos1q*srcmul] ; src[filterPos[1] + {0,1,2,3}] +%else ; mmsize == 16 +%if %1 > 8 + movhps m0, [srcq+pos1q*srcmul] ; src[filterPos[1] + {0,1,2,3}] +%else ; %1 == 8 + movd m4, [srcq+pos1q*srcmul] ; src[filterPos[1] + {0,1,2,3}] +%endif + mov32 pos0q, dword [fltposq+wq*4+ 8] ; filterPos[2] + mov32 pos1q, dword [fltposq+wq*4+12] ; filterPos[3] + movlh m1, [srcq+pos0q*srcmul] ; src[filterPos[2] + {0,1,2,3}] +%if %1 > 8 + movhps m1, [srcq+pos1q*srcmul] ; src[filterPos[3] + {0,1,2,3}] +%else ; %1 == 8 + movd m5, [srcq+pos1q*srcmul] ; src[filterPos[3] + {0,1,2,3}] + punpckldq m0, m4 + punpckldq m1, m5 +%endif ; %1 == 8 +%endif ; mmsize == 8/16 +%if %1 == 8 + punpcklbw m0, m3 ; byte -> word + punpcklbw m1, m3 ; byte -> word +%endif ; %1 == 8 + + ; multiply with filter coefficients +%if %1 == 16 ; pmaddwd needs signed adds, so this moves unsigned -> signed, we'll + ; add back 0x8000 * sum(coeffs) after the horizontal add + psubw m0, m6 + psubw m1, m6 +%endif ; %1 == 16 + pmaddwd m0, [filterq+wq*8+mmsize*0] ; *= filter[{0,1,..,6,7}] + pmaddwd m1, [filterq+wq*8+mmsize*1] ; *= filter[{8,9,..,14,15}] + + ; add up horizontally (4 srcpix * 4 coefficients -> 1 dstpix) +%if mmsize == 8 ; mmx + movq m4, m0 + punpckldq m0, m1 + punpckhdq m4, m1 + paddd m0, m4 +%elif notcpuflag(ssse3) ; sse2 + mova m4, m0 + shufps m0, m1, 10001000b + shufps m4, m1, 11011101b + paddd m0, m4 +%else ; ssse3/sse4 + phaddd m0, m1 ; filter[{ 0, 1, 2, 3}]*src[filterPos[0]+{0,1,2,3}], + ; filter[{ 4, 5, 6, 7}]*src[filterPos[1]+{0,1,2,3}], + ; filter[{ 8, 9,10,11}]*src[filterPos[2]+{0,1,2,3}], + ; filter[{12,13,14,15}]*src[filterPos[3]+{0,1,2,3}] +%endif ; mmx/sse2/ssse3/sse4 +%else ; %3 == 8, i.e. filterSize == 8 scaling + ; load 2x8 or 4x8 source pixels into m0, m1, m4 and m5 + mov32 pos0q, dword [fltposq+wq*2+0] ; filterPos[0] + mov32 pos1q, dword [fltposq+wq*2+4] ; filterPos[1] + movbh m0, [srcq+ pos0q *srcmul] ; src[filterPos[0] + {0,1,2,3,4,5,6,7}] +%if mmsize == 8 + movbh m1, [srcq+(pos0q+4)*srcmul] ; src[filterPos[0] + {4,5,6,7}] + movbh m4, [srcq+ pos1q *srcmul] ; src[filterPos[1] + {0,1,2,3}] + movbh m5, [srcq+(pos1q+4)*srcmul] ; src[filterPos[1] + {4,5,6,7}] +%else ; mmsize == 16 + movbh m1, [srcq+ pos1q *srcmul] ; src[filterPos[1] + {0,1,2,3,4,5,6,7}] + mov32 pos0q, dword [fltposq+wq*2+8] ; filterPos[2] + mov32 pos1q, dword [fltposq+wq*2+12] ; filterPos[3] + movbh m4, [srcq+ pos0q *srcmul] ; src[filterPos[2] + {0,1,2,3,4,5,6,7}] + movbh m5, [srcq+ pos1q *srcmul] ; src[filterPos[3] + {0,1,2,3,4,5,6,7}] +%endif ; mmsize == 8/16 +%if %1 == 8 + punpcklbw m0, m3 ; byte -> word + punpcklbw m1, m3 ; byte -> word + punpcklbw m4, m3 ; byte -> word + punpcklbw m5, m3 ; byte -> word +%endif ; %1 == 8 + + ; multiply +%if %1 == 16 ; pmaddwd needs signed adds, so this moves unsigned -> signed, we'll + ; add back 0x8000 * sum(coeffs) after the horizontal add + psubw m0, m6 + psubw m1, m6 + psubw m4, m6 + psubw m5, m6 +%endif ; %1 == 16 + pmaddwd m0, [filterq+wq*8+mmsize*0] ; *= filter[{0,1,..,6,7}] + pmaddwd m1, [filterq+wq*8+mmsize*1] ; *= filter[{8,9,..,14,15}] + pmaddwd m4, [filterq+wq*8+mmsize*2] ; *= filter[{16,17,..,22,23}] + pmaddwd m5, [filterq+wq*8+mmsize*3] ; *= filter[{24,25,..,30,31}] + + ; add up horizontally (8 srcpix * 8 coefficients -> 1 dstpix) +%if mmsize == 8 + paddd m0, m1 + paddd m4, m5 + movq m1, m0 + punpckldq m0, m4 + punpckhdq m1, m4 + paddd m0, m1 +%elif notcpuflag(ssse3) ; sse2 +%if %1 == 8 +%define mex m6 +%else +%define mex m3 +%endif + ; emulate horizontal add as transpose + vertical add + mova mex, m0 + punpckldq m0, m1 + punpckhdq mex, m1 + paddd m0, mex + mova m1, m4 + punpckldq m4, m5 + punpckhdq m1, m5 + paddd m4, m1 + mova m1, m0 + punpcklqdq m0, m4 + punpckhqdq m1, m4 + paddd m0, m1 +%else ; ssse3/sse4 + ; FIXME if we rearrange the filter in pairs of 4, we can + ; load pixels likewise and use 2 x paddd + phaddd instead + ; of 3 x phaddd here, faster on older cpus + phaddd m0, m1 + phaddd m4, m5 + phaddd m0, m4 ; filter[{ 0, 1,..., 6, 7}]*src[filterPos[0]+{0,1,...,6,7}], + ; filter[{ 8, 9,...,14,15}]*src[filterPos[1]+{0,1,...,6,7}], + ; filter[{16,17,...,22,23}]*src[filterPos[2]+{0,1,...,6,7}], + ; filter[{24,25,...,30,31}]*src[filterPos[3]+{0,1,...,6,7}] +%endif ; mmx/sse2/ssse3/sse4 +%endif ; %3 == 4/8 + +%else ; %3 == X, i.e. any filterSize scaling + +%ifidn %4, X4 +%define dlt 4 +%else ; %4 == X || %4 == X8 +%define dlt 0 +%endif ; %4 ==/!= X4 +%if ARCH_X86_64 +%define srcq r8 +%define pos1q r7 +%define srcendq r9 + movsxd fltsizeq, fltsized ; filterSize + lea srcendq, [srcmemq+(fltsizeq-dlt)*srcmul] ; &src[filterSize&~4] +%else ; x86-32 +%define srcq srcmemq +%define pos1q dstq +%define srcendq r6m + lea pos0q, [srcmemq+(fltsizeq-dlt)*srcmul] ; &src[filterSize&~4] + mov srcendq, pos0q +%endif ; x86-32/64 + lea fltposq, [fltposq+wq*4] +%if %2 == 15 + lea dstq, [dstq+wq*2] +%else ; %2 == 19 + lea dstq, [dstq+wq*4] +%endif ; %2 == 15/19 + movifnidn dstmp, dstq + neg wq + +.loop: + mov32 pos0q, dword [fltposq+wq*4+0] ; filterPos[0] + mov32 pos1q, dword [fltposq+wq*4+4] ; filterPos[1] + ; FIXME maybe do 4px/iteration on x86-64 (x86-32 wouldn't have enough regs)? + pxor m4, m4 + pxor m5, m5 + mov srcq, srcmemmp + +.innerloop: + ; load 2x4 (mmx) or 2x8 (sse) source pixels into m0/m1 -> m4/m5 + movbh m0, [srcq+ pos0q *srcmul] ; src[filterPos[0] + {0,1,2,3(,4,5,6,7)}] + movbh m1, [srcq+(pos1q+dlt)*srcmul] ; src[filterPos[1] + {0,1,2,3(,4,5,6,7)}] +%if %1 == 8 + punpcklbw m0, m3 + punpcklbw m1, m3 +%endif ; %1 == 8 + + ; multiply +%if %1 == 16 ; pmaddwd needs signed adds, so this moves unsigned -> signed, we'll + ; add back 0x8000 * sum(coeffs) after the horizontal add + psubw m0, m6 + psubw m1, m6 +%endif ; %1 == 16 + pmaddwd m0, [filterq] ; filter[{0,1,2,3(,4,5,6,7)}] + pmaddwd m1, [filterq+(fltsizeq+dlt)*2]; filter[filtersize+{0,1,2,3(,4,5,6,7)}] + paddd m4, m0 + paddd m5, m1 + add filterq, mmsize + add srcq, srcmul*mmsize/2 + cmp srcq, srcendq ; while (src += 4) < &src[filterSize] + jl .innerloop + +%ifidn %4, X4 + mov32 pos1q, dword [fltposq+wq*4+4] ; filterPos[1] + movlh m0, [srcq+ pos0q *srcmul] ; split last 4 srcpx of dstpx[0] + sub pos1q, fltsizeq ; and first 4 srcpx of dstpx[1] +%if %1 > 8 + movhps m0, [srcq+(pos1q+dlt)*srcmul] +%else ; %1 == 8 + movd m1, [srcq+(pos1q+dlt)*srcmul] + punpckldq m0, m1 +%endif ; %1 == 8 +%if %1 == 8 + punpcklbw m0, m3 +%endif ; %1 == 8 +%if %1 == 16 ; pmaddwd needs signed adds, so this moves unsigned -> signed, we'll + ; add back 0x8000 * sum(coeffs) after the horizontal add + psubw m0, m6 +%endif ; %1 == 16 + pmaddwd m0, [filterq] +%endif ; %4 == X4 + + lea filterq, [filterq+(fltsizeq+dlt)*2] + +%if mmsize == 8 ; mmx + movq m0, m4 + punpckldq m4, m5 + punpckhdq m0, m5 + paddd m0, m4 +%else ; mmsize == 16 +%if notcpuflag(ssse3) ; sse2 + mova m1, m4 + punpcklqdq m4, m5 + punpckhqdq m1, m5 + paddd m4, m1 +%else ; ssse3/sse4 + phaddd m4, m5 +%endif ; sse2/ssse3/sse4 +%ifidn %4, X4 + paddd m4, m0 +%endif ; %3 == X4 +%if notcpuflag(ssse3) ; sse2 + pshufd m4, m4, 11011000b + movhlps m0, m4 + paddd m0, m4 +%else ; ssse3/sse4 + phaddd m4, m4 + SWAP 0, 4 +%endif ; sse2/ssse3/sse4 +%endif ; mmsize == 8/16 +%endif ; %3 ==/!= X + +%if %1 == 16 ; add 0x8000 * sum(coeffs), i.e. back from signed -> unsigned + paddd m0, m7 +%endif ; %1 == 16 + + ; clip, store + psrad m0, 14 + %1 - %2 +%ifidn %3, X + movifnidn dstq, dstmp +%endif ; %3 == X +%if %2 == 15 + packssdw m0, m0 +%ifnidn %3, X + movh [dstq+wq*(2>>wshr)], m0 +%else ; %3 == X + movd [dstq+wq*2], m0 +%endif ; %3 ==/!= X +%else ; %2 == 19 +%if mmsize == 8 + PMINSD_MMX m0, m2, m4 +%elif cpuflag(sse4) + pminsd m0, m2 +%else ; sse2/ssse3 + cvtdq2ps m0, m0 + minps m0, m2 + cvtps2dq m0, m0 +%endif ; mmx/sse2/ssse3/sse4 +%ifnidn %3, X + mova [dstq+wq*(4>>wshr)], m0 +%else ; %3 == X + movq [dstq+wq*4], m0 +%endif ; %3 ==/!= X +%endif ; %2 == 15/19 +%ifnidn %3, X + add wq, (mmsize<<wshr)/4 ; both 8tap and 4tap really only do 4 pixels (or for mmx: 2 pixels) + ; per iteration. see "shl wq,1" above as for why we do this +%else ; %3 == X + add wq, 2 +%endif ; %3 ==/!= X + jl .loop + REP_RET +%endmacro + +; SCALE_FUNCS source_width, intermediate_nbits, n_xmm +%macro SCALE_FUNCS 3 +SCALE_FUNC %1, %2, 4, 4, 6, %3 +SCALE_FUNC %1, %2, 8, 8, 6, %3 +%if mmsize == 8 +SCALE_FUNC %1, %2, X, X, 7, %3 +%else +SCALE_FUNC %1, %2, X, X4, 7, %3 +SCALE_FUNC %1, %2, X, X8, 7, %3 +%endif +%endmacro + +; SCALE_FUNCS2 8_xmm_args, 9to10_xmm_args, 16_xmm_args +%macro SCALE_FUNCS2 3 +%if notcpuflag(sse4) +SCALE_FUNCS 8, 15, %1 +SCALE_FUNCS 9, 15, %2 +SCALE_FUNCS 10, 15, %2 +SCALE_FUNCS 12, 15, %2 +SCALE_FUNCS 14, 15, %2 +SCALE_FUNCS 16, 15, %3 +%endif ; !sse4 +SCALE_FUNCS 8, 19, %1 +SCALE_FUNCS 9, 19, %2 +SCALE_FUNCS 10, 19, %2 +SCALE_FUNCS 12, 19, %2 +SCALE_FUNCS 14, 19, %2 +SCALE_FUNCS 16, 19, %3 +%endmacro + +%if ARCH_X86_32 +INIT_MMX mmx +SCALE_FUNCS2 0, 0, 0 +%endif +INIT_XMM sse2 +SCALE_FUNCS2 6, 7, 8 +INIT_XMM ssse3 +SCALE_FUNCS2 6, 6, 8 +INIT_XMM sse4 +SCALE_FUNCS2 6, 6, 8 |