Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(94)

Issues Search
    My Issues | Starred     Open | Closed | All

Side by Side Diff: source/libvpx/vp9/encoder/vp9_rdopt.c

Issue 11555023: libvpx: Add VP9 decoder. (Closed) Base URL: svn://chrome-svn/chrome/trunk/deps/third_party/libvpx/
Patch Set: Created 8 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« libvpx.gyp ('K') | « source/libvpx/vp9/encoder/vp9_rdopt.h ('k') | source/libvpx/vp9/encoder/vp9_sad_c.c » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Property Changes:
Added: svn:eol-style
+ LF
OLDNEW
(Empty)
1 /*
2 * Copyright (c) 2010 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11
12 #include <stdio.h>
13 #include <math.h>
14 #include <limits.h>
15 #include <assert.h>
16 #include "vp9/common/vp9_pragmas.h"
17
18 #include "vp9/encoder/vp9_tokenize.h"
19 #include "vp9/encoder/vp9_treewriter.h"
20 #include "vp9/encoder/vp9_onyx_int.h"
21 #include "vp9/encoder/vp9_modecosts.h"
22 #include "vp9/encoder/vp9_encodeintra.h"
23 #include "vp9/common/vp9_entropymode.h"
24 #include "vp9/common/vp9_reconinter.h"
25 #include "vp9/common/vp9_reconintra.h"
26 #include "vp9/common/vp9_reconintra4x4.h"
27 #include "vp9/common/vp9_findnearmv.h"
28 #include "vp9/common/vp9_quant_common.h"
29 #include "vp9/encoder/vp9_encodemb.h"
30 #include "vp9/encoder/vp9_quantize.h"
31 #include "vp9/encoder/vp9_variance.h"
32 #include "vp9/encoder/vp9_mcomp.h"
33 #include "vp9/encoder/vp9_rdopt.h"
34 #include "vp9/encoder/vp9_ratectrl.h"
35 #include "vpx_mem/vpx_mem.h"
36 #include "vp9/common/vp9_systemdependent.h"
37 #include "vp9/encoder/vp9_encodemv.h"
38
39 #include "vp9/common/vp9_seg_common.h"
40 #include "vp9/common/vp9_pred_common.h"
41 #include "vp9/common/vp9_entropy.h"
42 #include "vp9_rtcd.h"
43 #include "vp9/common/vp9_mvref_common.h"
44
45 #define MAXF(a,b) (((a) > (b)) ? (a) : (b))
46
47 #define INVALID_MV 0x80008000
48
49 /* Factor to weigh the rate for switchable interp filters */
50 #define SWITCHABLE_INTERP_RATE_FACTOR 1
51
52 static const int auto_speed_thresh[17] = {
53 1000,
54 200,
55 150,
56 130,
57 150,
58 125,
59 120,
60 115,
61 115,
62 115,
63 115,
64 115,
65 115,
66 115,
67 115,
68 115,
69 105
70 };
71
72 #if CONFIG_PRED_FILTER
73 const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
74 {ZEROMV, LAST_FRAME, NONE, 0},
75 {ZEROMV, LAST_FRAME, NONE, 1},
76 {DC_PRED, INTRA_FRAME, NONE, 0},
77
78 {NEARESTMV, LAST_FRAME, NONE, 0},
79 {NEARESTMV, LAST_FRAME, NONE, 1},
80 {NEARMV, LAST_FRAME, NONE, 0},
81 {NEARMV, LAST_FRAME, NONE, 1},
82
83 {ZEROMV, GOLDEN_FRAME, NONE, 0},
84 {ZEROMV, GOLDEN_FRAME, NONE, 1},
85 {NEARESTMV, GOLDEN_FRAME, NONE, 0},
86 {NEARESTMV, GOLDEN_FRAME, NONE, 1},
87
88 {ZEROMV, ALTREF_FRAME, NONE, 0},
89 {ZEROMV, ALTREF_FRAME, NONE, 1},
90 {NEARESTMV, ALTREF_FRAME, NONE, 0},
91 {NEARESTMV, ALTREF_FRAME, NONE, 1},
92
93 {NEARMV, GOLDEN_FRAME, NONE, 0},
94 {NEARMV, GOLDEN_FRAME, NONE, 1},
95 {NEARMV, ALTREF_FRAME, NONE, 0},
96 {NEARMV, ALTREF_FRAME, NONE, 1},
97
98 {V_PRED, INTRA_FRAME, NONE, 0},
99 {H_PRED, INTRA_FRAME, NONE, 0},
100 {D45_PRED, INTRA_FRAME, NONE, 0},
101 {D135_PRED, INTRA_FRAME, NONE, 0},
102 {D117_PRED, INTRA_FRAME, NONE, 0},
103 {D153_PRED, INTRA_FRAME, NONE, 0},
104 {D27_PRED, INTRA_FRAME, NONE, 0},
105 {D63_PRED, INTRA_FRAME, NONE, 0},
106
107 {TM_PRED, INTRA_FRAME, NONE, 0},
108
109 {NEWMV, LAST_FRAME, NONE, 0},
110 {NEWMV, LAST_FRAME, NONE, 1},
111 {NEWMV, GOLDEN_FRAME, NONE, 0},
112 {NEWMV, GOLDEN_FRAME, NONE, 1},
113 {NEWMV, ALTREF_FRAME, NONE, 0},
114 {NEWMV, ALTREF_FRAME, NONE, 1},
115
116 {SPLITMV, LAST_FRAME, NONE, 0},
117 {SPLITMV, GOLDEN_FRAME, NONE, 0},
118 {SPLITMV, ALTREF_FRAME, NONE, 0},
119
120 {B_PRED, INTRA_FRAME, NONE, 0},
121 {I8X8_PRED, INTRA_FRAME, NONE, 0},
122
123 /* compound prediction modes */
124 {ZEROMV, LAST_FRAME, GOLDEN_FRAME, 0},
125 {NEARESTMV, LAST_FRAME, GOLDEN_FRAME, 0},
126 {NEARMV, LAST_FRAME, GOLDEN_FRAME, 0},
127
128 {ZEROMV, ALTREF_FRAME, LAST_FRAME, 0},
129 {NEARESTMV, ALTREF_FRAME, LAST_FRAME, 0},
130 {NEARMV, ALTREF_FRAME, LAST_FRAME, 0},
131
132 {ZEROMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
133 {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
134 {NEARMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
135
136 {NEWMV, LAST_FRAME, GOLDEN_FRAME, 0},
137 {NEWMV, ALTREF_FRAME, LAST_FRAME, 0},
138 {NEWMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
139
140 {SPLITMV, LAST_FRAME, GOLDEN_FRAME, 0},
141 {SPLITMV, ALTREF_FRAME, LAST_FRAME, 0},
142 {SPLITMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
143
144 #if CONFIG_COMP_INTERINTRA_PRED
145 /* compound inter-intra prediction */
146 {ZEROMV, LAST_FRAME, INTRA_FRAME, 0},
147 {NEARESTMV, LAST_FRAME, INTRA_FRAME, 0},
148 {NEARMV, LAST_FRAME, INTRA_FRAME, 0},
149 {NEWMV, LAST_FRAME, INTRA_FRAME, 0},
150
151 {ZEROMV, GOLDEN_FRAME, INTRA_FRAME, 0},
152 {NEARESTMV, GOLDEN_FRAME, INTRA_FRAME, 0},
153 {NEARMV, GOLDEN_FRAME, INTRA_FRAME, 0},
154 {NEWMV, GOLDEN_FRAME, INTRA_FRAME, 0},
155
156 {ZEROMV, ALTREF_FRAME, INTRA_FRAME, 0},
157 {NEARESTMV, ALTREF_FRAME, INTRA_FRAME, 0},
158 {NEARMV, ALTREF_FRAME, INTRA_FRAME, 0},
159 {NEWMV, ALTREF_FRAME, INTRA_FRAME, 0},
160 #endif
161 };
162 #else
163 const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
164 {ZEROMV, LAST_FRAME, NONE},
165 {DC_PRED, INTRA_FRAME, NONE},
166
167 {NEARESTMV, LAST_FRAME, NONE},
168 {NEARMV, LAST_FRAME, NONE},
169
170 {ZEROMV, GOLDEN_FRAME, NONE},
171 {NEARESTMV, GOLDEN_FRAME, NONE},
172
173 {ZEROMV, ALTREF_FRAME, NONE},
174 {NEARESTMV, ALTREF_FRAME, NONE},
175
176 {NEARMV, GOLDEN_FRAME, NONE},
177 {NEARMV, ALTREF_FRAME, NONE},
178
179 {V_PRED, INTRA_FRAME, NONE},
180 {H_PRED, INTRA_FRAME, NONE},
181 {D45_PRED, INTRA_FRAME, NONE},
182 {D135_PRED, INTRA_FRAME, NONE},
183 {D117_PRED, INTRA_FRAME, NONE},
184 {D153_PRED, INTRA_FRAME, NONE},
185 {D27_PRED, INTRA_FRAME, NONE},
186 {D63_PRED, INTRA_FRAME, NONE},
187
188 {TM_PRED, INTRA_FRAME, NONE},
189
190 {NEWMV, LAST_FRAME, NONE},
191 {NEWMV, GOLDEN_FRAME, NONE},
192 {NEWMV, ALTREF_FRAME, NONE},
193
194 {SPLITMV, LAST_FRAME, NONE},
195 {SPLITMV, GOLDEN_FRAME, NONE},
196 {SPLITMV, ALTREF_FRAME, NONE},
197
198 {B_PRED, INTRA_FRAME, NONE},
199 {I8X8_PRED, INTRA_FRAME, NONE},
200
201 /* compound prediction modes */
202 {ZEROMV, LAST_FRAME, GOLDEN_FRAME},
203 {NEARESTMV, LAST_FRAME, GOLDEN_FRAME},
204 {NEARMV, LAST_FRAME, GOLDEN_FRAME},
205
206 {ZEROMV, ALTREF_FRAME, LAST_FRAME},
207 {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
208 {NEARMV, ALTREF_FRAME, LAST_FRAME},
209
210 {ZEROMV, GOLDEN_FRAME, ALTREF_FRAME},
211 {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
212 {NEARMV, GOLDEN_FRAME, ALTREF_FRAME},
213
214 {NEWMV, LAST_FRAME, GOLDEN_FRAME},
215 {NEWMV, ALTREF_FRAME, LAST_FRAME },
216 {NEWMV, GOLDEN_FRAME, ALTREF_FRAME},
217
218 {SPLITMV, LAST_FRAME, GOLDEN_FRAME},
219 {SPLITMV, ALTREF_FRAME, LAST_FRAME },
220 {SPLITMV, GOLDEN_FRAME, ALTREF_FRAME},
221
222 #if CONFIG_COMP_INTERINTRA_PRED
223 /* compound inter-intra prediction */
224 {ZEROMV, LAST_FRAME, INTRA_FRAME},
225 {NEARESTMV, LAST_FRAME, INTRA_FRAME},
226 {NEARMV, LAST_FRAME, INTRA_FRAME},
227 {NEWMV, LAST_FRAME, INTRA_FRAME},
228
229 {ZEROMV, GOLDEN_FRAME, INTRA_FRAME},
230 {NEARESTMV, GOLDEN_FRAME, INTRA_FRAME},
231 {NEARMV, GOLDEN_FRAME, INTRA_FRAME},
232 {NEWMV, GOLDEN_FRAME, INTRA_FRAME},
233
234 {ZEROMV, ALTREF_FRAME, INTRA_FRAME},
235 {NEARESTMV, ALTREF_FRAME, INTRA_FRAME},
236 {NEARMV, ALTREF_FRAME, INTRA_FRAME},
237 {NEWMV, ALTREF_FRAME, INTRA_FRAME},
238 #endif
239 };
240 #endif
241
242 static void fill_token_costs(
243 unsigned int (*c)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS],
244 const vp9_prob(*p)[COEF_BANDS][PREV_COEF_CONTEXTS][ENTROPY_NODES],
245 int block_type_counts) {
246 int i, j, k;
247
248 for (i = 0; i < block_type_counts; i++)
249 for (j = 0; j < COEF_BANDS; j++)
250 for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
251 if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
252 vp9_cost_tokens_skip((int *)(c[i][j][k]),
253 p[i][j][k],
254 vp9_coef_tree);
255 else
256 vp9_cost_tokens((int *)(c[i][j][k]),
257 p[i][j][k],
258 vp9_coef_tree);
259 }
260 }
261
262
263 static int rd_iifactor[32] = { 4, 4, 3, 2, 1, 0, 0, 0,
264 0, 0, 0, 0, 0, 0, 0, 0,
265 0, 0, 0, 0, 0, 0, 0, 0,
266 0, 0, 0, 0, 0, 0, 0, 0, };
267
268 // 3* dc_qlookup[Q]*dc_qlookup[Q];
269
270 /* values are now correlated to quantizer */
271 static int sad_per_bit16lut[QINDEX_RANGE];
272 static int sad_per_bit4lut[QINDEX_RANGE];
273
274 void vp9_init_me_luts() {
275 int i;
276
277 // Initialize the sad lut tables using a formulaic calculation for now
278 // This is to make it easier to resolve the impact of experimental changes
279 // to the quantizer tables.
280 for (i = 0; i < QINDEX_RANGE; i++) {
281 sad_per_bit16lut[i] =
282 (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
283 sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
284 }
285 }
286
287 static int compute_rd_mult(int qindex) {
288 int q;
289
290 q = vp9_dc_quant(qindex, 0);
291 return (11 * q * q) >> 6;
292 }
293
294 void vp9_initialize_me_consts(VP9_COMP *cpi, int QIndex) {
295 cpi->mb.sadperbit16 = sad_per_bit16lut[QIndex];
296 cpi->mb.sadperbit4 = sad_per_bit4lut[QIndex];
297 }
298
299
300 void vp9_initialize_rd_consts(VP9_COMP *cpi, int QIndex) {
301 int q, i;
302
303 vp9_clear_system_state(); // __asm emms;
304
305 // Further tests required to see if optimum is different
306 // for key frames, golden frames and arf frames.
307 // if (cpi->common.refresh_golden_frame ||
308 // cpi->common.refresh_alt_ref_frame)
309 QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);
310
311 cpi->RDMULT = compute_rd_mult(QIndex);
312
313 // Extend rate multiplier along side quantizer zbin increases
314 if (cpi->zbin_over_quant > 0) {
315 double oq_factor;
316
317 // Experimental code using the same basic equation as used for Q above
318 // The units of cpi->zbin_over_quant are 1/128 of Q bin size
319 oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
320 cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
321 }
322
323 if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
324 if (cpi->twopass.next_iiratio > 31)
325 cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
326 else
327 cpi->RDMULT +=
328 (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
329 }
330
331 if (cpi->RDMULT < 7)
332 cpi->RDMULT = 7;
333
334 cpi->mb.errorperbit = (cpi->RDMULT / 110);
335 cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
336
337 vp9_set_speed_features(cpi);
338
339 q = (int)pow(vp9_dc_quant(QIndex, 0) >> 2, 1.25);
340 q = q << 2;
341 cpi->RDMULT = cpi->RDMULT << 4;
342
343 if (q < 8)
344 q = 8;
345
346 if (cpi->RDMULT > 1000) {
347 cpi->RDDIV = 1;
348 cpi->RDMULT /= 100;
349
350 for (i = 0; i < MAX_MODES; i++) {
351 if (cpi->sf.thresh_mult[i] < INT_MAX) {
352 cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
353 } else {
354 cpi->rd_threshes[i] = INT_MAX;
355 }
356
357 cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
358 }
359 } else {
360 cpi->RDDIV = 100;
361
362 for (i = 0; i < MAX_MODES; i++) {
363 if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
364 cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
365 } else {
366 cpi->rd_threshes[i] = INT_MAX;
367 }
368
369 cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
370 }
371 }
372
373 fill_token_costs(
374 cpi->mb.token_costs[TX_4X4],
375 (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs,
376 BLOCK_TYPES);
377 fill_token_costs(
378 cpi->mb.hybrid_token_costs[TX_4X4],
379 (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
380 cpi->common.fc.hybrid_coef_probs,
381 BLOCK_TYPES);
382
383 fill_token_costs(
384 cpi->mb.token_costs[TX_8X8],
385 (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_8x 8,
386 BLOCK_TYPES_8X8);
387 fill_token_costs(
388 cpi->mb.hybrid_token_costs[TX_8X8],
389 (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
390 cpi->common.fc.hybrid_coef_probs_8x8,
391 BLOCK_TYPES_8X8);
392
393 fill_token_costs(
394 cpi->mb.token_costs[TX_16X16],
395 (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_16x 16,
396 BLOCK_TYPES_16X16);
397 fill_token_costs(
398 cpi->mb.hybrid_token_costs[TX_16X16],
399 (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11])
400 cpi->common.fc.hybrid_coef_probs_16x16,
401 BLOCK_TYPES_16X16);
402
403 /*rough estimate for costing*/
404 cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
405 vp9_init_mode_costs(cpi);
406
407 if (cpi->common.frame_type != KEY_FRAME)
408 {
409 vp9_build_nmv_cost_table(
410 cpi->mb.nmvjointcost,
411 cpi->mb.e_mbd.allow_high_precision_mv ?
412 cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
413 &cpi->common.fc.nmvc,
414 cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
415 }
416 }
417
418 int vp9_block_error_c(short *coeff, short *dqcoeff, int block_size) {
419 int i, error = 0;
420
421 for (i = 0; i < block_size; i++) {
422 int this_diff = coeff[i] - dqcoeff[i];
423 error += this_diff * this_diff;
424 }
425
426 return error;
427 }
428
429 int vp9_mbblock_error_8x8_c(MACROBLOCK *mb, int dc) {
430 BLOCK *be;
431 BLOCKD *bd;
432 int i, j;
433 int berror, error = 0;
434
435 for (i = 0; i < 16; i+=4) {
436 be = &mb->block[i];
437 bd = &mb->e_mbd.block[i];
438 berror = 0;
439 for (j = dc; j < 64; j++) {
440 int this_diff = be->coeff[j] - bd->dqcoeff[j];
441 berror += this_diff * this_diff;
442 }
443 error += berror;
444 }
445 return error;
446 }
447
448 int vp9_mbblock_error_c(MACROBLOCK *mb, int dc) {
449 BLOCK *be;
450 BLOCKD *bd;
451 int i, j;
452 int berror, error = 0;
453
454 for (i = 0; i < 16; i++) {
455 be = &mb->block[i];
456 bd = &mb->e_mbd.block[i];
457 berror = 0;
458 for (j = dc; j < 16; j++) {
459 int this_diff = be->coeff[j] - bd->dqcoeff[j];
460 berror += this_diff * this_diff;
461 }
462 error += berror;
463 }
464 return error;
465 }
466
467 int vp9_mbuverror_c(MACROBLOCK *mb) {
468 BLOCK *be;
469 BLOCKD *bd;
470
471 int i, error = 0;
472
473 for (i = 16; i < 24; i++) {
474 be = &mb->block[i];
475 bd = &mb->e_mbd.block[i];
476
477 error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
478 }
479
480 return error;
481 }
482
483 int vp9_uvsse(MACROBLOCK *x) {
484 unsigned char *uptr, *vptr;
485 unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
486 unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
487 int uv_stride = x->block[16].src_stride;
488
489 unsigned int sse1 = 0;
490 unsigned int sse2 = 0;
491 int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
492 int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
493 int offset;
494 int pre_stride = x->e_mbd.block[16].pre_stride;
495
496 if (mv_row < 0)
497 mv_row -= 1;
498 else
499 mv_row += 1;
500
501 if (mv_col < 0)
502 mv_col -= 1;
503 else
504 mv_col += 1;
505
506 mv_row /= 2;
507 mv_col /= 2;
508
509 offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
510 uptr = x->e_mbd.pre.u_buffer + offset;
511 vptr = x->e_mbd.pre.v_buffer + offset;
512
513 if ((mv_row | mv_col) & 7) {
514 vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
515 (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
516 vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
517 (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
518 sse2 += sse1;
519 } else {
520 vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
521 vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
522 sse2 += sse1;
523 }
524 return sse2;
525
526 }
527
528 static int cost_coeffs_2x2(MACROBLOCK *mb,
529 BLOCKD *b, PLANE_TYPE type,
530 ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) {
531 int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
532 int eob = b->eob;
533 int pt; /* surrounding block/prev coef predictor */
534 int cost = 0;
535 short *qcoeff_ptr = b->qcoeff;
536
537 VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
538 assert(eob <= 4);
539
540 for (; c < eob; c++) {
541 int v = qcoeff_ptr[vp9_default_zig_zag1d[c]];
542 int t = vp9_dct_value_tokens_ptr[v].Token;
543 cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]][pt][t];
544 cost += vp9_dct_value_cost_ptr[v];
545 pt = vp9_prev_token_class[t];
546 }
547
548 if (c < 4)
549 cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]]
550 [pt] [DCT_EOB_TOKEN];
551 // is eob first coefficient;
552 pt = (c > !type);
553 *a = *l = pt;
554 return cost;
555 }
556
557 static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, PLANE_TYPE type,
558 ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
559 int tx_size) {
560 const int eob = b->eob;
561 int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
562 int cost = 0, default_eob, seg_eob;
563 int pt; /* surrounding block/prev coef predictor */
564 int const *scan, *band;
565 short *qcoeff_ptr = b->qcoeff;
566 MACROBLOCKD *xd = &mb->e_mbd;
567 MB_MODE_INFO *mbmi = &mb->e_mbd.mode_info_context->mbmi;
568 TX_TYPE tx_type = DCT_DCT;
569 int segment_id = mbmi->segment_id;
570 scan = vp9_default_zig_zag1d;
571 band = vp9_coef_bands;
572 default_eob = 16;
573
574 switch (tx_size) {
575 case TX_4X4:
576 if (type == PLANE_TYPE_Y_WITH_DC) {
577 tx_type = get_tx_type_4x4(xd, b);
578 if (tx_type != DCT_DCT) {
579 switch (tx_type) {
580 case ADST_DCT:
581 scan = vp9_row_scan;
582 break;
583
584 case DCT_ADST:
585 scan = vp9_col_scan;
586 break;
587
588 default:
589 scan = vp9_default_zig_zag1d;
590 break;
591 }
592 }
593 }
594
595 break;
596 case TX_8X8:
597 scan = vp9_default_zig_zag1d_8x8;
598 band = vp9_coef_bands_8x8;
599 default_eob = 64;
600 if (type == PLANE_TYPE_Y_WITH_DC) {
601 BLOCKD *bb;
602 int ib = (int)(b - xd->block);
603 if (ib < 16) {
604 ib = (ib & 8) + ((ib & 4) >> 1);
605 bb = xd->block + ib;
606 tx_type = get_tx_type_8x8(xd, bb);
607 }
608 }
609 break;
610 case TX_16X16:
611 scan = vp9_default_zig_zag1d_16x16;
612 band = vp9_coef_bands_16x16;
613 default_eob = 256;
614 if (type == PLANE_TYPE_Y_WITH_DC) {
615 tx_type = get_tx_type_16x16(xd, b);
616 }
617 break;
618 default:
619 break;
620 }
621 if (vp9_segfeature_active(&mb->e_mbd, segment_id, SEG_LVL_EOB))
622 seg_eob = vp9_get_segdata(&mb->e_mbd, segment_id, SEG_LVL_EOB);
623 else
624 seg_eob = default_eob;
625
626 VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
627
628 if (tx_type != DCT_DCT) {
629 for (; c < eob; c++) {
630 int v = qcoeff_ptr[scan[c]];
631 int t = vp9_dct_value_tokens_ptr[v].Token;
632 cost += mb->hybrid_token_costs[tx_size][type][band[c]][pt][t];
633 cost += vp9_dct_value_cost_ptr[v];
634 pt = vp9_prev_token_class[t];
635 }
636 if (c < seg_eob)
637 cost += mb->hybrid_token_costs[tx_size][type][band[c]]
638 [pt][DCT_EOB_TOKEN];
639 } else {
640 for (; c < eob; c++) {
641 int v = qcoeff_ptr[scan[c]];
642 int t = vp9_dct_value_tokens_ptr[v].Token;
643 cost += mb->token_costs[tx_size][type][band[c]][pt][t];
644 cost += vp9_dct_value_cost_ptr[v];
645 pt = vp9_prev_token_class[t];
646 }
647 if (c < seg_eob)
648 cost += mb->token_costs[tx_size][type][band[c]]
649 [pt][DCT_EOB_TOKEN];
650 }
651
652 // is eob first coefficient;
653 pt = (c > !type);
654 *a = *l = pt;
655 return cost;
656 }
657
658 static int rdcost_mby_4x4(MACROBLOCK *mb, int has_2nd_order, int backup) {
659 int cost = 0;
660 int b;
661 MACROBLOCKD *xd = &mb->e_mbd;
662 ENTROPY_CONTEXT_PLANES t_above, t_left;
663 ENTROPY_CONTEXT *ta;
664 ENTROPY_CONTEXT *tl;
665
666 if (backup) {
667 vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
668 vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
669
670 ta = (ENTROPY_CONTEXT *)&t_above;
671 tl = (ENTROPY_CONTEXT *)&t_left;
672 } else {
673 ta = (ENTROPY_CONTEXT *)xd->above_context;
674 tl = (ENTROPY_CONTEXT *)xd->left_context;
675 }
676
677 for (b = 0; b < 16; b++)
678 cost += cost_coeffs(mb, xd->block + b,
679 (has_2nd_order ?
680 PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC),
681 ta + vp9_block2above[b], tl + vp9_block2left[b],
682 TX_4X4);
683
684 if (has_2nd_order)
685 cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
686 ta + vp9_block2above[24], tl + vp9_block2left[24],
687 TX_4X4);
688
689 return cost;
690 }
691
692 static void macro_block_yrd_4x4(MACROBLOCK *mb,
693 int *Rate,
694 int *Distortion,
695 int *skippable, int backup) {
696 MACROBLOCKD *const xd = &mb->e_mbd;
697 BLOCK *const mb_y2 = mb->block + 24;
698 BLOCKD *const x_y2 = xd->block + 24;
699 int d, has_2nd_order;
700
701 xd->mode_info_context->mbmi.txfm_size = TX_4X4;
702 has_2nd_order = get_2nd_order_usage(xd);
703 // Fdct and building the 2nd order block
704 vp9_transform_mby_4x4(mb);
705 vp9_quantize_mby_4x4(mb);
706 d = vp9_mbblock_error(mb, has_2nd_order);
707 if (has_2nd_order)
708 d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
709
710 *Distortion = (d >> 2);
711 // rate
712 *Rate = rdcost_mby_4x4(mb, has_2nd_order, backup);
713 *skippable = vp9_mby_is_skippable_4x4(&mb->e_mbd, has_2nd_order);
714 }
715
716 static int rdcost_mby_8x8(MACROBLOCK *mb, int has_2nd_order, int backup) {
717 int cost = 0;
718 int b;
719 MACROBLOCKD *xd = &mb->e_mbd;
720 ENTROPY_CONTEXT_PLANES t_above, t_left;
721 ENTROPY_CONTEXT *ta;
722 ENTROPY_CONTEXT *tl;
723
724 if (backup) {
725 vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
726 vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
727
728 ta = (ENTROPY_CONTEXT *)&t_above;
729 tl = (ENTROPY_CONTEXT *)&t_left;
730 } else {
731 ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
732 tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
733 }
734
735 for (b = 0; b < 16; b += 4)
736 cost += cost_coeffs(mb, xd->block + b,
737 (has_2nd_order ?
738 PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC),
739 ta + vp9_block2above_8x8[b], tl + vp9_block2left_8x8[b],
740 TX_8X8);
741
742 if (has_2nd_order)
743 cost += cost_coeffs_2x2(mb, xd->block + 24, PLANE_TYPE_Y2,
744 ta + vp9_block2above[24], tl + vp9_block2left[24]);
745 return cost;
746 }
747
748 static void macro_block_yrd_8x8(MACROBLOCK *mb,
749 int *Rate,
750 int *Distortion,
751 int *skippable, int backup) {
752 MACROBLOCKD *const xd = &mb->e_mbd;
753 BLOCK *const mb_y2 = mb->block + 24;
754 BLOCKD *const x_y2 = xd->block + 24;
755 int d, has_2nd_order;
756
757 xd->mode_info_context->mbmi.txfm_size = TX_8X8;
758
759 vp9_transform_mby_8x8(mb);
760 vp9_quantize_mby_8x8(mb);
761 has_2nd_order = get_2nd_order_usage(xd);
762 d = vp9_mbblock_error_8x8_c(mb, has_2nd_order);
763 if (has_2nd_order)
764 d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
765
766 *Distortion = (d >> 2);
767 // rate
768 *Rate = rdcost_mby_8x8(mb, has_2nd_order, backup);
769 *skippable = vp9_mby_is_skippable_8x8(&mb->e_mbd, has_2nd_order);
770 }
771
772 static int rdcost_mby_16x16(MACROBLOCK *mb, int backup) {
773 int cost;
774 MACROBLOCKD *xd = &mb->e_mbd;
775 ENTROPY_CONTEXT_PLANES t_above, t_left;
776 ENTROPY_CONTEXT *ta, *tl;
777
778 if (backup) {
779 vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
780 vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
781
782 ta = (ENTROPY_CONTEXT *)&t_above;
783 tl = (ENTROPY_CONTEXT *)&t_left;
784 } else {
785 ta = (ENTROPY_CONTEXT *)xd->above_context;
786 tl = (ENTROPY_CONTEXT *)xd->left_context;
787 }
788
789 cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
790 return cost;
791 }
792
793 static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
794 int *skippable, int backup) {
795 int d;
796 MACROBLOCKD *xd = &mb->e_mbd;
797
798 xd->mode_info_context->mbmi.txfm_size = TX_16X16;
799 vp9_transform_mby_16x16(mb);
800 vp9_quantize_mby_16x16(mb);
801 // TODO(jingning) is it possible to quickly determine whether to force
802 // trailing coefficients to be zero, instead of running trellis
803 // optimization in the rate-distortion optimization loop?
804 if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
805 vp9_optimize_mby_16x16(mb);
806
807 d = vp9_mbblock_error(mb, 0);
808
809 *Distortion = (d >> 2);
810 // rate
811 *Rate = rdcost_mby_16x16(mb, backup);
812 *skippable = vp9_mby_is_skippable_16x16(&mb->e_mbd);
813 }
814
815 static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
816 int r[2][TX_SIZE_MAX], int *rate,
817 int d[TX_SIZE_MAX], int *distortion,
818 int s[TX_SIZE_MAX], int *skip,
819 int64_t txfm_cache[NB_TXFM_MODES]) {
820 VP9_COMMON *const cm = &cpi->common;
821 MACROBLOCKD *const xd = &x->e_mbd;
822 MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
823 vp9_prob skip_prob = cm->mb_no_coeff_skip ?
824 vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
825 int64_t rd[2][TX_SIZE_MAX];
826 int n;
827
828 r[1][TX_16X16] = r[0][TX_16X16] + vp9_cost_one(cm->prob_tx[0]) +
829 vp9_cost_one(cm->prob_tx[1]);
830 r[1][TX_8X8] = r[0][TX_8X8] + vp9_cost_one(cm->prob_tx[0]) +
831 vp9_cost_zero(cm->prob_tx[1]);
832 r[1][TX_4X4] = r[0][TX_4X4] + vp9_cost_zero(cm->prob_tx[0]);
833
834 if (cm->mb_no_coeff_skip) {
835 int s0, s1;
836
837 assert(skip_prob > 0);
838 s0 = vp9_cost_bit(skip_prob, 0);
839 s1 = vp9_cost_bit(skip_prob, 1);
840
841 for (n = TX_4X4; n <= TX_16X16; n++) {
842 if (s[n]) {
843 rd[0][n] = rd[1][n] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
844 } else {
845 rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n] + s0, d[n]);
846 rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n] + s0, d[n]);
847 }
848 }
849 } else {
850 for (n = TX_4X4; n <= TX_16X16; n++) {
851 rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n], d[n]);
852 rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n], d[n]);
853 }
854 }
855
856 if ( cm->txfm_mode == ALLOW_16X16 ||
857 (cm->txfm_mode == TX_MODE_SELECT &&
858 rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])) {
859 mbmi->txfm_size = TX_16X16;
860 } else if (cm->txfm_mode == ALLOW_8X8 ||
861 (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_8X8] < rd[1][TX_4X4])) {
862 mbmi->txfm_size = TX_8X8;
863 } else {
864 assert(cm->txfm_mode == ONLY_4X4 ||
865 (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_4X4] <= rd[1][TX_8X8]));
866 mbmi->txfm_size = TX_4X4;
867 }
868
869 *distortion = d[mbmi->txfm_size];
870 *rate = r[cm->txfm_mode == TX_MODE_SELECT][mbmi->txfm_size];
871 *skip = s[mbmi->txfm_size];
872
873 txfm_cache[ONLY_4X4] = rd[0][TX_4X4];
874 txfm_cache[ALLOW_8X8] = rd[0][TX_8X8];
875 txfm_cache[ALLOW_16X16] = rd[0][TX_16X16];
876 if (rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])
877 txfm_cache[TX_MODE_SELECT] = rd[1][TX_16X16];
878 else
879 txfm_cache[TX_MODE_SELECT] = rd[1][TX_4X4] < rd[1][TX_8X8] ?
880 rd[1][TX_4X4] : rd[1][TX_8X8];
881 }
882
883 static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
884 int *distortion, int *skippable,
885 int64_t txfm_cache[NB_TXFM_MODES]) {
886 MACROBLOCKD *const xd = &x->e_mbd;
887 int r[2][TX_SIZE_MAX], d[TX_SIZE_MAX], s[TX_SIZE_MAX];
888
889 vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
890 x->block[0].src_stride);
891
892 macro_block_yrd_16x16(x, &r[0][TX_16X16], &d[TX_16X16],
893 &s[TX_16X16], 1);
894 macro_block_yrd_8x8(x, &r[0][TX_8X8], &d[TX_8X8], &s[TX_8X8], 1);
895 macro_block_yrd_4x4(x, &r[0][TX_4X4], &d[TX_4X4], &s[TX_4X4], 1);
896
897 choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
898 txfm_cache);
899 }
900
901 static void copy_predictor(unsigned char *dst, const unsigned char *predictor) {
902 const unsigned int *p = (const unsigned int *)predictor;
903 unsigned int *d = (unsigned int *)dst;
904 d[0] = p[0];
905 d[4] = p[4];
906 d[8] = p[8];
907 d[12] = p[12];
908 }
909
910 #if CONFIG_SUPERBLOCKS
911 static void super_block_yrd(VP9_COMP *cpi,
912 MACROBLOCK *x, int *rate, int *distortion,
913 int *skip,
914 int64_t txfm_cache[NB_TXFM_MODES]) {
915 MACROBLOCKD *const xd = &x->e_mbd;
916 int r[2][TX_SIZE_MAX], d[TX_SIZE_MAX], s[TX_SIZE_MAX], n;
917 const uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
918 int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;
919 ENTROPY_CONTEXT_PLANES t_above[3][2], *orig_above = xd->above_context;
920 ENTROPY_CONTEXT_PLANES t_left[3][2], *orig_left = xd->left_context;
921
922 for (n = TX_4X4; n <= TX_16X16; n++) {
923 vpx_memcpy(t_above[n], xd->above_context, sizeof(t_above[n]));
924 vpx_memcpy(t_left[n], xd->left_context, sizeof(t_left[n]));
925 r[0][n] = 0;
926 d[n] = 0;
927 s[n] = 1;
928 }
929
930 for (n = 0; n < 4; n++) {
931 int x_idx = n & 1, y_idx = n >> 1;
932 int r_tmp, d_tmp, s_tmp;
933
934 vp9_subtract_mby_s_c(x->src_diff,
935 src + x_idx * 16 + y_idx * 16 * src_y_stride,
936 src_y_stride,
937 dst + x_idx * 16 + y_idx * 16 * dst_y_stride,
938 dst_y_stride);
939
940 xd->above_context = &t_above[TX_16X16][x_idx];
941 xd->left_context = &t_left[TX_16X16][y_idx];
942 macro_block_yrd_16x16(x, &r_tmp, &d_tmp, &s_tmp, 0);
943 d[TX_16X16] += d_tmp;
944 r[0][TX_16X16] += r_tmp;
945 s[TX_16X16] = s[TX_16X16] && s_tmp;
946
947 xd->above_context = &t_above[TX_4X4][x_idx];
948 xd->left_context = &t_left[TX_4X4][y_idx];
949 macro_block_yrd_4x4(x, &r_tmp, &d_tmp, &s_tmp, 0);
950 d[TX_4X4] += d_tmp;
951 r[0][TX_4X4] += r_tmp;
952 s[TX_4X4] = s[TX_4X4] && s_tmp;
953
954 xd->above_context = &t_above[TX_8X8][x_idx];
955 xd->left_context = &t_left[TX_8X8][y_idx];
956 macro_block_yrd_8x8(x, &r_tmp, &d_tmp, &s_tmp, 0);
957 d[TX_8X8] += d_tmp;
958 r[0][TX_8X8] += r_tmp;
959 s[TX_8X8] = s[TX_8X8] && s_tmp;
960 }
961
962 choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skip, txfm_cache);
963
964 xd->above_context = orig_above;
965 xd->left_context = orig_left;
966 }
967 #endif
968
969 static void copy_predictor_8x8(unsigned char *dst, const unsigned char *predicto r) {
970 const unsigned int *p = (const unsigned int *)predictor;
971 unsigned int *d = (unsigned int *)dst;
972 d[0] = p[0];
973 d[1] = p[1];
974 d[4] = p[4];
975 d[5] = p[5];
976 d[8] = p[8];
977 d[9] = p[9];
978 d[12] = p[12];
979 d[13] = p[13];
980 d[16] = p[16];
981 d[17] = p[17];
982 d[20] = p[20];
983 d[21] = p[21];
984 d[24] = p[24];
985 d[25] = p[25];
986 d[28] = p[28];
987 d[29] = p[29];
988 }
989
990 static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, BLOCK *be,
991 BLOCKD *b, B_PREDICTION_MODE *best_mode,
992 #if CONFIG_COMP_INTRA_PRED
993 B_PREDICTION_MODE *best_second_mode,
994 int allow_comp,
995 #endif
996 int *bmode_costs,
997 ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
998 int *bestrate, int *bestratey,
999 int *bestdistortion) {
1000 B_PREDICTION_MODE mode;
1001 MACROBLOCKD *xd = &x->e_mbd;
1002
1003 #if CONFIG_COMP_INTRA_PRED
1004 B_PREDICTION_MODE mode2;
1005 #endif
1006 int64_t best_rd = INT64_MAX;
1007 int rate = 0;
1008 int distortion;
1009
1010 ENTROPY_CONTEXT ta = *a, tempa = *a;
1011 ENTROPY_CONTEXT tl = *l, templ = *l;
1012 TX_TYPE tx_type = DCT_DCT;
1013 TX_TYPE best_tx_type = DCT_DCT;
1014 /*
1015 * The predictor buffer is a 2d buffer with a stride of 16. Create
1016 * a temp buffer that meets the stride requirements, but we are only
1017 * interested in the left 4x4 block
1018 * */
1019 DECLARE_ALIGNED_ARRAY(16, unsigned char, best_predictor, 16 * 4);
1020 DECLARE_ALIGNED_ARRAY(16, short, best_dqcoeff, 16);
1021
1022 #if CONFIG_NEWBINTRAMODES
1023 b->bmi.as_mode.context = vp9_find_bpred_context(b);
1024 #endif
1025 for (mode = B_DC_PRED; mode < LEFT4X4; mode++) {
1026 #if CONFIG_COMP_INTRA_PRED
1027 for (mode2 = (allow_comp ? 0 : (B_DC_PRED - 1));
1028 mode2 != (allow_comp ? (mode + 1) : 0); mode2++) {
1029 #endif
1030 int64_t this_rd;
1031 int ratey;
1032
1033 #if CONFIG_NEWBINTRAMODES
1034 if (xd->frame_type == KEY_FRAME) {
1035 if (mode == B_CONTEXT_PRED) continue;
1036 #if CONFIG_COMP_INTRA_PRED
1037 if (mode2 == B_CONTEXT_PRED) continue;
1038 #endif
1039 } else {
1040 if (mode >= B_CONTEXT_PRED - CONTEXT_PRED_REPLACEMENTS &&
1041 mode < B_CONTEXT_PRED)
1042 continue;
1043 #if CONFIG_COMP_INTRA_PRED
1044 if (mode2 >= B_CONTEXT_PRED - CONTEXT_PRED_REPLACEMENTS &&
1045 mode2 < B_CONTEXT_PRED)
1046 continue;
1047 #endif
1048 }
1049 #endif
1050
1051 b->bmi.as_mode.first = mode;
1052 #if CONFIG_NEWBINTRAMODES
1053 rate = bmode_costs[
1054 mode == B_CONTEXT_PRED ? mode - CONTEXT_PRED_REPLACEMENTS : mode];
1055 #else
1056 rate = bmode_costs[mode];
1057 #endif
1058
1059 #if CONFIG_COMP_INTRA_PRED
1060 if (mode2 == (B_PREDICTION_MODE)(B_DC_PRED - 1)) {
1061 #endif
1062 vp9_intra4x4_predict(b, mode, b->predictor);
1063 #if CONFIG_COMP_INTRA_PRED
1064 } else {
1065 vp9_comp_intra4x4_predict(b, mode, mode2, b->predictor);
1066 #if CONFIG_NEWBINTRAMODES
1067 rate += bmode_costs[
1068 mode2 == B_CONTEXT_PRED ?
1069 mode2 - CONTEXT_PRED_REPLACEMENTS : mode2];
1070 #else
1071 rate += bmode_costs[mode2];
1072 #endif
1073 }
1074 #endif
1075 vp9_subtract_b(be, b, 16);
1076
1077 b->bmi.as_mode.first = mode;
1078 tx_type = get_tx_type_4x4(xd, b);
1079 if (tx_type != DCT_DCT) {
1080 vp9_fht(be->src_diff, 32, be->coeff, tx_type, 4);
1081 vp9_ht_quantize_b_4x4(be, b, tx_type);
1082 } else {
1083 x->vp9_short_fdct4x4(be->src_diff, be->coeff, 32);
1084 x->quantize_b_4x4(be, b);
1085 }
1086
1087 tempa = ta;
1088 templ = tl;
1089
1090 ratey = cost_coeffs(x, b, PLANE_TYPE_Y_WITH_DC, &tempa, &templ, TX_4X4);
1091 rate += ratey;
1092 distortion = vp9_block_error(be->coeff, b->dqcoeff, 16) >> 2;
1093
1094 this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
1095
1096 if (this_rd < best_rd) {
1097 *bestrate = rate;
1098 *bestratey = ratey;
1099 *bestdistortion = distortion;
1100 best_rd = this_rd;
1101 *best_mode = mode;
1102 best_tx_type = tx_type;
1103
1104 #if CONFIG_COMP_INTRA_PRED
1105 *best_second_mode = mode2;
1106 #endif
1107 *a = tempa;
1108 *l = templ;
1109 copy_predictor(best_predictor, b->predictor);
1110 vpx_memcpy(best_dqcoeff, b->dqcoeff, 32);
1111 }
1112 #if CONFIG_COMP_INTRA_PRED
1113 }
1114 #endif
1115 }
1116 b->bmi.as_mode.first = (B_PREDICTION_MODE)(*best_mode);
1117 #if CONFIG_COMP_INTRA_PRED
1118 b->bmi.as_mode.second = (B_PREDICTION_MODE)(*best_second_mode);
1119 #endif
1120
1121 // inverse transform
1122 if (best_tx_type != DCT_DCT)
1123 vp9_ihtllm(best_dqcoeff, b->diff, 32, best_tx_type, 4);
1124 else
1125 xd->inv_xform4x4_x8(best_dqcoeff, b->diff, 32);
1126
1127 vp9_recon_b(best_predictor, b->diff, *(b->base_dst) + b->dst, b->dst_stride);
1128
1129 return best_rd;
1130 }
1131
1132 static int64_t rd_pick_intra4x4mby_modes(VP9_COMP *cpi, MACROBLOCK *mb, int *Rat e,
1133 int *rate_y, int *Distortion, int64_t best_ rd,
1134 #if CONFIG_COMP_INTRA_PRED
1135 int allow_comp,
1136 #endif
1137 int update_contexts) {
1138 int i;
1139 MACROBLOCKD *const xd = &mb->e_mbd;
1140 int cost = mb->mbmode_cost [xd->frame_type] [B_PRED];
1141 int distortion = 0;
1142 int tot_rate_y = 0;
1143 int64_t total_rd = 0;
1144 ENTROPY_CONTEXT_PLANES t_above, t_left;
1145 ENTROPY_CONTEXT *ta, *tl;
1146 int *bmode_costs;
1147
1148 if (update_contexts) {
1149 ta = (ENTROPY_CONTEXT *)xd->above_context;
1150 tl = (ENTROPY_CONTEXT *)xd->left_context;
1151 } else {
1152 vpx_memcpy(&t_above, xd->above_context,
1153 sizeof(ENTROPY_CONTEXT_PLANES));
1154 vpx_memcpy(&t_left, xd->left_context,
1155 sizeof(ENTROPY_CONTEXT_PLANES));
1156
1157 ta = (ENTROPY_CONTEXT *)&t_above;
1158 tl = (ENTROPY_CONTEXT *)&t_left;
1159 }
1160
1161 xd->mode_info_context->mbmi.mode = B_PRED;
1162 bmode_costs = mb->inter_bmode_costs;
1163
1164 for (i = 0; i < 16; i++) {
1165 MODE_INFO *const mic = xd->mode_info_context;
1166 const int mis = xd->mode_info_stride;
1167 B_PREDICTION_MODE UNINITIALIZED_IS_SAFE(best_mode);
1168 #if CONFIG_COMP_INTRA_PRED
1169 B_PREDICTION_MODE UNINITIALIZED_IS_SAFE(best_second_mode);
1170 #endif
1171 int UNINITIALIZED_IS_SAFE(r), UNINITIALIZED_IS_SAFE(ry), UNINITIALIZED_IS_SA FE(d);
1172
1173 if (xd->frame_type == KEY_FRAME) {
1174 const B_PREDICTION_MODE A = above_block_mode(mic, i, mis);
1175 const B_PREDICTION_MODE L = left_block_mode(mic, i);
1176
1177 bmode_costs = mb->bmode_costs[A][L];
1178 }
1179 #if CONFIG_NEWBINTRAMODES
1180 mic->bmi[i].as_mode.context = vp9_find_bpred_context(xd->block + i);
1181 #endif
1182
1183 total_rd += rd_pick_intra4x4block(
1184 cpi, mb, mb->block + i, xd->block + i, &best_mode,
1185 #if CONFIG_COMP_INTRA_PRED
1186 & best_second_mode, allow_comp,
1187 #endif
1188 bmode_costs, ta + vp9_block2above[i],
1189 tl + vp9_block2left[i], &r, &ry, &d);
1190
1191 cost += r;
1192 distortion += d;
1193 tot_rate_y += ry;
1194
1195 mic->bmi[i].as_mode.first = best_mode;
1196 #if CONFIG_COMP_INTRA_PRED
1197 mic->bmi[i].as_mode.second = best_second_mode;
1198 #endif
1199
1200 #if 0 // CONFIG_NEWBINTRAMODES
1201 printf("%d %d\n", mic->bmi[i].as_mode.first, mic->bmi[i].as_mode.context);
1202 #endif
1203
1204 if (total_rd >= best_rd)
1205 break;
1206 }
1207
1208 if (total_rd >= best_rd)
1209 return INT64_MAX;
1210
1211 #if CONFIG_COMP_INTRA_PRED
1212 cost += vp9_cost_bit(128, allow_comp);
1213 #endif
1214 *Rate = cost;
1215 *rate_y = tot_rate_y;
1216 *Distortion = distortion;
1217
1218 return RDCOST(mb->rdmult, mb->rddiv, cost, distortion);
1219 }
1220
1221 #if CONFIG_SUPERBLOCKS
1222 static int64_t rd_pick_intra_sby_mode(VP9_COMP *cpi,
1223 MACROBLOCK *x,
1224 int *rate,
1225 int *rate_tokenonly,
1226 int *distortion,
1227 int *skippable,
1228 int64_t txfm_cache[NB_TXFM_MODES]) {
1229 MB_PREDICTION_MODE mode;
1230 MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
1231 int this_rate, this_rate_tokenonly;
1232 int this_distortion, s;
1233 int64_t best_rd = INT64_MAX, this_rd;
1234
1235 /* Y Search for 32x32 intra prediction mode */
1236 for (mode = DC_PRED; mode <= TM_PRED; mode++) {
1237 x->e_mbd.mode_info_context->mbmi.mode = mode;
1238 vp9_build_intra_predictors_sby_s(&x->e_mbd);
1239
1240 super_block_yrd(cpi, x, &this_rate_tokenonly,
1241 &this_distortion, &s, txfm_cache);
1242 this_rate = this_rate_tokenonly +
1243 x->mbmode_cost[x->e_mbd.frame_type]
1244 [x->e_mbd.mode_info_context->mbmi.mode];
1245 this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, this_distortion);
1246
1247 if (this_rd < best_rd) {
1248 mode_selected = mode;
1249 best_rd = this_rd;
1250 *rate = this_rate;
1251 *rate_tokenonly = this_rate_tokenonly;
1252 *distortion = this_distortion;
1253 *skippable = s;
1254 }
1255 }
1256
1257 x->e_mbd.mode_info_context->mbmi.mode = mode_selected;
1258
1259 return best_rd;
1260 }
1261 #endif
1262
1263 static int64_t rd_pick_intra16x16mby_mode(VP9_COMP *cpi,
1264 MACROBLOCK *x,
1265 int *Rate,
1266 int *rate_y,
1267 int *Distortion,
1268 int *skippable,
1269 int64_t txfm_cache[NB_TXFM_MODES]) {
1270 MB_PREDICTION_MODE mode;
1271 TX_SIZE txfm_size;
1272 MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
1273 #if CONFIG_COMP_INTRA_PRED
1274 MB_PREDICTION_MODE mode2;
1275 MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode2_selected);
1276 #endif
1277 MACROBLOCKD *const xd = &x->e_mbd;
1278 MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
1279 int rate, ratey;
1280 int distortion, skip;
1281 int64_t best_rd = INT64_MAX;
1282 int64_t this_rd;
1283
1284 int i;
1285 for (i = 0; i < NB_TXFM_MODES; i++)
1286 txfm_cache[i] = INT64_MAX;
1287
1288 // Y Search for 16x16 intra prediction mode
1289 for (mode = DC_PRED; mode <= TM_PRED; mode++) {
1290 int64_t local_txfm_cache[NB_TXFM_MODES];
1291
1292 mbmi->mode = mode;
1293
1294 #if CONFIG_COMP_INTRA_PRED
1295 for (mode2 = DC_PRED - 1; mode2 != TM_PRED + 1; mode2++) {
1296 mbmi->second_mode = mode2;
1297 if (mode2 == (MB_PREDICTION_MODE)(DC_PRED - 1)) {
1298 #endif
1299 vp9_build_intra_predictors_mby(xd);
1300 #if CONFIG_COMP_INTRA_PRED
1301 } else {
1302 continue; // i.e. disable for now
1303 vp9_build_comp_intra_predictors_mby(xd);
1304 }
1305 #endif
1306
1307 macro_block_yrd(cpi, x, &ratey, &distortion, &skip, local_txfm_cache);
1308
1309 // FIXME add compoundmode cost
1310 // FIXME add rate for mode2
1311 rate = ratey + x->mbmode_cost[xd->frame_type][mbmi->mode];
1312
1313 this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
1314
1315
1316 if (this_rd < best_rd) {
1317 mode_selected = mode;
1318 txfm_size = mbmi->txfm_size;
1319 #if CONFIG_COMP_INTRA_PRED
1320 mode2_selected = mode2;
1321 #endif
1322 best_rd = this_rd;
1323 *Rate = rate;
1324 *rate_y = ratey;
1325 *Distortion = distortion;
1326 *skippable = skip;
1327 }
1328
1329 for (i = 0; i < NB_TXFM_MODES; i++) {
1330 int64_t adj_rd = this_rd + local_txfm_cache[i] -
1331 local_txfm_cache[cpi->common.txfm_mode];
1332 if (adj_rd < txfm_cache[i]) {
1333 txfm_cache[i] = adj_rd;
1334 }
1335 }
1336
1337 #if CONFIG_COMP_INTRA_PRED
1338 }
1339 #endif
1340 }
1341
1342 mbmi->txfm_size = txfm_size;
1343 mbmi->mode = mode_selected;
1344
1345 #if CONFIG_COMP_INTRA_PRED
1346 mbmi->second_mode = mode2_selected;
1347 #endif
1348 return best_rd;
1349 }
1350
1351
1352 static int64_t rd_pick_intra8x8block(VP9_COMP *cpi, MACROBLOCK *x, int ib,
1353 B_PREDICTION_MODE *best_mode,
1354 #if CONFIG_COMP_INTRA_PRED
1355 B_PREDICTION_MODE *best_second_mode,
1356 #endif
1357 int *mode_costs,
1358 ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
1359 int *bestrate, int *bestratey,
1360 int *bestdistortion) {
1361 MB_PREDICTION_MODE mode;
1362 #if CONFIG_COMP_INTRA_PRED
1363 MB_PREDICTION_MODE mode2;
1364 #endif
1365 MACROBLOCKD *xd = &x->e_mbd;
1366 int64_t best_rd = INT64_MAX;
1367 int distortion = 0, rate = 0;
1368 BLOCK *be = x->block + ib;
1369 BLOCKD *b = xd->block + ib;
1370 ENTROPY_CONTEXT ta0, ta1, besta0 = 0, besta1 = 0;
1371 ENTROPY_CONTEXT tl0, tl1, bestl0 = 0, bestl1 = 0;
1372
1373 /*
1374 * The predictor buffer is a 2d buffer with a stride of 16. Create
1375 * a temp buffer that meets the stride requirements, but we are only
1376 * interested in the left 8x8 block
1377 * */
1378 DECLARE_ALIGNED_ARRAY(16, unsigned char, best_predictor, 16 * 8);
1379 DECLARE_ALIGNED_ARRAY(16, short, best_dqcoeff, 16 * 4);
1380
1381 // perform transformation of dimension 8x8
1382 // note the input and output index mapping
1383 int idx = (ib & 0x02) ? (ib + 2) : ib;
1384
1385 for (mode = DC_PRED; mode <= TM_PRED; mode++) {
1386 #if CONFIG_COMP_INTRA_PRED
1387 for (mode2 = DC_PRED - 1; mode2 != TM_PRED + 1; mode2++) {
1388 #endif
1389 int64_t this_rd;
1390 int rate_t = 0;
1391
1392 // FIXME rate for compound mode and second intrapred mode
1393 rate = mode_costs[mode];
1394 b->bmi.as_mode.first = mode;
1395
1396 #if CONFIG_COMP_INTRA_PRED
1397 if (mode2 == (MB_PREDICTION_MODE)(DC_PRED - 1)) {
1398 #endif
1399 vp9_intra8x8_predict(b, mode, b->predictor);
1400 #if CONFIG_COMP_INTRA_PRED
1401 } else {
1402 continue; // i.e. disable for now
1403 vp9_comp_intra8x8_predict(b, mode, mode2, b->predictor);
1404 }
1405 #endif
1406
1407 vp9_subtract_4b_c(be, b, 16);
1408
1409 assert(get_2nd_order_usage(xd) == 0);
1410 if (xd->mode_info_context->mbmi.txfm_size == TX_8X8) {
1411 TX_TYPE tx_type = get_tx_type_8x8(xd, b);
1412 if (tx_type != DCT_DCT)
1413 vp9_fht(be->src_diff, 32, (x->block + idx)->coeff, tx_type, 8);
1414 else
1415 x->vp9_short_fdct8x8(be->src_diff, (x->block + idx)->coeff, 32);
1416 x->quantize_b_8x8(x->block + idx, xd->block + idx);
1417
1418 // compute quantization mse of 8x8 block
1419 distortion = vp9_block_error_c((x->block + idx)->coeff,
1420 (xd->block + idx)->dqcoeff, 64);
1421 ta0 = a[vp9_block2above_8x8[idx]];
1422 tl0 = l[vp9_block2left_8x8[idx]];
1423
1424 rate_t = cost_coeffs(x, xd->block + idx, PLANE_TYPE_Y_WITH_DC,
1425 &ta0, &tl0, TX_8X8);
1426
1427 rate += rate_t;
1428 ta1 = ta0;
1429 tl1 = tl0;
1430 } else {
1431 static const int iblock[4] = {0, 1, 4, 5};
1432 TX_TYPE tx_type;
1433 int i;
1434 ta0 = a[vp9_block2above[ib]];
1435 ta1 = a[vp9_block2above[ib + 1]];
1436 tl0 = l[vp9_block2left[ib]];
1437 tl1 = l[vp9_block2left[ib + 4]];
1438 distortion = 0;
1439 rate_t = 0;
1440 for (i = 0; i < 4; ++i) {
1441 b = &xd->block[ib + iblock[i]];
1442 be = &x->block[ib + iblock[i]];
1443 tx_type = get_tx_type_4x4(xd, b);
1444 if (tx_type != DCT_DCT) {
1445 vp9_fht_c(be->src_diff, 32, be->coeff, tx_type, 4);
1446 vp9_ht_quantize_b_4x4(be, b, tx_type);
1447 } else {
1448 x->vp9_short_fdct4x4(be->src_diff, be->coeff, 32);
1449 x->quantize_b_4x4(be, b);
1450 }
1451 distortion += vp9_block_error_c(be->coeff, b->dqcoeff, 16);
1452 rate_t += cost_coeffs(x, b, PLANE_TYPE_Y_WITH_DC,
1453 // i&1 ? &ta1 : &ta0, i&2 ? &tl1 : &tl0,
1454 &ta0, &tl0,
1455 TX_4X4);
1456 }
1457 rate += rate_t;
1458 }
1459
1460 distortion >>= 2;
1461 this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
1462 if (this_rd < best_rd) {
1463 *bestrate = rate;
1464 *bestratey = rate_t;
1465 *bestdistortion = distortion;
1466 besta0 = ta0;
1467 besta1 = ta1;
1468 bestl0 = tl0;
1469 bestl1 = tl1;
1470 best_rd = this_rd;
1471 *best_mode = mode;
1472 #if CONFIG_COMP_INTRA_PRED
1473 *best_second_mode = mode2;
1474 #endif
1475 copy_predictor_8x8(best_predictor, b->predictor);
1476 vpx_memcpy(best_dqcoeff, b->dqcoeff, 64);
1477 vpx_memcpy(best_dqcoeff + 32, b->dqcoeff + 64, 64);
1478 #if CONFIG_COMP_INTRA_PRED
1479 }
1480 #endif
1481 }
1482 }
1483 b->bmi.as_mode.first = (*best_mode);
1484 #if CONFIG_COMP_INTRA_PRED
1485 b->bmi.as_mode.second = (*best_second_mode);
1486 #endif
1487 vp9_encode_intra8x8(x, ib);
1488
1489 if (xd->mode_info_context->mbmi.txfm_size == TX_8X8) {
1490 a[vp9_block2above_8x8[idx]] = besta0;
1491 a[vp9_block2above_8x8[idx] + 1] = besta1;
1492 l[vp9_block2left_8x8[idx]] = bestl0;
1493 l[vp9_block2left_8x8[idx] + 1] = bestl1;
1494 } else {
1495 a[vp9_block2above[ib]] = besta0;
1496 a[vp9_block2above[ib + 1]] = besta1;
1497 l[vp9_block2left[ib]] = bestl0;
1498 l[vp9_block2left[ib + 4]] = bestl1;
1499 }
1500
1501 return best_rd;
1502 }
1503
1504 static int64_t rd_pick_intra8x8mby_modes(VP9_COMP *cpi, MACROBLOCK *mb,
1505 int *Rate, int *rate_y,
1506 int *Distortion, int64_t best_rd) {
1507 MACROBLOCKD *const xd = &mb->e_mbd;
1508 int i, ib;
1509 int cost = mb->mbmode_cost [xd->frame_type] [I8X8_PRED];
1510 int distortion = 0;
1511 int tot_rate_y = 0;
1512 long long total_rd = 0;
1513 ENTROPY_CONTEXT_PLANES t_above, t_left;
1514 ENTROPY_CONTEXT *ta, *tl;
1515 int *i8x8mode_costs;
1516
1517 vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
1518 vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
1519
1520 ta = (ENTROPY_CONTEXT *)&t_above;
1521 tl = (ENTROPY_CONTEXT *)&t_left;
1522
1523 xd->mode_info_context->mbmi.mode = I8X8_PRED;
1524 i8x8mode_costs = mb->i8x8_mode_costs;
1525
1526 for (i = 0; i < 4; i++) {
1527 MODE_INFO *const mic = xd->mode_info_context;
1528 B_PREDICTION_MODE UNINITIALIZED_IS_SAFE(best_mode);
1529 #if CONFIG_COMP_INTRA_PRED
1530 B_PREDICTION_MODE UNINITIALIZED_IS_SAFE(best_second_mode);
1531 #endif
1532 int UNINITIALIZED_IS_SAFE(r), UNINITIALIZED_IS_SAFE(ry), UNINITIALIZED_IS_SA FE(d);
1533
1534 ib = vp9_i8x8_block[i];
1535 total_rd += rd_pick_intra8x8block(
1536 cpi, mb, ib, &best_mode,
1537 #if CONFIG_COMP_INTRA_PRED
1538 & best_second_mode,
1539 #endif
1540 i8x8mode_costs, ta, tl, &r, &ry, &d);
1541 cost += r;
1542 distortion += d;
1543 tot_rate_y += ry;
1544 mic->bmi[ib].as_mode.first = best_mode;
1545 #if CONFIG_COMP_INTRA_PRED
1546 mic->bmi[ib].as_mode.second = best_second_mode;
1547 #endif
1548 }
1549 *Rate = cost;
1550 *rate_y = tot_rate_y;
1551 *Distortion = distortion;
1552 return RDCOST(mb->rdmult, mb->rddiv, cost, distortion);
1553 }
1554
1555 static int rd_cost_mbuv_4x4(MACROBLOCK *mb, int backup) {
1556 int b;
1557 int cost = 0;
1558 MACROBLOCKD *xd = &mb->e_mbd;
1559 ENTROPY_CONTEXT_PLANES t_above, t_left;
1560 ENTROPY_CONTEXT *ta, *tl;
1561
1562 if (backup) {
1563 vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
1564 vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
1565
1566 ta = (ENTROPY_CONTEXT *)&t_above;
1567 tl = (ENTROPY_CONTEXT *)&t_left;
1568 } else {
1569 ta = (ENTROPY_CONTEXT *)xd->above_context;
1570 tl = (ENTROPY_CONTEXT *)xd->left_context;
1571 }
1572
1573 for (b = 16; b < 24; b++)
1574 cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_UV,
1575 ta + vp9_block2above[b], tl + vp9_block2left[b],
1576 TX_4X4);
1577
1578 return cost;
1579 }
1580
1581
1582 static int64_t rd_inter16x16_uv_4x4(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
1583 int *distortion, int fullpixel, int *skip,
1584 int do_ctx_backup) {
1585 vp9_transform_mbuv_4x4(x);
1586 vp9_quantize_mbuv_4x4(x);
1587
1588 *rate = rd_cost_mbuv_4x4(x, do_ctx_backup);
1589 *distortion = vp9_mbuverror(x) / 4;
1590 *skip = vp9_mbuv_is_skippable_4x4(&x->e_mbd);
1591
1592 return RDCOST(x->rdmult, x->rddiv, *rate, *distortion);
1593 }
1594
1595 static int rd_cost_mbuv_8x8(MACROBLOCK *mb, int backup) {
1596 int b;
1597 int cost = 0;
1598 MACROBLOCKD *xd = &mb->e_mbd;
1599 ENTROPY_CONTEXT_PLANES t_above, t_left;
1600 ENTROPY_CONTEXT *ta, *tl;
1601
1602 if (backup) {
1603 vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
1604 vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
1605
1606 ta = (ENTROPY_CONTEXT *)&t_above;
1607 tl = (ENTROPY_CONTEXT *)&t_left;
1608 } else {
1609 ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
1610 tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
1611 }
1612
1613 for (b = 16; b < 24; b += 4)
1614 cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_UV,
1615 ta + vp9_block2above_8x8[b],
1616 tl + vp9_block2left_8x8[b], TX_8X8);
1617
1618 return cost;
1619 }
1620
1621 static int64_t rd_inter16x16_uv_8x8(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
1622 int *distortion, int fullpixel, int *skip,
1623 int do_ctx_backup) {
1624 vp9_transform_mbuv_8x8(x);
1625 vp9_quantize_mbuv_8x8(x);
1626
1627 *rate = rd_cost_mbuv_8x8(x, do_ctx_backup);
1628 *distortion = vp9_mbuverror(x) / 4;
1629 *skip = vp9_mbuv_is_skippable_8x8(&x->e_mbd);
1630
1631 return RDCOST(x->rdmult, x->rddiv, *rate, *distortion);
1632 }
1633
1634 #if CONFIG_SUPERBLOCKS
1635 static int64_t rd_inter32x32_uv(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
1636 int *distortion, int fullpixel, int *skip) {
1637 MACROBLOCKD *xd = &x->e_mbd;
1638 MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
1639 int n, r = 0, d = 0;
1640 const uint8_t *usrc = x->src.u_buffer, *udst = xd->dst.u_buffer;
1641 const uint8_t *vsrc = x->src.v_buffer, *vdst = xd->dst.v_buffer;
1642 int src_uv_stride = x->src.uv_stride, dst_uv_stride = xd->dst.uv_stride;
1643 int skippable = 1;
1644 ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
1645 ENTROPY_CONTEXT_PLANES *ta = xd->above_context;
1646 ENTROPY_CONTEXT_PLANES *tl = xd->left_context;
1647
1648 memcpy(t_above, xd->above_context, sizeof(t_above));
1649 memcpy(t_left, xd->left_context, sizeof(t_left));
1650
1651 for (n = 0; n < 4; n++) {
1652 int x_idx = n & 1, y_idx = n >> 1;
1653 int d_tmp, s_tmp, r_tmp;
1654
1655 xd->above_context = ta + x_idx;
1656 xd->left_context = tl + y_idx;
1657 vp9_subtract_mbuv_s_c(x->src_diff,
1658 usrc + x_idx * 8 + y_idx * 8 * src_uv_stride,
1659 vsrc + x_idx * 8 + y_idx * 8 * src_uv_stride,
1660 src_uv_stride,
1661 udst + x_idx * 8 + y_idx * 8 * dst_uv_stride,
1662 vdst + x_idx * 8 + y_idx * 8 * dst_uv_stride,
1663 dst_uv_stride);
1664
1665 if (mbmi->txfm_size == TX_4X4) {
1666 rd_inter16x16_uv_4x4(cpi, x, &r_tmp, &d_tmp, fullpixel, &s_tmp, 0);
1667 } else {
1668 rd_inter16x16_uv_8x8(cpi, x, &r_tmp, &d_tmp, fullpixel, &s_tmp, 0);
1669 }
1670
1671 r += r_tmp;
1672 d += d_tmp;
1673 skippable = skippable && s_tmp;
1674 }
1675
1676 *rate = r;
1677 *distortion = d;
1678 *skip = skippable;
1679 xd->left_context = tl;
1680 xd->above_context = ta;
1681 memcpy(xd->above_context, t_above, sizeof(t_above));
1682 memcpy(xd->left_context, t_left, sizeof(t_left));
1683
1684 return RDCOST(x->rdmult, x->rddiv, r, d);
1685 }
1686 #endif
1687
1688 static int64_t rd_inter4x4_uv(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
1689 int *distortion, int *skip, int fullpixel) {
1690 vp9_build_inter4x4_predictors_mbuv(&x->e_mbd);
1691 vp9_subtract_mbuv(x->src_diff, x->src.u_buffer, x->src.v_buffer,
1692 x->e_mbd.predictor, x->src.uv_stride);
1693 return rd_inter16x16_uv_4x4(cpi, x, rate, distortion, fullpixel, skip, 1);
1694 }
1695
1696 static void rd_pick_intra_mbuv_mode(VP9_COMP *cpi,
1697 MACROBLOCK *x,
1698 int *rate,
1699 int *rate_tokenonly,
1700 int *distortion,
1701 int *skippable) {
1702 MB_PREDICTION_MODE mode;
1703 MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
1704 #if CONFIG_COMP_INTRA_PRED
1705 MB_PREDICTION_MODE mode2;
1706 MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode2_selected);
1707 #endif
1708 MACROBLOCKD *xd = &x->e_mbd;
1709 MB_MODE_INFO * mbmi = &x->e_mbd.mode_info_context->mbmi;
1710 int64_t best_rd = INT64_MAX;
1711 int UNINITIALIZED_IS_SAFE(d), UNINITIALIZED_IS_SAFE(r);
1712 int rate_to, UNINITIALIZED_IS_SAFE(skip);
1713
1714 for (mode = DC_PRED; mode <= TM_PRED; mode++) {
1715 #if CONFIG_COMP_INTRA_PRED
1716 for (mode2 = DC_PRED - 1; mode2 != TM_PRED + 1; mode2++) {
1717 #endif
1718 int rate;
1719 int distortion;
1720 int64_t this_rd;
1721
1722 mbmi->uv_mode = mode;
1723 #if CONFIG_COMP_INTRA_PRED
1724 mbmi->second_uv_mode = mode2;
1725 if (mode2 == (MB_PREDICTION_MODE)(DC_PRED - 1)) {
1726 #endif
1727 vp9_build_intra_predictors_mbuv(&x->e_mbd);
1728 #if CONFIG_COMP_INTRA_PRED
1729 } else {
1730 continue;
1731 vp9_build_comp_intra_predictors_mbuv(&x->e_mbd);
1732 }
1733 #endif
1734
1735 vp9_subtract_mbuv(x->src_diff, x->src.u_buffer, x->src.v_buffer,
1736 x->e_mbd.predictor, x->src.uv_stride);
1737 vp9_transform_mbuv_4x4(x);
1738 vp9_quantize_mbuv_4x4(x);
1739
1740 rate_to = rd_cost_mbuv_4x4(x, 1);
1741 rate = rate_to
1742 + x->intra_uv_mode_cost[x->e_mbd.frame_type][mbmi->uv_mode];
1743
1744 distortion = vp9_mbuverror(x) / 4;
1745
1746 this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
1747
1748 if (this_rd < best_rd) {
1749 skip = vp9_mbuv_is_skippable_4x4(xd);
1750 best_rd = this_rd;
1751 d = distortion;
1752 r = rate;
1753 *rate_tokenonly = rate_to;
1754 mode_selected = mode;
1755 #if CONFIG_COMP_INTRA_PRED
1756 mode2_selected = mode2;
1757 }
1758 #endif
1759 }
1760 }
1761
1762 *rate = r;
1763 *distortion = d;
1764 *skippable = skip;
1765
1766 mbmi->uv_mode = mode_selected;
1767 #if CONFIG_COMP_INTRA_PRED
1768 mbmi->second_uv_mode = mode2_selected;
1769 #endif
1770 }
1771
1772 static void rd_pick_intra_mbuv_mode_8x8(VP9_COMP *cpi,
1773 MACROBLOCK *x,
1774 int *rate,
1775 int *rate_tokenonly,
1776 int *distortion,
1777 int *skippable) {
1778 MACROBLOCKD *xd = &x->e_mbd;
1779 MB_PREDICTION_MODE mode;
1780 MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
1781 MB_MODE_INFO * mbmi = &x->e_mbd.mode_info_context->mbmi;
1782 int64_t best_rd = INT64_MAX;
1783 int UNINITIALIZED_IS_SAFE(d), UNINITIALIZED_IS_SAFE(r);
1784 int rate_to, UNINITIALIZED_IS_SAFE(skip);
1785
1786 for (mode = DC_PRED; mode <= TM_PRED; mode++) {
1787 int rate;
1788 int distortion;
1789 int64_t this_rd;
1790
1791 mbmi->uv_mode = mode;
1792 vp9_build_intra_predictors_mbuv(&x->e_mbd);
1793 vp9_subtract_mbuv(x->src_diff, x->src.u_buffer, x->src.v_buffer,
1794 x->e_mbd.predictor, x->src.uv_stride);
1795 vp9_transform_mbuv_8x8(x);
1796
1797 vp9_quantize_mbuv_8x8(x);
1798
1799 rate_to = rd_cost_mbuv_8x8(x, 1);
1800 rate = rate_to + x->intra_uv_mode_cost[x->e_mbd.frame_type][mbmi->uv_mode];
1801
1802 distortion = vp9_mbuverror(x) / 4;
1803 this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
1804
1805 if (this_rd < best_rd) {
1806 skip = vp9_mbuv_is_skippable_8x8(xd);
1807 best_rd = this_rd;
1808 d = distortion;
1809 r = rate;
1810 *rate_tokenonly = rate_to;
1811 mode_selected = mode;
1812 }
1813 }
1814 *rate = r;
1815 *distortion = d;
1816 *skippable = skip;
1817 mbmi->uv_mode = mode_selected;
1818 }
1819
1820 #if CONFIG_SUPERBLOCKS
1821 static void super_block_uvrd_8x8(MACROBLOCK *x,
1822 int *rate,
1823 int *distortion,
1824 int *skippable) {
1825 MACROBLOCKD *const xd = &x->e_mbd;
1826 int d = 0, r = 0, n, s = 1;
1827 const uint8_t *usrc = x->src.u_buffer, *udst = xd->dst.u_buffer;
1828 const uint8_t *vsrc = x->src.v_buffer, *vdst = xd->dst.v_buffer;
1829 int src_uv_stride = x->src.uv_stride, dst_uv_stride = xd->dst.uv_stride;
1830 ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
1831 ENTROPY_CONTEXT_PLANES *ta = xd->above_context;
1832 ENTROPY_CONTEXT_PLANES *tl = xd->left_context;
1833
1834 memcpy(t_above, xd->above_context, sizeof(t_above));
1835 memcpy(t_left, xd->left_context, sizeof(t_left));
1836
1837 for (n = 0; n < 4; n++) {
1838 int x_idx = n & 1, y_idx = n >> 1;
1839
1840 vp9_subtract_mbuv_s_c(x->src_diff,
1841 usrc + x_idx * 8 + y_idx * 8 * src_uv_stride,
1842 vsrc + x_idx * 8 + y_idx * 8 * src_uv_stride,
1843 src_uv_stride,
1844 udst + x_idx * 8 + y_idx * 8 * dst_uv_stride,
1845 vdst + x_idx * 8 + y_idx * 8 * dst_uv_stride,
1846 dst_uv_stride);
1847 vp9_transform_mbuv_8x8(x);
1848 vp9_quantize_mbuv_8x8(x);
1849 s &= vp9_mbuv_is_skippable_8x8(xd);
1850
1851 d += vp9_mbuverror(x) >> 2;
1852 xd->above_context = ta + x_idx;
1853 xd->left_context = tl + y_idx;
1854 r += rd_cost_mbuv_8x8(x, 0);
1855 }
1856
1857 xd->above_context = ta;
1858 xd->left_context = tl;
1859 *distortion = d;
1860 *rate = r;
1861 *skippable = s;
1862
1863 xd->left_context = tl;
1864 xd->above_context = ta;
1865 memcpy(xd->above_context, t_above, sizeof(t_above));
1866 memcpy(xd->left_context, t_left, sizeof(t_left));
1867 }
1868
1869 static int64_t rd_pick_intra_sbuv_mode(VP9_COMP *cpi,
1870 MACROBLOCK *x,
1871 int *rate,
1872 int *rate_tokenonly,
1873 int *distortion,
1874 int *skippable) {
1875 MB_PREDICTION_MODE mode;
1876 MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
1877 int64_t best_rd = INT64_MAX, this_rd;
1878 int this_rate_tokenonly, this_rate;
1879 int this_distortion, s;
1880
1881 for (mode = DC_PRED; mode <= TM_PRED; mode++) {
1882 x->e_mbd.mode_info_context->mbmi.uv_mode = mode;
1883 vp9_build_intra_predictors_sbuv_s(&x->e_mbd);
1884
1885 super_block_uvrd_8x8(x, &this_rate_tokenonly,
1886 &this_distortion, &s);
1887 this_rate = this_rate_tokenonly +
1888 x->intra_uv_mode_cost[x->e_mbd.frame_type][mode];
1889 this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, this_distortion);
1890
1891 if (this_rd < best_rd) {
1892 mode_selected = mode;
1893 best_rd = this_rd;
1894 *rate = this_rate;
1895 *rate_tokenonly = this_rate_tokenonly;
1896 *distortion = this_distortion;
1897 *skippable = s;
1898 }
1899 }
1900
1901 x->e_mbd.mode_info_context->mbmi.uv_mode = mode_selected;
1902
1903 return best_rd;
1904 }
1905 #endif
1906
1907 int vp9_cost_mv_ref(VP9_COMP *cpi,
1908 MB_PREDICTION_MODE m,
1909 const int mode_context) {
1910 MACROBLOCKD *xd = &cpi->mb.e_mbd;
1911 int segment_id = xd->mode_info_context->mbmi.segment_id;
1912
1913 // If the mode coding is done entirely at the segment level
1914 // we should not account for it at the per mb level in rd code.
1915 // Note that if the segment level coding is expanded from single mode
1916 // to multiple mode masks as per reference frame coding we will need
1917 // to do something different here.
1918 if (!vp9_segfeature_active(xd, segment_id, SEG_LVL_MODE)) {
1919 VP9_COMMON *pc = &cpi->common;
1920
1921 vp9_prob p [VP9_MVREFS - 1];
1922 assert(NEARESTMV <= m && m <= SPLITMV);
1923 vp9_mv_ref_probs(pc, p, mode_context);
1924 return cost_token(vp9_mv_ref_tree, p,
1925 vp9_mv_ref_encoding_array - NEARESTMV + m);
1926 } else
1927 return 0;
1928 }
1929
1930 void vp9_set_mbmode_and_mvs(MACROBLOCK *x, MB_PREDICTION_MODE mb, int_mv *mv) {
1931 x->e_mbd.mode_info_context->mbmi.mode = mb;
1932 x->e_mbd.mode_info_context->mbmi.mv[0].as_int = mv->as_int;
1933 }
1934
1935 static int labels2mode(
1936 MACROBLOCK *x,
1937 int const *labelings, int which_label,
1938 B_PREDICTION_MODE this_mode,
1939 int_mv *this_mv, int_mv *this_second_mv,
1940 int_mv seg_mvs[MAX_REF_FRAMES - 1],
1941 int_mv *best_ref_mv,
1942 int_mv *second_best_ref_mv,
1943 int *mvjcost, int *mvcost[2]) {
1944 MACROBLOCKD *const xd = &x->e_mbd;
1945 MODE_INFO *const mic = xd->mode_info_context;
1946 MB_MODE_INFO * mbmi = &mic->mbmi;
1947 const int mis = xd->mode_info_stride;
1948
1949 int i, cost = 0, thismvcost = 0;
1950
1951 /* We have to be careful retrieving previously-encoded motion vectors.
1952 Ones from this macroblock have to be pulled from the BLOCKD array
1953 as they have not yet made it to the bmi array in our MB_MODE_INFO. */
1954 for (i = 0; i < 16; ++i) {
1955 BLOCKD *const d = xd->block + i;
1956 const int row = i >> 2, col = i & 3;
1957
1958 B_PREDICTION_MODE m;
1959
1960 if (labelings[i] != which_label)
1961 continue;
1962
1963 if (col && labelings[i] == labelings[i - 1])
1964 m = LEFT4X4;
1965 else if (row && labelings[i] == labelings[i - 4])
1966 m = ABOVE4X4;
1967 else {
1968 // the only time we should do costing for new motion vector or mode
1969 // is when we are on a new label (jbb May 08, 2007)
1970 switch (m = this_mode) {
1971 case NEW4X4 :
1972 if (mbmi->second_ref_frame > 0) {
1973 this_mv->as_int = seg_mvs[mbmi->ref_frame - 1].as_int;
1974 this_second_mv->as_int =
1975 seg_mvs[mbmi->second_ref_frame - 1].as_int;
1976 }
1977
1978 thismvcost = vp9_mv_bit_cost(this_mv, best_ref_mv, mvjcost, mvcost,
1979 102, xd->allow_high_precision_mv);
1980 if (mbmi->second_ref_frame > 0) {
1981 thismvcost += vp9_mv_bit_cost(this_second_mv, second_best_ref_mv,
1982 mvjcost, mvcost, 102,
1983 xd->allow_high_precision_mv);
1984 }
1985 break;
1986 case LEFT4X4:
1987 this_mv->as_int = col ? d[-1].bmi.as_mv.first.as_int : left_block_mv(m ic, i);
1988 if (mbmi->second_ref_frame > 0)
1989 this_second_mv->as_int = col ? d[-1].bmi.as_mv.second.as_int : left_ block_second_mv(mic, i);
1990 break;
1991 case ABOVE4X4:
1992 this_mv->as_int = row ? d[-4].bmi.as_mv.first.as_int : above_block_mv( mic, i, mis);
1993 if (mbmi->second_ref_frame > 0)
1994 this_second_mv->as_int = row ? d[-4].bmi.as_mv.second.as_int : above _block_second_mv(mic, i, mis);
1995 break;
1996 case ZERO4X4:
1997 this_mv->as_int = 0;
1998 if (mbmi->second_ref_frame > 0)
1999 this_second_mv->as_int = 0;
2000 break;
2001 default:
2002 break;
2003 }
2004
2005 if (m == ABOVE4X4) { // replace above with left if same
2006 int_mv left_mv, left_second_mv;
2007
2008 left_second_mv.as_int = 0;
2009 left_mv.as_int = col ? d[-1].bmi.as_mv.first.as_int :
2010 left_block_mv(mic, i);
2011 if (mbmi->second_ref_frame > 0)
2012 left_second_mv.as_int = col ? d[-1].bmi.as_mv.second.as_int :
2013 left_block_second_mv(mic, i);
2014
2015 if (left_mv.as_int == this_mv->as_int &&
2016 (mbmi->second_ref_frame <= 0 ||
2017 left_second_mv.as_int == this_second_mv->as_int))
2018 m = LEFT4X4;
2019 }
2020
2021 #if CONFIG_NEWBINTRAMODES
2022 cost = x->inter_bmode_costs[
2023 m == B_CONTEXT_PRED ? m - CONTEXT_PRED_REPLACEMENTS : m];
2024 #else
2025 cost = x->inter_bmode_costs[m];
2026 #endif
2027 }
2028
2029 d->bmi.as_mv.first.as_int = this_mv->as_int;
2030 if (mbmi->second_ref_frame > 0)
2031 d->bmi.as_mv.second.as_int = this_second_mv->as_int;
2032
2033 x->partition_info->bmi[i].mode = m;
2034 x->partition_info->bmi[i].mv.as_int = this_mv->as_int;
2035 if (mbmi->second_ref_frame > 0)
2036 x->partition_info->bmi[i].second_mv.as_int = this_second_mv->as_int;
2037 }
2038
2039 cost += thismvcost;
2040 return cost;
2041 }
2042
2043 static int64_t encode_inter_mb_segment(MACROBLOCK *x,
2044 int const *labels,
2045 int which_label,
2046 int *labelyrate,
2047 int *distortion,
2048 ENTROPY_CONTEXT *ta,
2049 ENTROPY_CONTEXT *tl) {
2050 int i;
2051 MACROBLOCKD *xd = &x->e_mbd;
2052
2053 *labelyrate = 0;
2054 *distortion = 0;
2055 for (i = 0; i < 16; i++) {
2056 if (labels[i] == which_label) {
2057 BLOCKD *bd = &x->e_mbd.block[i];
2058 BLOCK *be = &x->block[i];
2059 int thisdistortion;
2060
2061 vp9_build_inter_predictors_b(bd, 16, xd->subpixel_predict);
2062 if (xd->mode_info_context->mbmi.second_ref_frame > 0)
2063 vp9_build_2nd_inter_predictors_b(bd, 16, xd->subpixel_predict_avg);
2064 vp9_subtract_b(be, bd, 16);
2065 x->vp9_short_fdct4x4(be->src_diff, be->coeff, 32);
2066 x->quantize_b_4x4(be, bd);
2067 thisdistortion = vp9_block_error(be->coeff, bd->dqcoeff, 16);
2068 *distortion += thisdistortion;
2069 *labelyrate += cost_coeffs(x, bd, PLANE_TYPE_Y_WITH_DC,
2070 ta + vp9_block2above[i],
2071 tl + vp9_block2left[i], TX_4X4);
2072 }
2073 }
2074 *distortion >>= 2;
2075 return RDCOST(x->rdmult, x->rddiv, *labelyrate, *distortion);
2076 }
2077
2078 static int64_t encode_inter_mb_segment_8x8(MACROBLOCK *x,
2079 int const *labels,
2080 int which_label,
2081 int *labelyrate,
2082 int *distortion,
2083 int64_t *otherrd,
2084 ENTROPY_CONTEXT *ta,
2085 ENTROPY_CONTEXT *tl) {
2086 int i, j;
2087 MACROBLOCKD *xd = &x->e_mbd;
2088 const int iblock[4] = { 0, 1, 4, 5 };
2089 int othercost = 0, otherdist = 0;
2090 ENTROPY_CONTEXT_PLANES tac, tlc;
2091 ENTROPY_CONTEXT *tacp = (ENTROPY_CONTEXT *) &tac,
2092 *tlcp = (ENTROPY_CONTEXT *) &tlc;
2093
2094 if (otherrd) {
2095 memcpy(&tac, ta, sizeof(ENTROPY_CONTEXT_PLANES));
2096 memcpy(&tlc, tl, sizeof(ENTROPY_CONTEXT_PLANES));
2097 }
2098
2099 *distortion = 0;
2100 *labelyrate = 0;
2101 for (i = 0; i < 4; i++) {
2102 int ib = vp9_i8x8_block[i];
2103
2104 if (labels[ib] == which_label) {
2105 int idx = (ib & 8) + ((ib & 2) << 1);
2106 BLOCKD *bd = &xd->block[ib], *bd2 = &xd->block[idx];
2107 BLOCK *be = &x->block[ib], *be2 = &x->block[idx];
2108 int thisdistortion;
2109
2110 vp9_build_inter_predictors4b(xd, bd, 16);
2111 if (xd->mode_info_context->mbmi.second_ref_frame > 0)
2112 vp9_build_2nd_inter_predictors4b(xd, bd, 16);
2113 vp9_subtract_4b_c(be, bd, 16);
2114
2115 if (xd->mode_info_context->mbmi.txfm_size == TX_4X4) {
2116 if (otherrd) {
2117 x->vp9_short_fdct8x8(be->src_diff, be2->coeff, 32);
2118 x->quantize_b_8x8(be2, bd2);
2119 thisdistortion = vp9_block_error_c(be2->coeff, bd2->dqcoeff, 64);
2120 otherdist += thisdistortion;
2121 othercost += cost_coeffs(x, bd2, PLANE_TYPE_Y_WITH_DC,
2122 tacp + vp9_block2above_8x8[idx],
2123 tlcp + vp9_block2left_8x8[idx], TX_8X8);
2124 }
2125 for (j = 0; j < 4; j += 2) {
2126 bd = &xd->block[ib + iblock[j]];
2127 be = &x->block[ib + iblock[j]];
2128 x->vp9_short_fdct8x4(be->src_diff, be->coeff, 32);
2129 x->quantize_b_4x4_pair(be, be + 1, bd, bd + 1);
2130 thisdistortion = vp9_block_error_c(be->coeff, bd->dqcoeff, 32);
2131 *distortion += thisdistortion;
2132 *labelyrate += cost_coeffs(x, bd, PLANE_TYPE_Y_WITH_DC,
2133 ta + vp9_block2above[ib + iblock[j]],
2134 tl + vp9_block2left[ib + iblock[j]],
2135 TX_4X4);
2136 *labelyrate += cost_coeffs(x, bd + 1, PLANE_TYPE_Y_WITH_DC,
2137 ta + vp9_block2above[ib + iblock[j] + 1],
2138 tl + vp9_block2left[ib + iblock[j]],
2139 TX_4X4);
2140 }
2141 } else /* 8x8 */ {
2142 if (otherrd) {
2143 for (j = 0; j < 4; j += 2) {
2144 BLOCKD *bd = &xd->block[ib + iblock[j]];
2145 BLOCK *be = &x->block[ib + iblock[j]];
2146 x->vp9_short_fdct8x4(be->src_diff, be->coeff, 32);
2147 x->quantize_b_4x4_pair(be, be + 1, bd, bd + 1);
2148 thisdistortion = vp9_block_error_c(be->coeff, bd->dqcoeff, 32);
2149 otherdist += thisdistortion;
2150 othercost += cost_coeffs(x, bd, PLANE_TYPE_Y_WITH_DC,
2151 tacp + vp9_block2above[ib + iblock[j]],
2152 tlcp + vp9_block2left[ib + iblock[j]],
2153 TX_4X4);
2154 othercost += cost_coeffs(x, bd + 1, PLANE_TYPE_Y_WITH_DC,
2155 tacp + vp9_block2above[ib + iblock[j] + 1],
2156 tlcp + vp9_block2left[ib + iblock[j]],
2157 TX_4X4);
2158 }
2159 }
2160 x->vp9_short_fdct8x8(be->src_diff, be2->coeff, 32);
2161 x->quantize_b_8x8(be2, bd2);
2162 thisdistortion = vp9_block_error_c(be2->coeff, bd2->dqcoeff, 64);
2163 *distortion += thisdistortion;
2164 *labelyrate += cost_coeffs(x, bd2, PLANE_TYPE_Y_WITH_DC,
2165 ta + vp9_block2above_8x8[idx],
2166 tl + vp9_block2left_8x8[idx], TX_8X8);
2167 }
2168 }
2169 }
2170 *distortion >>= 2;
2171 if (otherrd) {
2172 otherdist >>= 2;
2173 *otherrd = RDCOST(x->rdmult, x->rddiv, othercost, otherdist);
2174 }
2175 return RDCOST(x->rdmult, x->rddiv, *labelyrate, *distortion);
2176 }
2177
2178 static const unsigned int segmentation_to_sseshift[4] = {3, 3, 2, 0};
2179
2180
2181 typedef struct {
2182 int_mv *ref_mv, *second_ref_mv;
2183 int_mv mvp;
2184
2185 int64_t segment_rd;
2186 SPLITMV_PARTITIONING_TYPE segment_num;
2187 TX_SIZE txfm_size;
2188 int r;
2189 int d;
2190 int segment_yrate;
2191 B_PREDICTION_MODE modes[16];
2192 int_mv mvs[16], second_mvs[16];
2193 int eobs[16];
2194
2195 int mvthresh;
2196 int *mdcounts;
2197
2198 int_mv sv_mvp[4]; // save 4 mvp from 8x8
2199 int sv_istep[2]; // save 2 initial step_param for 16x8/8x16
2200
2201 } BEST_SEG_INFO;
2202
2203 static __inline
2204 int mv_check_bounds(MACROBLOCK *x, int_mv *mv) {
2205 int r = 0;
2206 r |= (mv->as_mv.row >> 3) < x->mv_row_min;
2207 r |= (mv->as_mv.row >> 3) > x->mv_row_max;
2208 r |= (mv->as_mv.col >> 3) < x->mv_col_min;
2209 r |= (mv->as_mv.col >> 3) > x->mv_col_max;
2210 return r;
2211 }
2212
2213 static void rd_check_segment_txsize(VP9_COMP *cpi, MACROBLOCK *x,
2214 BEST_SEG_INFO *bsi,
2215 SPLITMV_PARTITIONING_TYPE segmentation,
2216 TX_SIZE tx_size, int64_t *otherrds,
2217 int64_t *rds, int *completed,
2218 /* 16 = n_blocks */
2219 int_mv seg_mvs[16 /* n_blocks */]
2220 [MAX_REF_FRAMES - 1]) {
2221 int i, j;
2222 int const *labels;
2223 int br = 0, bd = 0;
2224 B_PREDICTION_MODE this_mode;
2225 MB_MODE_INFO * mbmi = &x->e_mbd.mode_info_context->mbmi;
2226
2227 int label_count;
2228 int64_t this_segment_rd = 0, other_segment_rd;
2229 int label_mv_thresh;
2230 int rate = 0;
2231 int sbr = 0, sbd = 0;
2232 int segmentyrate = 0;
2233 int best_eobs[16] = { 0 };
2234
2235 vp9_variance_fn_ptr_t *v_fn_ptr;
2236
2237 ENTROPY_CONTEXT_PLANES t_above, t_left;
2238 ENTROPY_CONTEXT *ta, *tl;
2239 ENTROPY_CONTEXT_PLANES t_above_b, t_left_b;
2240 ENTROPY_CONTEXT *ta_b, *tl_b;
2241
2242 vpx_memcpy(&t_above, x->e_mbd.above_context, sizeof(ENTROPY_CONTEXT_PLANES));
2243 vpx_memcpy(&t_left, x->e_mbd.left_context, sizeof(ENTROPY_CONTEXT_PLANES));
2244
2245 ta = (ENTROPY_CONTEXT *)&t_above;
2246 tl = (ENTROPY_CONTEXT *)&t_left;
2247 ta_b = (ENTROPY_CONTEXT *)&t_above_b;
2248 tl_b = (ENTROPY_CONTEXT *)&t_left_b;
2249
2250 v_fn_ptr = &cpi->fn_ptr[segmentation];
2251 labels = vp9_mbsplits[segmentation];
2252 label_count = vp9_mbsplit_count[segmentation];
2253
2254 // 64 makes this threshold really big effectively
2255 // making it so that we very rarely check mvs on
2256 // segments. setting this to 1 would make mv thresh
2257 // roughly equal to what it is for macroblocks
2258 label_mv_thresh = 1 * bsi->mvthresh / label_count;
2259
2260 // Segmentation method overheads
2261 rate = cost_token(vp9_mbsplit_tree, vp9_mbsplit_probs,
2262 vp9_mbsplit_encodings + segmentation);
2263 rate += vp9_cost_mv_ref(cpi, SPLITMV,
2264 mbmi->mb_mode_context[mbmi->ref_frame]);
2265 this_segment_rd += RDCOST(x->rdmult, x->rddiv, rate, 0);
2266 br += rate;
2267 other_segment_rd = this_segment_rd;
2268
2269 mbmi->txfm_size = tx_size;
2270 for (i = 0; i < label_count && this_segment_rd < bsi->segment_rd; i++) {
2271 int_mv mode_mv[B_MODE_COUNT], second_mode_mv[B_MODE_COUNT];
2272 int64_t best_label_rd = INT64_MAX, best_other_rd = INT64_MAX;
2273 B_PREDICTION_MODE mode_selected = ZERO4X4;
2274 int bestlabelyrate = 0;
2275
2276 // search for the best motion vector on this segment
2277 for (this_mode = LEFT4X4; this_mode <= NEW4X4; this_mode ++) {
2278 int64_t this_rd, other_rd;
2279 int distortion;
2280 int labelyrate;
2281 ENTROPY_CONTEXT_PLANES t_above_s, t_left_s;
2282 ENTROPY_CONTEXT *ta_s;
2283 ENTROPY_CONTEXT *tl_s;
2284
2285 vpx_memcpy(&t_above_s, &t_above, sizeof(ENTROPY_CONTEXT_PLANES));
2286 vpx_memcpy(&t_left_s, &t_left, sizeof(ENTROPY_CONTEXT_PLANES));
2287
2288 ta_s = (ENTROPY_CONTEXT *)&t_above_s;
2289 tl_s = (ENTROPY_CONTEXT *)&t_left_s;
2290
2291 // motion search for newmv (single predictor case only)
2292 if (mbmi->second_ref_frame <= 0 && this_mode == NEW4X4) {
2293 int sseshift, n;
2294 int step_param = 0;
2295 int further_steps;
2296 int thissme, bestsme = INT_MAX;
2297 BLOCK *c;
2298 BLOCKD *e;
2299
2300 /* Is the best so far sufficiently good that we cant justify doing
2301 * and new motion search. */
2302 if (best_label_rd < label_mv_thresh)
2303 break;
2304
2305 if (cpi->compressor_speed) {
2306 if (segmentation == PARTITIONING_8X16 ||
2307 segmentation == PARTITIONING_16X8) {
2308 bsi->mvp.as_int = bsi->sv_mvp[i].as_int;
2309 if (i == 1 && segmentation == PARTITIONING_16X8)
2310 bsi->mvp.as_int = bsi->sv_mvp[2].as_int;
2311
2312 step_param = bsi->sv_istep[i];
2313 }
2314
2315 // use previous block's result as next block's MV predictor.
2316 if (segmentation == PARTITIONING_4X4 && i > 0) {
2317 bsi->mvp.as_int = x->e_mbd.block[i - 1].bmi.as_mv.first.as_int;
2318 if (i == 4 || i == 8 || i == 12)
2319 bsi->mvp.as_int = x->e_mbd.block[i - 4].bmi.as_mv.first.as_int;
2320 step_param = 2;
2321 }
2322 }
2323
2324 further_steps = (MAX_MVSEARCH_STEPS - 1) - step_param;
2325
2326 {
2327 int sadpb = x->sadperbit4;
2328 int_mv mvp_full;
2329
2330 mvp_full.as_mv.row = bsi->mvp.as_mv.row >> 3;
2331 mvp_full.as_mv.col = bsi->mvp.as_mv.col >> 3;
2332
2333 // find first label
2334 n = vp9_mbsplit_offset[segmentation][i];
2335
2336 c = &x->block[n];
2337 e = &x->e_mbd.block[n];
2338
2339 bestsme = vp9_full_pixel_diamond(cpi, x, c, e, &mvp_full, step_param,
2340 sadpb, further_steps, 0, v_fn_ptr,
2341 bsi->ref_mv, &mode_mv[NEW4X4]);
2342
2343 sseshift = segmentation_to_sseshift[segmentation];
2344
2345 // Should we do a full search (best quality only)
2346 if ((cpi->compressor_speed == 0) && (bestsme >> sseshift) > 4000) {
2347 /* Check if mvp_full is within the range. */
2348 clamp_mv(&mvp_full, x->mv_col_min, x->mv_col_max,
2349 x->mv_row_min, x->mv_row_max);
2350
2351 thissme = cpi->full_search_sad(x, c, e, &mvp_full,
2352 sadpb, 16, v_fn_ptr,
2353 x->nmvjointcost, x->mvcost,
2354 bsi->ref_mv);
2355
2356 if (thissme < bestsme) {
2357 bestsme = thissme;
2358 mode_mv[NEW4X4].as_int = e->bmi.as_mv.first.as_int;
2359 } else {
2360 /* The full search result is actually worse so re-instate the
2361 * previous best vector */
2362 e->bmi.as_mv.first.as_int = mode_mv[NEW4X4].as_int;
2363 }
2364 }
2365 }
2366
2367 if (bestsme < INT_MAX) {
2368 int distortion;
2369 unsigned int sse;
2370 cpi->find_fractional_mv_step(x, c, e, &mode_mv[NEW4X4],
2371 bsi->ref_mv, x->errorperbit, v_fn_ptr,
2372 x->nmvjointcost, x->mvcost,
2373 &distortion, &sse);
2374
2375 // safe motion search result for use in compound prediction
2376 seg_mvs[i][mbmi->ref_frame - 1].as_int = mode_mv[NEW4X4].as_int;
2377 }
2378 } else if (mbmi->second_ref_frame > 0 && this_mode == NEW4X4) {
2379 /* NEW4X4 */
2380 /* motion search not completed? Then skip newmv for this block with
2381 * comppred */
2382 if (seg_mvs[i][mbmi->second_ref_frame - 1].as_int == INVALID_MV ||
2383 seg_mvs[i][mbmi->ref_frame - 1].as_int == INVALID_MV) {
2384 continue;
2385 }
2386 }
2387
2388 rate = labels2mode(x, labels, i, this_mode, &mode_mv[this_mode],
2389 &second_mode_mv[this_mode], seg_mvs[i],
2390 bsi->ref_mv, bsi->second_ref_mv, x->nmvjointcost,
2391 x->mvcost);
2392
2393 // Trap vectors that reach beyond the UMV borders
2394 if (((mode_mv[this_mode].as_mv.row >> 3) < x->mv_row_min) ||
2395 ((mode_mv[this_mode].as_mv.row >> 3) > x->mv_row_max) ||
2396 ((mode_mv[this_mode].as_mv.col >> 3) < x->mv_col_min) ||
2397 ((mode_mv[this_mode].as_mv.col >> 3) > x->mv_col_max)) {
2398 continue;
2399 }
2400 if (mbmi->second_ref_frame > 0 &&
2401 mv_check_bounds(x, &second_mode_mv[this_mode]))
2402 continue;
2403
2404 if (segmentation == PARTITIONING_4X4) {
2405 this_rd = encode_inter_mb_segment(x, labels, i, &labelyrate,
2406 &distortion, ta_s, tl_s);
2407 other_rd = this_rd;
2408 } else {
2409 this_rd = encode_inter_mb_segment_8x8(x, labels, i, &labelyrate,
2410 &distortion, &other_rd,
2411 ta_s, tl_s);
2412 }
2413 this_rd += RDCOST(x->rdmult, x->rddiv, rate, 0);
2414 rate += labelyrate;
2415
2416 if (this_rd < best_label_rd) {
2417 sbr = rate;
2418 sbd = distortion;
2419 bestlabelyrate = labelyrate;
2420 mode_selected = this_mode;
2421 best_label_rd = this_rd;
2422 if (x->e_mbd.mode_info_context->mbmi.txfm_size == TX_4X4) {
2423 for (j = 0; j < 16; j++)
2424 if (labels[j] == i)
2425 best_eobs[j] = x->e_mbd.block[j].eob;
2426 } else {
2427 for (j = 0; j < 4; j++) {
2428 int ib = vp9_i8x8_block[j], idx = j * 4;
2429
2430 if (labels[ib] == i)
2431 best_eobs[idx] = x->e_mbd.block[idx].eob;
2432 }
2433 }
2434 if (other_rd < best_other_rd)
2435 best_other_rd = other_rd;
2436
2437 vpx_memcpy(ta_b, ta_s, sizeof(ENTROPY_CONTEXT_PLANES));
2438 vpx_memcpy(tl_b, tl_s, sizeof(ENTROPY_CONTEXT_PLANES));
2439
2440 }
2441 } /*for each 4x4 mode*/
2442
2443 vpx_memcpy(ta, ta_b, sizeof(ENTROPY_CONTEXT_PLANES));
2444 vpx_memcpy(tl, tl_b, sizeof(ENTROPY_CONTEXT_PLANES));
2445
2446 labels2mode(x, labels, i, mode_selected, &mode_mv[mode_selected],
2447 &second_mode_mv[mode_selected], seg_mvs[i],
2448 bsi->ref_mv, bsi->second_ref_mv, x->nmvjointcost, x->mvcost);
2449
2450 br += sbr;
2451 bd += sbd;
2452 segmentyrate += bestlabelyrate;
2453 this_segment_rd += best_label_rd;
2454 other_segment_rd += best_other_rd;
2455 if (rds)
2456 rds[i] = this_segment_rd;
2457 if (otherrds)
2458 otherrds[i] = other_segment_rd;
2459 } /* for each label */
2460
2461 if (this_segment_rd < bsi->segment_rd) {
2462 bsi->r = br;
2463 bsi->d = bd;
2464 bsi->segment_yrate = segmentyrate;
2465 bsi->segment_rd = this_segment_rd;
2466 bsi->segment_num = segmentation;
2467 bsi->txfm_size = mbmi->txfm_size;
2468
2469 // store everything needed to come back to this!!
2470 for (i = 0; i < 16; i++) {
2471 bsi->mvs[i].as_mv = x->partition_info->bmi[i].mv.as_mv;
2472 if (mbmi->second_ref_frame > 0)
2473 bsi->second_mvs[i].as_mv = x->partition_info->bmi[i].second_mv.as_mv;
2474 bsi->modes[i] = x->partition_info->bmi[i].mode;
2475 bsi->eobs[i] = best_eobs[i];
2476 }
2477 }
2478
2479 if (completed) {
2480 *completed = i;
2481 }
2482 }
2483
2484 static void rd_check_segment(VP9_COMP *cpi, MACROBLOCK *x,
2485 BEST_SEG_INFO *bsi,
2486 unsigned int segmentation,
2487 /* 16 = n_blocks */
2488 int_mv seg_mvs[16][MAX_REF_FRAMES - 1],
2489 int64_t txfm_cache[NB_TXFM_MODES]) {
2490 int i, n, c = vp9_mbsplit_count[segmentation];
2491
2492 if (segmentation == PARTITIONING_4X4) {
2493 int64_t rd[16];
2494
2495 rd_check_segment_txsize(cpi, x, bsi, segmentation, TX_4X4, NULL,
2496 rd, &n, seg_mvs);
2497 if (n == c) {
2498 for (i = 0; i < NB_TXFM_MODES; i++) {
2499 if (rd[c - 1] < txfm_cache[i])
2500 txfm_cache[i] = rd[c - 1];
2501 }
2502 }
2503 } else {
2504 int64_t diff, base_rd;
2505 int cost4x4 = vp9_cost_bit(cpi->common.prob_tx[0], 0);
2506 int cost8x8 = vp9_cost_bit(cpi->common.prob_tx[0], 1);
2507
2508 if (cpi->common.txfm_mode == TX_MODE_SELECT) {
2509 int64_t rd4x4[4], rd8x8[4];
2510 int n4x4, n8x8, nmin;
2511 BEST_SEG_INFO bsi4x4, bsi8x8;
2512
2513 /* factor in cost of cost4x4/8x8 in decision */
2514 vpx_memcpy(&bsi4x4, bsi, sizeof(*bsi));
2515 vpx_memcpy(&bsi8x8, bsi, sizeof(*bsi));
2516 rd_check_segment_txsize(cpi, x, &bsi4x4, segmentation,
2517 TX_4X4, NULL, rd4x4, &n4x4, seg_mvs);
2518 rd_check_segment_txsize(cpi, x, &bsi8x8, segmentation,
2519 TX_8X8, NULL, rd8x8, &n8x8, seg_mvs);
2520 if (bsi4x4.segment_num == segmentation) {
2521 bsi4x4.segment_rd += RDCOST(x->rdmult, x->rddiv, cost4x4, 0);
2522 if (bsi4x4.segment_rd < bsi->segment_rd)
2523 vpx_memcpy(bsi, &bsi4x4, sizeof(*bsi));
2524 }
2525 if (bsi8x8.segment_num == segmentation) {
2526 bsi8x8.segment_rd += RDCOST(x->rdmult, x->rddiv, cost8x8, 0);
2527 if (bsi8x8.segment_rd < bsi->segment_rd)
2528 vpx_memcpy(bsi, &bsi8x8, sizeof(*bsi));
2529 }
2530 n = n4x4 > n8x8 ? n4x4 : n8x8;
2531 if (n == c) {
2532 nmin = n4x4 < n8x8 ? n4x4 : n8x8;
2533 diff = rd8x8[nmin - 1] - rd4x4[nmin - 1];
2534 if (n == n4x4) {
2535 base_rd = rd4x4[c - 1];
2536 } else {
2537 base_rd = rd8x8[c - 1] - diff;
2538 }
2539 }
2540 } else {
2541 int64_t rd[4], otherrd[4];
2542
2543 if (cpi->common.txfm_mode == ONLY_4X4) {
2544 rd_check_segment_txsize(cpi, x, bsi, segmentation, TX_4X4, otherrd,
2545 rd, &n, seg_mvs);
2546 if (n == c) {
2547 base_rd = rd[c - 1];
2548 diff = otherrd[c - 1] - rd[c - 1];
2549 }
2550 } else /* use 8x8 transform */ {
2551 rd_check_segment_txsize(cpi, x, bsi, segmentation, TX_8X8, otherrd,
2552 rd, &n, seg_mvs);
2553 if (n == c) {
2554 diff = rd[c - 1] - otherrd[c - 1];
2555 base_rd = otherrd[c - 1];
2556 }
2557 }
2558 }
2559
2560 if (n == c) {
2561 if (base_rd < txfm_cache[ONLY_4X4]) {
2562 txfm_cache[ONLY_4X4] = base_rd;
2563 }
2564 if (base_rd + diff < txfm_cache[1]) {
2565 txfm_cache[ALLOW_8X8] = txfm_cache[ALLOW_16X16] = base_rd + diff;
2566 }
2567 if (diff < 0) {
2568 base_rd += diff + RDCOST(x->rdmult, x->rddiv, cost8x8, 0);
2569 } else {
2570 base_rd += RDCOST(x->rdmult, x->rddiv, cost4x4, 0);
2571 }
2572 if (base_rd < txfm_cache[TX_MODE_SELECT]) {
2573 txfm_cache[TX_MODE_SELECT] = base_rd;
2574 }
2575 }
2576 }
2577 }
2578
2579 static __inline void cal_step_param(int sr, int *sp) {
2580 int step = 0;
2581
2582 if (sr > MAX_FIRST_STEP) sr = MAX_FIRST_STEP;
2583 else if (sr < 1) sr = 1;
2584
2585 while (sr >>= 1)
2586 step++;
2587
2588 *sp = MAX_MVSEARCH_STEPS - 1 - step;
2589 }
2590
2591 static int rd_pick_best_mbsegmentation(VP9_COMP *cpi, MACROBLOCK *x,
2592 int_mv *best_ref_mv,
2593 int_mv *second_best_ref_mv,
2594 int64_t best_rd,
2595 int *mdcounts,
2596 int *returntotrate,
2597 int *returnyrate,
2598 int *returndistortion,
2599 int *skippable, int mvthresh,
2600 int_mv seg_mvs[NB_PARTITIONINGS]
2601 [16 /* n_blocks */]
2602 [MAX_REF_FRAMES - 1],
2603 int64_t txfm_cache[NB_TXFM_MODES]) {
2604 int i;
2605 BEST_SEG_INFO bsi;
2606 MB_MODE_INFO * mbmi = &x->e_mbd.mode_info_context->mbmi;
2607
2608 vpx_memset(&bsi, 0, sizeof(bsi));
2609 for (i = 0; i < NB_TXFM_MODES; i++)
2610 txfm_cache[i] = INT64_MAX;
2611
2612 bsi.segment_rd = best_rd;
2613 bsi.ref_mv = best_ref_mv;
2614 bsi.second_ref_mv = second_best_ref_mv;
2615 bsi.mvp.as_int = best_ref_mv->as_int;
2616 bsi.mvthresh = mvthresh;
2617 bsi.mdcounts = mdcounts;
2618 bsi.txfm_size = TX_4X4;
2619
2620 for (i = 0; i < 16; i++)
2621 bsi.modes[i] = ZERO4X4;
2622
2623 if (cpi->compressor_speed == 0) {
2624 /* for now, we will keep the original segmentation order
2625 when in best quality mode */
2626 rd_check_segment(cpi, x, &bsi, PARTITIONING_16X8,
2627 seg_mvs[PARTITIONING_16X8], txfm_cache);
2628 rd_check_segment(cpi, x, &bsi, PARTITIONING_8X16,
2629 seg_mvs[PARTITIONING_8X16], txfm_cache);
2630 rd_check_segment(cpi, x, &bsi, PARTITIONING_8X8,
2631 seg_mvs[PARTITIONING_8X8], txfm_cache);
2632 rd_check_segment(cpi, x, &bsi, PARTITIONING_4X4,
2633 seg_mvs[PARTITIONING_4X4], txfm_cache);
2634 } else {
2635 int sr;
2636
2637 rd_check_segment(cpi, x, &bsi, PARTITIONING_8X8,
2638 seg_mvs[PARTITIONING_8X8], txfm_cache);
2639
2640 if (bsi.segment_rd < best_rd) {
2641 int tmp_col_min = x->mv_col_min;
2642 int tmp_col_max = x->mv_col_max;
2643 int tmp_row_min = x->mv_row_min;
2644 int tmp_row_max = x->mv_row_max;
2645
2646 vp9_clamp_mv_min_max(x, best_ref_mv);
2647
2648 /* Get 8x8 result */
2649 bsi.sv_mvp[0].as_int = bsi.mvs[0].as_int;
2650 bsi.sv_mvp[1].as_int = bsi.mvs[2].as_int;
2651 bsi.sv_mvp[2].as_int = bsi.mvs[8].as_int;
2652 bsi.sv_mvp[3].as_int = bsi.mvs[10].as_int;
2653
2654 /* Use 8x8 result as 16x8/8x16's predictor MV. Adjust search range
2655 * according to the closeness of 2 MV. */
2656 /* block 8X16 */
2657 sr = MAXF((abs(bsi.sv_mvp[0].as_mv.row - bsi.sv_mvp[2].as_mv.row)) >> 3,
2658 (abs(bsi.sv_mvp[0].as_mv.col - bsi.sv_mvp[2].as_mv.col)) >> 3);
2659 cal_step_param(sr, &bsi.sv_istep[0]);
2660
2661 sr = MAXF((abs(bsi.sv_mvp[1].as_mv.row - bsi.sv_mvp[3].as_mv.row)) >> 3,
2662 (abs(bsi.sv_mvp[1].as_mv.col - bsi.sv_mvp[3].as_mv.col)) >> 3);
2663 cal_step_param(sr, &bsi.sv_istep[1]);
2664
2665 rd_check_segment(cpi, x, &bsi, PARTITIONING_8X16,
2666 seg_mvs[PARTITIONING_8X16], txfm_cache);
2667
2668 /* block 16X8 */
2669 sr = MAXF((abs(bsi.sv_mvp[0].as_mv.row - bsi.sv_mvp[1].as_mv.row)) >> 3,
2670 (abs(bsi.sv_mvp[0].as_mv.col - bsi.sv_mvp[1].as_mv.col)) >> 3);
2671 cal_step_param(sr, &bsi.sv_istep[0]);
2672
2673 sr = MAXF((abs(bsi.sv_mvp[2].as_mv.row - bsi.sv_mvp[3].as_mv.row)) >> 3,
2674 (abs(bsi.sv_mvp[2].as_mv.col - bsi.sv_mvp[3].as_mv.col)) >> 3);
2675 cal_step_param(sr, &bsi.sv_istep[1]);
2676
2677 rd_check_segment(cpi, x, &bsi, PARTITIONING_16X8,
2678 seg_mvs[PARTITIONING_16X8], txfm_cache);
2679
2680 /* If 8x8 is better than 16x8/8x16, then do 4x4 search */
2681 /* Not skip 4x4 if speed=0 (good quality) */
2682 if (cpi->sf.no_skip_block4x4_search ||
2683 bsi.segment_num == PARTITIONING_8X8) {
2684 /* || (sv_segment_rd8x8-bsi.segment_rd) < sv_segment_rd8x8>>5) */
2685 bsi.mvp.as_int = bsi.sv_mvp[0].as_int;
2686 rd_check_segment(cpi, x, &bsi, PARTITIONING_4X4,
2687 seg_mvs[PARTITIONING_4X4], txfm_cache);
2688 }
2689
2690 /* restore UMV window */
2691 x->mv_col_min = tmp_col_min;
2692 x->mv_col_max = tmp_col_max;
2693 x->mv_row_min = tmp_row_min;
2694 x->mv_row_max = tmp_row_max;
2695 }
2696 }
2697
2698 /* set it to the best */
2699 for (i = 0; i < 16; i++) {
2700 BLOCKD *bd = &x->e_mbd.block[i];
2701
2702 bd->bmi.as_mv.first.as_int = bsi.mvs[i].as_int;
2703 if (mbmi->second_ref_frame > 0)
2704 bd->bmi.as_mv.second.as_int = bsi.second_mvs[i].as_int;
2705 bd->eob = bsi.eobs[i];
2706 }
2707
2708 *returntotrate = bsi.r;
2709 *returndistortion = bsi.d;
2710 *returnyrate = bsi.segment_yrate;
2711 *skippable = bsi.txfm_size == TX_4X4 ?
2712 vp9_mby_is_skippable_4x4(&x->e_mbd, 0) :
2713 vp9_mby_is_skippable_8x8(&x->e_mbd, 0);
2714
2715 /* save partitions */
2716 mbmi->txfm_size = bsi.txfm_size;
2717 mbmi->partitioning = bsi.segment_num;
2718 x->partition_info->count = vp9_mbsplit_count[bsi.segment_num];
2719
2720 for (i = 0; i < x->partition_info->count; i++) {
2721 int j;
2722
2723 j = vp9_mbsplit_offset[bsi.segment_num][i];
2724
2725 x->partition_info->bmi[i].mode = bsi.modes[j];
2726 x->partition_info->bmi[i].mv.as_mv = bsi.mvs[j].as_mv;
2727 if (mbmi->second_ref_frame > 0)
2728 x->partition_info->bmi[i].second_mv.as_mv = bsi.second_mvs[j].as_mv;
2729 }
2730 /*
2731 * used to set mbmi->mv.as_int
2732 */
2733 x->partition_info->bmi[15].mv.as_int = bsi.mvs[15].as_int;
2734 if (mbmi->second_ref_frame > 0)
2735 x->partition_info->bmi[15].second_mv.as_int = bsi.second_mvs[15].as_int;
2736
2737 return (int)(bsi.segment_rd);
2738 }
2739
2740 static void mv_pred(VP9_COMP *cpi, MACROBLOCK *x,
2741 unsigned char *ref_y_buffer, int ref_y_stride,
2742 int_mv *mvp, int ref_frame, enum BlockSize block_size ) {
2743 MACROBLOCKD *xd = &x->e_mbd;
2744 MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
2745 int_mv this_mv;
2746 int i;
2747 int zero_seen = FALSE;
2748 int best_index = 0;
2749 int best_sad = INT_MAX;
2750 int this_sad = INT_MAX;
2751
2752 BLOCK *b = &x->block[0];
2753 unsigned char *src_y_ptr = *(b->base_src);
2754 unsigned char *ref_y_ptr;
2755 int row_offset, col_offset;
2756
2757 // Get the sad for each candidate reference mv
2758 for (i = 0; i < 4; i++) {
2759 this_mv.as_int = mbmi->ref_mvs[ref_frame][i].as_int;
2760
2761 // The list is at an end if we see 0 for a second time.
2762 if (!this_mv.as_int && zero_seen)
2763 break;
2764 zero_seen = zero_seen || !this_mv.as_int;
2765
2766 row_offset = this_mv.as_mv.row >> 3;
2767 col_offset = this_mv.as_mv.col >> 3;
2768 ref_y_ptr = ref_y_buffer + (ref_y_stride * row_offset) + col_offset;
2769
2770 // Find sad for current vector.
2771 this_sad = cpi->fn_ptr[block_size].sdf(src_y_ptr, b->src_stride,
2772 ref_y_ptr, ref_y_stride,
2773 0x7fffffff);
2774
2775 // Note if it is the best so far.
2776 if (this_sad < best_sad) {
2777 best_sad = this_sad;
2778 best_index = i;
2779 }
2780 }
2781
2782 // Return the mv that had the best sad for use in the motion search.
2783 mvp->as_int = mbmi->ref_mvs[ref_frame][best_index].as_int;
2784 clamp_mv2(mvp, xd);
2785 }
2786
2787 static void set_i8x8_block_modes(MACROBLOCK *x, int modes[2][4]) {
2788 int i;
2789 MACROBLOCKD *xd = &x->e_mbd;
2790 for (i = 0; i < 4; i++) {
2791 int ib = vp9_i8x8_block[i];
2792 xd->mode_info_context->bmi[ib + 0].as_mode.first = modes[0][i];
2793 xd->mode_info_context->bmi[ib + 1].as_mode.first = modes[0][i];
2794 xd->mode_info_context->bmi[ib + 4].as_mode.first = modes[0][i];
2795 xd->mode_info_context->bmi[ib + 5].as_mode.first = modes[0][i];
2796 #if CONFIG_COMP_INTRA_PRED
2797 xd->mode_info_context->bmi[ib + 0].as_mode.second = modes[1][i];
2798 xd->mode_info_context->bmi[ib + 1].as_mode.second = modes[1][i];
2799 xd->mode_info_context->bmi[ib + 4].as_mode.second = modes[1][i];
2800 xd->mode_info_context->bmi[ib + 5].as_mode.second = modes[1][i];
2801 #endif
2802 // printf("%d,%d,%d,%d %d,%d,%d,%d\n",
2803 // modes[0][0], modes[0][1], modes[0][2], modes[0][3],
2804 // modes[1][0], modes[1][1], modes[1][2], modes[1][3]);
2805 }
2806
2807 for (i = 0; i < 16; i++) {
2808 xd->block[i].bmi = xd->mode_info_context->bmi[i];
2809 }
2810 }
2811
2812 extern void vp9_calc_ref_probs(int *count, vp9_prob *probs);
2813 static void estimate_curframe_refprobs(VP9_COMP *cpi, vp9_prob mod_refprobs[3], int pred_ref) {
2814 int norm_cnt[MAX_REF_FRAMES];
2815 const int *const rfct = cpi->count_mb_ref_frame_usage;
2816 int intra_count = rfct[INTRA_FRAME];
2817 int last_count = rfct[LAST_FRAME];
2818 int gf_count = rfct[GOLDEN_FRAME];
2819 int arf_count = rfct[ALTREF_FRAME];
2820
2821 // Work out modified reference frame probabilities to use where prediction
2822 // of the reference frame fails
2823 if (pred_ref == INTRA_FRAME) {
2824 norm_cnt[0] = 0;
2825 norm_cnt[1] = last_count;
2826 norm_cnt[2] = gf_count;
2827 norm_cnt[3] = arf_count;
2828 vp9_calc_ref_probs(norm_cnt, mod_refprobs);
2829 mod_refprobs[0] = 0; // This branch implicit
2830 } else if (pred_ref == LAST_FRAME) {
2831 norm_cnt[0] = intra_count;
2832 norm_cnt[1] = 0;
2833 norm_cnt[2] = gf_count;
2834 norm_cnt[3] = arf_count;
2835 vp9_calc_ref_probs(norm_cnt, mod_refprobs);
2836 mod_refprobs[1] = 0; // This branch implicit
2837 } else if (pred_ref == GOLDEN_FRAME) {
2838 norm_cnt[0] = intra_count;
2839 norm_cnt[1] = last_count;
2840 norm_cnt[2] = 0;
2841 norm_cnt[3] = arf_count;
2842 vp9_calc_ref_probs(norm_cnt, mod_refprobs);
2843 mod_refprobs[2] = 0; // This branch implicit
2844 } else {
2845 norm_cnt[0] = intra_count;
2846 norm_cnt[1] = last_count;
2847 norm_cnt[2] = gf_count;
2848 norm_cnt[3] = 0;
2849 vp9_calc_ref_probs(norm_cnt, mod_refprobs);
2850 mod_refprobs[2] = 0; // This branch implicit
2851 }
2852 }
2853
2854 static __inline unsigned weighted_cost(vp9_prob *tab0, vp9_prob *tab1, int idx, int val, int weight) {
2855 unsigned cost0 = tab0[idx] ? vp9_cost_bit(tab0[idx], val) : 0;
2856 unsigned cost1 = tab1[idx] ? vp9_cost_bit(tab1[idx], val) : 0;
2857 // weight is 16-bit fixed point, so this basically calculates:
2858 // 0.5 + weight * cost1 + (1.0 - weight) * cost0
2859 return (0x8000 + weight * cost1 + (0x10000 - weight) * cost0) >> 16;
2860 }
2861
2862 static void estimate_ref_frame_costs(VP9_COMP *cpi, int segment_id, unsigned int *ref_costs) {
2863 VP9_COMMON *cm = &cpi->common;
2864 MACROBLOCKD *xd = &cpi->mb.e_mbd;
2865 vp9_prob *mod_refprobs;
2866
2867 unsigned int cost;
2868 int pred_ref;
2869 int pred_flag;
2870 int pred_ctx;
2871 int i;
2872 int tot_count;
2873
2874 vp9_prob pred_prob, new_pred_prob;
2875 int seg_ref_active;
2876 int seg_ref_count = 0;
2877 seg_ref_active = vp9_segfeature_active(xd,
2878 segment_id,
2879 SEG_LVL_REF_FRAME);
2880
2881 if (seg_ref_active) {
2882 seg_ref_count = vp9_check_segref(xd, segment_id, INTRA_FRAME) +
2883 vp9_check_segref(xd, segment_id, LAST_FRAME) +
2884 vp9_check_segref(xd, segment_id, GOLDEN_FRAME) +
2885 vp9_check_segref(xd, segment_id, ALTREF_FRAME);
2886 }
2887
2888 // Get the predicted reference for this mb
2889 pred_ref = vp9_get_pred_ref(cm, xd);
2890
2891 // Get the context probability for the prediction flag (based on last frame)
2892 pred_prob = vp9_get_pred_prob(cm, xd, PRED_REF);
2893
2894 // Predict probability for current frame based on stats so far
2895 pred_ctx = vp9_get_pred_context(cm, xd, PRED_REF);
2896 tot_count = cpi->ref_pred_count[pred_ctx][0] + cpi->ref_pred_count[pred_ctx][1 ];
2897 if (tot_count) {
2898 new_pred_prob =
2899 (cpi->ref_pred_count[pred_ctx][0] * 255 + (tot_count >> 1)) / tot_count;
2900 new_pred_prob += !new_pred_prob;
2901 } else
2902 new_pred_prob = 128;
2903
2904 // Get the set of probabilities to use if prediction fails
2905 mod_refprobs = cm->mod_refprobs[pred_ref];
2906
2907 // For each possible selected reference frame work out a cost.
2908 for (i = 0; i < MAX_REF_FRAMES; i++) {
2909 if (seg_ref_active && seg_ref_count == 1) {
2910 cost = 0;
2911 } else {
2912 pred_flag = (i == pred_ref);
2913
2914 // Get the prediction for the current mb
2915 cost = weighted_cost(&pred_prob, &new_pred_prob, 0,
2916 pred_flag, cpi->seg0_progress);
2917 if (cost > 1024) cost = 768; // i.e. account for 4 bits max.
2918
2919 // for incorrectly predicted cases
2920 if (! pred_flag) {
2921 vp9_prob curframe_mod_refprobs[3];
2922
2923 if (cpi->seg0_progress) {
2924 estimate_curframe_refprobs(cpi, curframe_mod_refprobs, pred_ref);
2925 } else {
2926 vpx_memset(curframe_mod_refprobs, 0, sizeof(curframe_mod_refprobs));
2927 }
2928
2929 cost += weighted_cost(mod_refprobs, curframe_mod_refprobs, 0,
2930 (i != INTRA_FRAME), cpi->seg0_progress);
2931 if (i != INTRA_FRAME) {
2932 cost += weighted_cost(mod_refprobs, curframe_mod_refprobs, 1,
2933 (i != LAST_FRAME), cpi->seg0_progress);
2934 if (i != LAST_FRAME) {
2935 cost += weighted_cost(mod_refprobs, curframe_mod_refprobs, 2,
2936 (i != GOLDEN_FRAME), cpi->seg0_progress);
2937 }
2938 }
2939 }
2940 }
2941
2942 ref_costs[i] = cost;
2943 }
2944 }
2945
2946 static void store_coding_context(MACROBLOCK *x, PICK_MODE_CONTEXT *ctx,
2947 int mode_index,
2948 PARTITION_INFO *partition,
2949 int_mv *ref_mv,
2950 int_mv *second_ref_mv,
2951 int64_t comp_pred_diff[NB_PREDICTION_TYPES],
2952 int64_t txfm_size_diff[NB_TXFM_MODES]) {
2953 MACROBLOCKD *const xd = &x->e_mbd;
2954
2955 // Take a snapshot of the coding context so it can be
2956 // restored if we decide to encode this way
2957 ctx->best_mode_index = mode_index;
2958 vpx_memcpy(&ctx->mic, xd->mode_info_context,
2959 sizeof(MODE_INFO));
2960 if (partition)
2961 vpx_memcpy(&ctx->partition_info, partition,
2962 sizeof(PARTITION_INFO));
2963 ctx->best_ref_mv.as_int = ref_mv->as_int;
2964 ctx->second_best_ref_mv.as_int = second_ref_mv->as_int;
2965
2966 // ctx[mb_index].rddiv = x->rddiv;
2967 // ctx[mb_index].rdmult = x->rdmult;
2968
2969 ctx->single_pred_diff = comp_pred_diff[SINGLE_PREDICTION_ONLY];
2970 ctx->comp_pred_diff = comp_pred_diff[COMP_PREDICTION_ONLY];
2971 ctx->hybrid_pred_diff = comp_pred_diff[HYBRID_PREDICTION];
2972
2973 memcpy(ctx->txfm_rd_diff, txfm_size_diff, sizeof(ctx->txfm_rd_diff));
2974 }
2975
2976 static void inter_mode_cost(VP9_COMP *cpi, MACROBLOCK *x,
2977 int *rate2, int *distortion2, int *rate_y,
2978 int *distortion, int* rate_uv, int *distortion_uv,
2979 int *skippable, int64_t txfm_cache[NB_TXFM_MODES]) {
2980 int y_skippable, uv_skippable;
2981
2982 // Y cost and distortion
2983 macro_block_yrd(cpi, x, rate_y, distortion, &y_skippable, txfm_cache);
2984
2985 *rate2 += *rate_y;
2986 *distortion2 += *distortion;
2987
2988 // UV cost and distortion
2989 vp9_subtract_mbuv(x->src_diff, x->src.u_buffer, x->src.v_buffer,
2990 x->e_mbd.predictor, x->src.uv_stride);
2991 if (x->e_mbd.mode_info_context->mbmi.txfm_size != TX_4X4)
2992 rd_inter16x16_uv_8x8(cpi, x, rate_uv, distortion_uv,
2993 cpi->common.full_pixel, &uv_skippable, 1);
2994 else
2995 rd_inter16x16_uv_4x4(cpi, x, rate_uv, distortion_uv,
2996 cpi->common.full_pixel, &uv_skippable, 1);
2997
2998 *rate2 += *rate_uv;
2999 *distortion2 += *distortion_uv;
3000 *skippable = y_skippable && uv_skippable;
3001 }
3002
3003 #define MIN(x,y) (((x)<(y))?(x):(y))
3004 #define MAX(x,y) (((x)>(y))?(x):(y))
3005 static void setup_buffer_inter(VP9_COMP *cpi, MACROBLOCK *x,
3006 int idx, MV_REFERENCE_FRAME frame_type,
3007 int block_size,
3008 int recon_yoffset, int recon_uvoffset,
3009 int_mv frame_nearest_mv[MAX_REF_FRAMES],
3010 int_mv frame_near_mv[MAX_REF_FRAMES],
3011 int_mv frame_best_ref_mv[MAX_REF_FRAMES],
3012 int_mv mv_search_ref[MAX_REF_FRAMES],
3013 int frame_mdcounts[4][4],
3014 unsigned char *y_buffer[4],
3015 unsigned char *u_buffer[4],
3016 unsigned char *v_buffer[4]) {
3017 YV12_BUFFER_CONFIG *yv12 = &cpi->common.yv12_fb[idx];
3018 MACROBLOCKD *const xd = &x->e_mbd;
3019 MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
3020
3021 y_buffer[frame_type] = yv12->y_buffer + recon_yoffset;
3022 u_buffer[frame_type] = yv12->u_buffer + recon_uvoffset;
3023 v_buffer[frame_type] = yv12->v_buffer + recon_uvoffset;
3024
3025 // Gets an initial list of candidate vectors from neighbours and orders them
3026 vp9_find_mv_refs(xd, xd->mode_info_context,
3027 xd->prev_mode_info_context,
3028 frame_type,
3029 mbmi->ref_mvs[frame_type],
3030 cpi->common.ref_frame_sign_bias);
3031
3032 // Candidate refinement carried out at encoder and decoder
3033 vp9_find_best_ref_mvs(xd, y_buffer[frame_type],
3034 yv12->y_stride,
3035 mbmi->ref_mvs[frame_type],
3036 &frame_best_ref_mv[frame_type],
3037 &frame_nearest_mv[frame_type],
3038 &frame_near_mv[frame_type]);
3039
3040
3041 // Further refinement that is encode side only to test the top few candidates
3042 // in full and choose the best as the centre point for subsequent searches.
3043 mv_pred(cpi, x, y_buffer[frame_type], yv12->y_stride,
3044 &mv_search_ref[frame_type], frame_type, block_size);
3045
3046 #if CONFIG_NEW_MVREF
3047 // TODO(paulwilkins): Final choice of which of the best 4 candidates from
3048 // above gives lowest error score when used in isolation. This stage encoder
3049 // and sets the reference MV
3050 #endif
3051 }
3052
3053 static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
3054 enum BlockSize block_size,
3055 int *saddone, int near_sadidx[],
3056 int mdcounts[4], int64_t txfm_cache[],
3057 int *rate2, int *distortion, int *skippable,
3058 int *compmode_cost,
3059 #if CONFIG_COMP_INTERINTRA_PRED
3060 int *compmode_interintra_cost,
3061 #endif
3062 int *rate_y, int *distortion_y,
3063 int *rate_uv, int *distortion_uv,
3064 int *mode_excluded, int *disable_skip,
3065 int recon_yoffset, int mode_index,
3066 int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES],
3067 int_mv frame_best_ref_mv[MAX_REF_FRAMES],
3068 int_mv mv_search_ref[MAX_REF_FRAMES]) {
3069 VP9_COMMON *cm = &cpi->common;
3070 MACROBLOCKD *xd = &x->e_mbd;
3071 MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
3072 BLOCK *b = &x->block[0];
3073 BLOCKD *d = &xd->block[0];
3074 const int is_comp_pred = (mbmi->second_ref_frame > 0);
3075 #if CONFIG_COMP_INTERINTRA_PRED
3076 const int is_comp_interintra_pred = (mbmi->second_ref_frame == INTRA_FRAME);
3077 #endif
3078 const int num_refs = is_comp_pred ? 2 : 1;
3079 const int this_mode = mbmi->mode;
3080 int i;
3081 int refs[2] = { mbmi->ref_frame,
3082 (mbmi->second_ref_frame < 0 ? 0 : mbmi->second_ref_frame) };
3083 int_mv cur_mv[2];
3084 int64_t this_rd = 0;
3085
3086 switch (this_mode) {
3087 case NEWMV:
3088 if (is_comp_pred) {
3089 if (frame_mv[NEWMV][refs[0]].as_int == INVALID_MV ||
3090 frame_mv[NEWMV][refs[1]].as_int == INVALID_MV)
3091 return INT64_MAX;
3092 *rate2 += vp9_mv_bit_cost(&frame_mv[NEWMV][refs[0]],
3093 &frame_best_ref_mv[refs[0]],
3094 x->nmvjointcost, x->mvcost, 96,
3095 x->e_mbd.allow_high_precision_mv);
3096 *rate2 += vp9_mv_bit_cost(&frame_mv[NEWMV][refs[1]],
3097 &frame_best_ref_mv[refs[1]],
3098 x->nmvjointcost, x->mvcost, 96,
3099 x->e_mbd.allow_high_precision_mv);
3100 } else {
3101 int bestsme = INT_MAX;
3102 int further_steps, step_param = cpi->sf.first_step;
3103 int sadpb = x->sadperbit16;
3104 int_mv mvp_full, tmp_mv;
3105 int sr = 0;
3106
3107 int tmp_col_min = x->mv_col_min;
3108 int tmp_col_max = x->mv_col_max;
3109 int tmp_row_min = x->mv_row_min;
3110 int tmp_row_max = x->mv_row_max;
3111
3112 vp9_clamp_mv_min_max(x, &frame_best_ref_mv[refs[0]]);
3113
3114 mvp_full.as_mv.col = mv_search_ref[mbmi->ref_frame].as_mv.col >> 3;
3115 mvp_full.as_mv.row = mv_search_ref[mbmi->ref_frame].as_mv.row >> 3;
3116
3117 // adjust search range according to sr from mv prediction
3118 step_param = MAX(step_param, sr);
3119
3120 // Further step/diamond searches as necessary
3121 further_steps = (cpi->sf.max_step_search_steps - 1) - step_param;
3122
3123 bestsme = vp9_full_pixel_diamond(cpi, x, b, d, &mvp_full, step_param,
3124 sadpb, further_steps, 1,
3125 &cpi->fn_ptr[block_size],
3126 &frame_best_ref_mv[refs[0]], &tmp_mv);
3127
3128 x->mv_col_min = tmp_col_min;
3129 x->mv_col_max = tmp_col_max;
3130 x->mv_row_min = tmp_row_min;
3131 x->mv_row_max = tmp_row_max;
3132
3133 if (bestsme < INT_MAX) {
3134 int dis; /* TODO: use dis in distortion calculation later. */
3135 unsigned int sse;
3136 cpi->find_fractional_mv_step(x, b, d, &tmp_mv,
3137 &frame_best_ref_mv[refs[0]],
3138 x->errorperbit,
3139 &cpi->fn_ptr[block_size],
3140 x->nmvjointcost, x->mvcost,
3141 &dis, &sse);
3142 }
3143 d->bmi.as_mv.first.as_int = tmp_mv.as_int;
3144 frame_mv[NEWMV][refs[0]].as_int = d->bmi.as_mv.first.as_int;
3145
3146 // Add the new motion vector cost to our rolling cost variable
3147 *rate2 += vp9_mv_bit_cost(&tmp_mv, &frame_best_ref_mv[refs[0]],
3148 x->nmvjointcost, x->mvcost,
3149 96, xd->allow_high_precision_mv);
3150 }
3151 break;
3152 case NEARESTMV:
3153 case NEARMV:
3154 // Do not bother proceeding if the vector (from newmv, nearest or
3155 // near) is 0,0 as this should then be coded using the zeromv mode.
3156 for (i = 0; i < num_refs; ++i)
3157 if (frame_mv[this_mode][refs[i]].as_int == 0)
3158 return INT64_MAX;
3159 case ZEROMV:
3160 default:
3161 break;
3162 }
3163 for (i = 0; i < num_refs; ++i) {
3164 cur_mv[i] = frame_mv[this_mode][refs[i]];
3165 // Clip "next_nearest" so that it does not extend to far out of image
3166 clamp_mv2(&cur_mv[i], xd);
3167 if (mv_check_bounds(x, &cur_mv[i]))
3168 return INT64_MAX;
3169 mbmi->mv[i].as_int = cur_mv[i].as_int;
3170 }
3171
3172 #if CONFIG_PRED_FILTER
3173 // Filtered prediction:
3174 mbmi->pred_filter_enabled = vp9_mode_order[mode_index].pred_filter_flag;
3175 *rate2 += vp9_cost_bit(cpi->common.prob_pred_filter_off,
3176 mbmi->pred_filter_enabled);
3177 #endif
3178 if (cpi->common.mcomp_filter_type == SWITCHABLE) {
3179 const int c = vp9_get_pred_context(cm, xd, PRED_SWITCHABLE_INTERP);
3180 const int m = vp9_switchable_interp_map[mbmi->interp_filter];
3181 *rate2 += SWITCHABLE_INTERP_RATE_FACTOR * x->switchable_interp_costs[c][m];
3182 }
3183
3184 /* We don't include the cost of the second reference here, because there
3185 * are only three options: Last/Golden, ARF/Last or Golden/ARF, or in other
3186 * words if you present them in that order, the second one is always known
3187 * if the first is known */
3188 *compmode_cost = vp9_cost_bit(vp9_get_pred_prob(cm, xd, PRED_COMP),
3189 is_comp_pred);
3190 *rate2 += vp9_cost_mv_ref(cpi, this_mode,
3191 mbmi->mb_mode_context[mbmi->ref_frame]);
3192 #if CONFIG_COMP_INTERINTRA_PRED
3193 if (!is_comp_pred) {
3194 *compmode_interintra_cost = vp9_cost_bit(cm->fc.interintra_prob,
3195 is_comp_interintra_pred);
3196 if (is_comp_interintra_pred) {
3197 *compmode_interintra_cost +=
3198 x->mbmode_cost[xd->frame_type][mbmi->interintra_mode];
3199 #if SEPARATE_INTERINTRA_UV
3200 *compmode_interintra_cost +=
3201 x->intra_uv_mode_cost[xd->frame_type][mbmi->interintra_uv_mode];
3202 #endif
3203 }
3204 }
3205 #endif
3206
3207 if (block_size == BLOCK_16X16) {
3208 vp9_build_1st_inter16x16_predictors_mby(xd, xd->predictor, 16, 0);
3209 if (is_comp_pred)
3210 vp9_build_2nd_inter16x16_predictors_mby(xd, xd->predictor, 16);
3211 #if CONFIG_COMP_INTERINTRA_PRED
3212 if (is_comp_interintra_pred) {
3213 vp9_build_interintra_16x16_predictors_mby(xd, xd->predictor, 16);
3214 }
3215 #endif
3216 } else {
3217 #if CONFIG_SUPERBLOCKS
3218 vp9_build_inter32x32_predictors_sb(xd,
3219 xd->dst.y_buffer,
3220 xd->dst.u_buffer,
3221 xd->dst.v_buffer,
3222 xd->dst.y_stride,
3223 xd->dst.uv_stride);
3224 #endif
3225 }
3226
3227 if (cpi->active_map_enabled && x->active_ptr[0] == 0)
3228 x->skip = 1;
3229 else if (x->encode_breakout) {
3230 unsigned int sse, var;
3231 int threshold = (xd->block[0].dequant[1]
3232 * xd->block[0].dequant[1] >> 4);
3233
3234 if (threshold < x->encode_breakout)
3235 threshold = x->encode_breakout;
3236
3237 if (block_size == BLOCK_16X16) {
3238 var = vp9_variance16x16(*(b->base_src), b->src_stride,
3239 xd->predictor, 16, &sse);
3240 } else {
3241 #if CONFIG_SUPERBLOCKS
3242 var = vp9_variance32x32(*(b->base_src), b->src_stride,
3243 xd->dst.y_buffer, xd->dst.y_stride, &sse);
3244 #endif
3245 }
3246
3247 if ((int)sse < threshold) {
3248 unsigned int q2dc = xd->block[24].dequant[0];
3249 /* If there is no codeable 2nd order dc
3250 or a very small uniform pixel change change */
3251 if ((sse - var < q2dc * q2dc >> 4) ||
3252 (sse / 2 > var && sse - var < 64)) {
3253 // Check u and v to make sure skip is ok
3254 int sse2;
3255
3256 if (block_size == BLOCK_16X16) {
3257 sse2 = vp9_uvsse(x);
3258 } else {
3259 unsigned int sse2u, sse2v;
3260 var = vp9_variance16x16(x->src.u_buffer, x->src.uv_stride,
3261 xd->dst.u_buffer, xd->dst.uv_stride, &sse2u);
3262 var = vp9_variance16x16(x->src.v_buffer, x->src.uv_stride,
3263 xd->dst.v_buffer, xd->dst.uv_stride, &sse2v);
3264 sse2 = sse2u + sse2v;
3265 }
3266
3267 if (sse2 * 2 < threshold) {
3268 x->skip = 1;
3269 *distortion = sse + sse2;
3270 *rate2 = 500;
3271
3272 /* for best_yrd calculation */
3273 *rate_uv = 0;
3274 *distortion_uv = sse2;
3275
3276 *disable_skip = 1;
3277 this_rd = RDCOST(x->rdmult, x->rddiv, *rate2, *distortion);
3278 }
3279 }
3280 }
3281 }
3282
3283 if (is_comp_pred) {
3284 *mode_excluded = (cpi->common.comp_pred_mode == SINGLE_PREDICTION_ONLY);
3285 } else {
3286 *mode_excluded = (cpi->common.comp_pred_mode == COMP_PREDICTION_ONLY);
3287 }
3288 #if CONFIG_COMP_INTERINTRA_PRED
3289 if (is_comp_interintra_pred && !cm->use_interintra) *mode_excluded = 1;
3290 #endif
3291
3292 if (!x->skip) {
3293 if (block_size == BLOCK_16X16) {
3294 vp9_build_1st_inter16x16_predictors_mbuv(xd, &xd->predictor[256],
3295 &xd->predictor[320], 8);
3296 if (is_comp_pred)
3297 vp9_build_2nd_inter16x16_predictors_mbuv(xd, &xd->predictor[256],
3298 &xd->predictor[320], 8);
3299 #if CONFIG_COMP_INTERINTRA_PRED
3300 if (is_comp_interintra_pred) {
3301 vp9_build_interintra_16x16_predictors_mbuv(xd, &xd->predictor[256],
3302 &xd->predictor[320], 8);
3303 }
3304 #endif
3305 inter_mode_cost(cpi, x, rate2, distortion,
3306 rate_y, distortion_y, rate_uv, distortion_uv,
3307 skippable, txfm_cache);
3308 } else {
3309 #if CONFIG_SUPERBLOCKS
3310 int skippable_y, skippable_uv;
3311
3312 // Y cost and distortion
3313 super_block_yrd(cpi, x, rate_y, distortion_y,
3314 &skippable_y, txfm_cache);
3315 *rate2 += *rate_y;
3316 *distortion += *distortion_y;
3317
3318 rd_inter32x32_uv(cpi, x, rate_uv, distortion_uv,
3319 cm->full_pixel, &skippable_uv);
3320
3321 *rate2 += *rate_uv;
3322 *distortion += *distortion_uv;
3323 *skippable = skippable_y && skippable_uv;
3324 #endif
3325 }
3326 }
3327 return this_rd; // if 0, this will be re-calculated by caller
3328 }
3329
3330 static void rd_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
3331 int recon_yoffset, int recon_uvoffset,
3332 int *returnrate, int *returndistortion,
3333 int64_t *returnintra) {
3334 VP9_COMMON *cm = &cpi->common;
3335 MACROBLOCKD *xd = &x->e_mbd;
3336 union b_mode_info best_bmodes[16];
3337 MB_MODE_INFO best_mbmode;
3338 PARTITION_INFO best_partition;
3339 int_mv best_ref_mv, second_best_ref_mv;
3340 MB_PREDICTION_MODE this_mode;
3341 MB_MODE_INFO * mbmi = &xd->mode_info_context->mbmi;
3342 int i, best_mode_index = 0;
3343 int mode8x8[2][4];
3344 unsigned char segment_id = mbmi->segment_id;
3345
3346 int mode_index;
3347 int mdcounts[4];
3348 int rate, distortion;
3349 int rate2, distortion2;
3350 int64_t best_txfm_rd[NB_TXFM_MODES];
3351 int64_t best_txfm_diff[NB_TXFM_MODES];
3352 int64_t best_pred_diff[NB_PREDICTION_TYPES];
3353 int64_t best_pred_rd[NB_PREDICTION_TYPES];
3354 int64_t best_rd = INT64_MAX, best_intra_rd = INT64_MAX;
3355 #if CONFIG_COMP_INTERINTRA_PRED
3356 int is_best_interintra = 0;
3357 int64_t best_intra16_rd = INT64_MAX;
3358 int best_intra16_mode = DC_PRED, best_intra16_uv_mode = DC_PRED;
3359 #endif
3360 int64_t best_overall_rd = INT64_MAX;
3361 int uv_intra_rate, uv_intra_distortion, uv_intra_rate_tokenonly;
3362 int uv_intra_skippable = 0;
3363 int uv_intra_rate_8x8 = 0, uv_intra_distortion_8x8 = 0, uv_intra_rate_tokenonl y_8x8 = 0;
3364 int uv_intra_skippable_8x8 = 0;
3365 int rate_y, UNINITIALIZED_IS_SAFE(rate_uv);
3366 int distortion_uv = INT_MAX;
3367 int64_t best_yrd = INT64_MAX;
3368 #if CONFIG_PRED_FILTER
3369 int best_filter_state = 0;
3370 #endif
3371 int switchable_filter_index = 0;
3372
3373 MB_PREDICTION_MODE uv_intra_mode;
3374 MB_PREDICTION_MODE uv_intra_mode_8x8 = 0;
3375
3376 int near_sadidx[8] = {0, 1, 2, 3, 4, 5, 6, 7};
3377 int saddone = 0;
3378
3379 int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];
3380 int_mv frame_best_ref_mv[MAX_REF_FRAMES];
3381 int_mv mv_search_ref[MAX_REF_FRAMES];
3382 int frame_mdcounts[4][4];
3383 unsigned char *y_buffer[4], *u_buffer[4], *v_buffer[4];
3384
3385 unsigned int ref_costs[MAX_REF_FRAMES];
3386 int_mv seg_mvs[NB_PARTITIONINGS][16 /* n_blocks */][MAX_REF_FRAMES - 1];
3387
3388 vpx_memset(mode8x8, 0, sizeof(mode8x8));
3389 vpx_memset(&frame_mv, 0, sizeof(frame_mv));
3390 vpx_memset(&best_mbmode, 0, sizeof(best_mbmode));
3391 vpx_memset(&best_bmodes, 0, sizeof(best_bmodes));
3392 vpx_memset(&x->mb_context[xd->mb_index], 0, sizeof(PICK_MODE_CONTEXT));
3393
3394 for (i = 0; i < MAX_REF_FRAMES; i++)
3395 frame_mv[NEWMV][i].as_int = INVALID_MV;
3396 for (i = 0; i < NB_PREDICTION_TYPES; ++i)
3397 best_pred_rd[i] = INT64_MAX;
3398 for (i = 0; i < NB_TXFM_MODES; i++)
3399 best_txfm_rd[i] = INT64_MAX;
3400
3401 for (i = 0; i < NB_PARTITIONINGS; i++) {
3402 int j, k;
3403
3404 for (j = 0; j < 16; j++)
3405 for (k = 0; k < MAX_REF_FRAMES - 1; k++)
3406 seg_mvs[i][j][k].as_int = INVALID_MV;
3407 }
3408
3409 if (cpi->ref_frame_flags & VP9_LAST_FLAG) {
3410 setup_buffer_inter(cpi, x, cpi->common.lst_fb_idx, LAST_FRAME,
3411 BLOCK_16X16, recon_yoffset, recon_uvoffset,
3412 frame_mv[NEARESTMV], frame_mv[NEARMV], frame_best_ref_mv,
3413 mv_search_ref, frame_mdcounts,
3414 y_buffer, u_buffer, v_buffer);
3415 }
3416
3417 if (cpi->ref_frame_flags & VP9_GOLD_FLAG) {
3418 setup_buffer_inter(cpi, x, cpi->common.gld_fb_idx, GOLDEN_FRAME,
3419 BLOCK_16X16, recon_yoffset, recon_uvoffset,
3420 frame_mv[NEARESTMV], frame_mv[NEARMV], frame_best_ref_mv,
3421 mv_search_ref, frame_mdcounts,
3422 y_buffer, u_buffer, v_buffer);
3423 }
3424
3425 if (cpi->ref_frame_flags & VP9_ALT_FLAG) {
3426 setup_buffer_inter(cpi, x, cpi->common.alt_fb_idx, ALTREF_FRAME,
3427 BLOCK_16X16, recon_yoffset, recon_uvoffset,
3428 frame_mv[NEARESTMV], frame_mv[NEARMV], frame_best_ref_mv,
3429 mv_search_ref, frame_mdcounts,
3430 y_buffer, u_buffer, v_buffer);
3431 }
3432
3433 *returnintra = INT64_MAX;
3434
3435 x->skip = 0;
3436
3437 mbmi->ref_frame = INTRA_FRAME;
3438
3439 /* Initialize zbin mode boost for uv costing */
3440 cpi->zbin_mode_boost = 0;
3441 vp9_update_zbin_extra(cpi, x);
3442
3443 rd_pick_intra_mbuv_mode(cpi, x, &uv_intra_rate,
3444 &uv_intra_rate_tokenonly, &uv_intra_distortion,
3445 &uv_intra_skippable);
3446 uv_intra_mode = mbmi->uv_mode;
3447
3448 /* rough estimate for now */
3449 if (cpi->common.txfm_mode != ONLY_4X4) {
3450 rd_pick_intra_mbuv_mode_8x8(cpi, x, &uv_intra_rate_8x8,
3451 &uv_intra_rate_tokenonly_8x8,
3452 &uv_intra_distortion_8x8,
3453 &uv_intra_skippable_8x8);
3454 uv_intra_mode_8x8 = mbmi->uv_mode;
3455 }
3456
3457 // Get estimates of reference frame costs for each reference frame
3458 // that depend on the current prediction etc.
3459 estimate_ref_frame_costs(cpi, segment_id, ref_costs);
3460
3461 for (mode_index = 0; mode_index < MAX_MODES;
3462 mode_index += (!switchable_filter_index)) {
3463 int64_t this_rd = INT64_MAX;
3464 int disable_skip = 0, skippable = 0;
3465 int other_cost = 0;
3466 int compmode_cost = 0;
3467 #if CONFIG_COMP_INTERINTRA_PRED
3468 int compmode_interintra_cost = 0;
3469 #endif
3470 int mode_excluded = 0;
3471 int64_t txfm_cache[NB_TXFM_MODES] = { 0 };
3472
3473 // These variables hold are rolling total cost and distortion for this mode
3474 rate2 = 0;
3475 distortion2 = 0;
3476 rate_y = 0;
3477 rate_uv = 0;
3478
3479 this_mode = vp9_mode_order[mode_index].mode;
3480 mbmi->mode = this_mode;
3481 mbmi->uv_mode = DC_PRED;
3482 mbmi->ref_frame = vp9_mode_order[mode_index].ref_frame;
3483 mbmi->second_ref_frame = vp9_mode_order[mode_index].second_ref_frame;
3484 #if CONFIG_PRED_FILTER
3485 mbmi->pred_filter_enabled = 0;
3486 #endif
3487 if (cpi->common.mcomp_filter_type == SWITCHABLE &&
3488 this_mode >= NEARESTMV && this_mode <= SPLITMV) {
3489 mbmi->interp_filter =
3490 vp9_switchable_interp[switchable_filter_index++];
3491 if (switchable_filter_index == VP9_SWITCHABLE_FILTERS)
3492 switchable_filter_index = 0;
3493 } else {
3494 mbmi->interp_filter = cpi->common.mcomp_filter_type;
3495 }
3496 vp9_setup_interp_filters(xd, mbmi->interp_filter, &cpi->common);
3497
3498 // Test best rd so far against threshold for trying this mode.
3499 if (best_rd <= cpi->rd_threshes[mode_index])
3500 continue;
3501
3502 // current coding mode under rate-distortion optimization test loop
3503 #if CONFIG_COMP_INTRA_PRED
3504 mbmi->second_mode = (MB_PREDICTION_MODE)(DC_PRED - 1);
3505 mbmi->second_uv_mode = (MB_PREDICTION_MODE)(DC_PRED - 1);
3506 #endif
3507 #if CONFIG_COMP_INTERINTRA_PRED
3508 mbmi->interintra_mode = (MB_PREDICTION_MODE)(DC_PRED - 1);
3509 mbmi->interintra_uv_mode = (MB_PREDICTION_MODE)(DC_PRED - 1);
3510 #endif
3511
3512 // If the segment reference frame feature is enabled....
3513 // then do nothing if the current ref frame is not allowed..
3514 if (vp9_segfeature_active(xd, segment_id, SEG_LVL_REF_FRAME) &&
3515 !vp9_check_segref(xd, segment_id, mbmi->ref_frame)) {
3516 continue;
3517 // If the segment mode feature is enabled....
3518 // then do nothing if the current mode is not allowed..
3519 } else if (vp9_segfeature_active(xd, segment_id, SEG_LVL_MODE) &&
3520 (this_mode !=
3521 vp9_get_segdata(xd, segment_id, SEG_LVL_MODE))) {
3522 continue;
3523 // Disable this drop out case if either the mode or ref frame
3524 // segment level feature is enabled for this segment. This is to
3525 // prevent the possibility that the we end up unable to pick any mode.
3526 } else if (!vp9_segfeature_active(xd, segment_id, SEG_LVL_REF_FRAME) &&
3527 !vp9_segfeature_active(xd, segment_id, SEG_LVL_MODE)) {
3528 // Only consider ZEROMV/ALTREF_FRAME for alt ref frame,
3529 // unless ARNR filtering is enabled in which case we want
3530 // an unfiltered alternative
3531 if (cpi->is_src_frame_alt_ref && (cpi->oxcf.arnr_max_frames == 0)) {
3532 if (this_mode != ZEROMV ||
3533 mbmi->ref_frame != ALTREF_FRAME) {
3534 continue;
3535 }
3536 }
3537 }
3538
3539 /* everything but intra */
3540 if (mbmi->ref_frame) {
3541 int ref = mbmi->ref_frame;
3542
3543 xd->pre.y_buffer = y_buffer[ref];
3544 xd->pre.u_buffer = u_buffer[ref];
3545 xd->pre.v_buffer = v_buffer[ref];
3546 best_ref_mv = frame_best_ref_mv[ref];
3547 vpx_memcpy(mdcounts, frame_mdcounts[ref], sizeof(mdcounts));
3548 }
3549
3550 if (mbmi->second_ref_frame > 0) {
3551 int ref = mbmi->second_ref_frame;
3552
3553 xd->second_pre.y_buffer = y_buffer[ref];
3554 xd->second_pre.u_buffer = u_buffer[ref];
3555 xd->second_pre.v_buffer = v_buffer[ref];
3556 second_best_ref_mv = frame_best_ref_mv[ref];
3557 }
3558
3559 // Experimental code. Special case for gf and arf zeromv modes.
3560 // Increase zbin size to suppress noise
3561 if (cpi->zbin_mode_boost_enabled) {
3562 if (vp9_mode_order[mode_index].ref_frame == INTRA_FRAME)
3563 cpi->zbin_mode_boost = 0;
3564 else {
3565 if (vp9_mode_order[mode_index].mode == ZEROMV) {
3566 if (vp9_mode_order[mode_index].ref_frame != LAST_FRAME)
3567 cpi->zbin_mode_boost = GF_ZEROMV_ZBIN_BOOST;
3568 else
3569 cpi->zbin_mode_boost = LF_ZEROMV_ZBIN_BOOST;
3570 } else if (vp9_mode_order[mode_index].mode == SPLITMV)
3571 cpi->zbin_mode_boost = 0;
3572 else
3573 cpi->zbin_mode_boost = MV_ZBIN_BOOST;
3574 }
3575
3576 vp9_update_zbin_extra(cpi, x);
3577 }
3578
3579 // Intra
3580 if (!mbmi->ref_frame) {
3581 switch (this_mode) {
3582 default:
3583 case DC_PRED:
3584 case V_PRED:
3585 case H_PRED:
3586 case TM_PRED:
3587 case D45_PRED:
3588 case D135_PRED:
3589 case D117_PRED:
3590 case D153_PRED:
3591 case D27_PRED:
3592 case D63_PRED:
3593 mbmi->ref_frame = INTRA_FRAME;
3594 // FIXME compound intra prediction
3595 vp9_build_intra_predictors_mby(&x->e_mbd);
3596 macro_block_yrd(cpi, x, &rate_y, &distortion, &skippable, txfm_cache);
3597 rate2 += rate_y;
3598 distortion2 += distortion;
3599 rate2 += x->mbmode_cost[xd->frame_type][mbmi->mode];
3600 if (mbmi->txfm_size != TX_4X4) {
3601 rate2 += uv_intra_rate_8x8;
3602 rate_uv = uv_intra_rate_tokenonly_8x8;
3603 distortion2 += uv_intra_distortion_8x8;
3604 distortion_uv = uv_intra_distortion_8x8;
3605 skippable = skippable && uv_intra_skippable_8x8;
3606 } else {
3607 rate2 += uv_intra_rate;
3608 rate_uv = uv_intra_rate_tokenonly;
3609 distortion2 += uv_intra_distortion;
3610 distortion_uv = uv_intra_distortion;
3611 skippable = skippable && uv_intra_skippable;
3612 }
3613 break;
3614 case B_PRED: {
3615 int64_t tmp_rd;
3616
3617 // Note the rate value returned here includes the cost of coding
3618 // the BPRED mode : x->mbmode_cost[xd->frame_type][BPRED];
3619 mbmi->txfm_size = TX_4X4;
3620 tmp_rd = rd_pick_intra4x4mby_modes(cpi, x, &rate, &rate_y, &distortion , best_yrd,
3621 #if CONFIG_COMP_INTRA_PRED
3622 0,
3623 #endif
3624 0);
3625 rate2 += rate;
3626 distortion2 += distortion;
3627
3628 if (tmp_rd < best_yrd) {
3629 rate2 += uv_intra_rate;
3630 rate_uv = uv_intra_rate_tokenonly;
3631 distortion2 += uv_intra_distortion;
3632 distortion_uv = uv_intra_distortion;
3633 } else {
3634 this_rd = INT64_MAX;
3635 disable_skip = 1;
3636 }
3637 }
3638 break;
3639 case I8X8_PRED: {
3640 int cost0 = vp9_cost_bit(cm->prob_tx[0], 0);
3641 int cost1 = vp9_cost_bit(cm->prob_tx[0], 1);
3642 int64_t tmp_rd_4x4s, tmp_rd_8x8s;
3643 int64_t tmp_rd_4x4, tmp_rd_8x8, tmp_rd;
3644 int r4x4, tok4x4, d4x4, r8x8, tok8x8, d8x8;
3645 mbmi->txfm_size = TX_4X4;
3646 tmp_rd_4x4 = rd_pick_intra8x8mby_modes(cpi, x, &r4x4, &tok4x4,
3647 &d4x4, best_yrd);
3648 mode8x8[0][0] = xd->mode_info_context->bmi[0].as_mode.first;
3649 mode8x8[0][1] = xd->mode_info_context->bmi[2].as_mode.first;
3650 mode8x8[0][2] = xd->mode_info_context->bmi[8].as_mode.first;
3651 mode8x8[0][3] = xd->mode_info_context->bmi[10].as_mode.first;
3652 #if CONFIG_COMP_INTRA_PRED
3653 mode8x8[1][0] = xd->mode_info_context->bmi[0].as_mode.second;
3654 mode8x8[1][1] = xd->mode_info_context->bmi[2].as_mode.second;
3655 mode8x8[1][2] = xd->mode_info_context->bmi[8].as_mode.second;
3656 mode8x8[1][3] = xd->mode_info_context->bmi[10].as_mode.second;
3657 #endif
3658 mbmi->txfm_size = TX_8X8;
3659 tmp_rd_8x8 = rd_pick_intra8x8mby_modes(cpi, x, &r8x8, &tok8x8,
3660 &d8x8, best_yrd);
3661 txfm_cache[ONLY_4X4] = tmp_rd_4x4;
3662 txfm_cache[ALLOW_8X8] = tmp_rd_8x8;
3663 txfm_cache[ALLOW_16X16] = tmp_rd_8x8;
3664 tmp_rd_4x4s = tmp_rd_4x4 + RDCOST(x->rdmult, x->rddiv, cost0, 0);
3665 tmp_rd_8x8s = tmp_rd_8x8 + RDCOST(x->rdmult, x->rddiv, cost1, 0);
3666 txfm_cache[TX_MODE_SELECT] = tmp_rd_4x4s < tmp_rd_8x8s ? tmp_rd_4x4s : tmp_rd_8x8s;
3667 if (cm->txfm_mode == TX_MODE_SELECT) {
3668 if (tmp_rd_4x4s < tmp_rd_8x8s) {
3669 rate = r4x4 + cost0;
3670 rate_y = tok4x4 + cost0;
3671 distortion = d4x4;
3672 mbmi->txfm_size = TX_4X4;
3673 tmp_rd = tmp_rd_4x4s;
3674 } else {
3675 rate = r8x8 + cost1;
3676 rate_y = tok8x8 + cost1;
3677 distortion = d8x8;
3678 mbmi->txfm_size = TX_8X8;
3679 tmp_rd = tmp_rd_8x8s;
3680
3681 mode8x8[0][0] = xd->mode_info_context->bmi[0].as_mode.first;
3682 mode8x8[0][1] = xd->mode_info_context->bmi[2].as_mode.first;
3683 mode8x8[0][2] = xd->mode_info_context->bmi[8].as_mode.first;
3684 mode8x8[0][3] = xd->mode_info_context->bmi[10].as_mode.first;
3685 #if CONFIG_COMP_INTRA_PRED
3686 mode8x8[1][0] = xd->mode_info_context->bmi[0].as_mode.second;
3687 mode8x8[1][1] = xd->mode_info_context->bmi[2].as_mode.second;
3688 mode8x8[1][2] = xd->mode_info_context->bmi[8].as_mode.second;
3689 mode8x8[1][3] = xd->mode_info_context->bmi[10].as_mode.second;
3690 #endif
3691 }
3692 } else if (cm->txfm_mode == ONLY_4X4) {
3693 rate = r4x4;
3694 rate_y = tok4x4;
3695 distortion = d4x4;
3696 mbmi->txfm_size = TX_4X4;
3697 tmp_rd = tmp_rd_4x4;
3698 } else {
3699 rate = r8x8;
3700 rate_y = tok8x8;
3701 distortion = d8x8;
3702 mbmi->txfm_size = TX_8X8;
3703 tmp_rd = tmp_rd_8x8;
3704
3705 mode8x8[0][0] = xd->mode_info_context->bmi[0].as_mode.first;
3706 mode8x8[0][1] = xd->mode_info_context->bmi[2].as_mode.first;
3707 mode8x8[0][2] = xd->mode_info_context->bmi[8].as_mode.first;
3708 mode8x8[0][3] = xd->mode_info_context->bmi[10].as_mode.first;
3709 #if CONFIG_COMP_INTRA_PRED
3710 mode8x8[1][0] = xd->mode_info_context->bmi[0].as_mode.second;
3711 mode8x8[1][1] = xd->mode_info_context->bmi[2].as_mode.second;
3712 mode8x8[1][2] = xd->mode_info_context->bmi[8].as_mode.second;
3713 mode8x8[1][3] = xd->mode_info_context->bmi[10].as_mode.second;
3714 #endif
3715 }
3716
3717 rate2 += rate;
3718 distortion2 += distortion;
3719
3720 /* TODO: uv rate maybe over-estimated here since there is UV intra
3721 mode coded in I8X8_PRED prediction */
3722 if (tmp_rd < best_yrd) {
3723 rate2 += uv_intra_rate;
3724 rate_uv = uv_intra_rate_tokenonly;
3725 distortion2 += uv_intra_distortion;
3726 distortion_uv = uv_intra_distortion;
3727 } else {
3728 this_rd = INT64_MAX;
3729 disable_skip = 1;
3730 }
3731 }
3732 break;
3733 }
3734 }
3735 // Split MV. The code is very different from the other inter modes so
3736 // special case it.
3737 else if (this_mode == SPLITMV) {
3738 const int is_comp_pred = mbmi->second_ref_frame > 0;
3739 int64_t tmp_rd, this_rd_thresh;
3740 int_mv *second_ref = is_comp_pred ? &second_best_ref_mv : NULL;
3741
3742 this_rd_thresh =
3743 (mbmi->ref_frame == LAST_FRAME) ?
3744 cpi->rd_threshes[THR_NEWMV] : cpi->rd_threshes[THR_NEWA];
3745 this_rd_thresh =
3746 (mbmi->ref_frame == GOLDEN_FRAME) ?
3747 cpi->rd_threshes[THR_NEWG] : this_rd_thresh;
3748
3749 tmp_rd = rd_pick_best_mbsegmentation(cpi, x, &best_ref_mv,
3750 second_ref, best_yrd, mdcounts,
3751 &rate, &rate_y, &distortion,
3752 &skippable,
3753 (int)this_rd_thresh, seg_mvs,
3754 txfm_cache);
3755 rate2 += rate;
3756 distortion2 += distortion;
3757
3758 if (cpi->common.mcomp_filter_type == SWITCHABLE)
3759 rate2 += SWITCHABLE_INTERP_RATE_FACTOR * x->switchable_interp_costs
3760 [vp9_get_pred_context(&cpi->common, xd, PRED_SWITCHABLE_INTERP)]
3761 [vp9_switchable_interp_map[mbmi->interp_filter]];
3762 // If even the 'Y' rd value of split is higher than best so far
3763 // then dont bother looking at UV
3764 if (tmp_rd < best_yrd) {
3765 int uv_skippable;
3766
3767 rd_inter4x4_uv(cpi, x, &rate_uv, &distortion_uv, &uv_skippable,
3768 cpi->common.full_pixel);
3769 rate2 += rate_uv;
3770 distortion2 += distortion_uv;
3771 skippable = skippable && uv_skippable;
3772 } else {
3773 this_rd = INT64_MAX;
3774 disable_skip = 1;
3775 }
3776
3777 if (is_comp_pred)
3778 mode_excluded = cpi->common.comp_pred_mode == SINGLE_PREDICTION_ONLY;
3779 else
3780 mode_excluded = cpi->common.comp_pred_mode == COMP_PREDICTION_ONLY;
3781
3782 compmode_cost =
3783 vp9_cost_bit(vp9_get_pred_prob(cm, xd, PRED_COMP), is_comp_pred);
3784 mbmi->mode = this_mode;
3785 }
3786 else {
3787 #if CONFIG_COMP_INTERINTRA_PRED
3788 if (mbmi->second_ref_frame == INTRA_FRAME) {
3789 if (best_intra16_mode == DC_PRED - 1) continue;
3790 mbmi->interintra_mode = best_intra16_mode;
3791 #if SEPARATE_INTERINTRA_UV
3792 mbmi->interintra_uv_mode = best_intra16_uv_mode;
3793 #else
3794 mbmi->interintra_uv_mode = best_intra16_mode;
3795 #endif
3796 }
3797 #endif
3798 this_rd = handle_inter_mode(cpi, x, BLOCK_16X16,
3799 &saddone, near_sadidx, mdcounts, txfm_cache,
3800 &rate2, &distortion2, &skippable,
3801 &compmode_cost,
3802 #if CONFIG_COMP_INTERINTRA_PRED
3803 &compmode_interintra_cost,
3804 #endif
3805 &rate_y, &distortion,
3806 &rate_uv, &distortion_uv,
3807 &mode_excluded, &disable_skip, recon_yoffset,
3808 mode_index, frame_mv, frame_best_ref_mv,
3809 mv_search_ref);
3810 if (this_rd == INT64_MAX)
3811 continue;
3812 }
3813
3814 #if CONFIG_COMP_INTERINTRA_PRED
3815 if (cpi->common.use_interintra)
3816 rate2 += compmode_interintra_cost;
3817 #endif
3818
3819 if (cpi->common.comp_pred_mode == HYBRID_PREDICTION)
3820 rate2 += compmode_cost;
3821
3822 // Estimate the reference frame signaling cost and add it
3823 // to the rolling cost variable.
3824 rate2 += ref_costs[mbmi->ref_frame];
3825
3826 if (!disable_skip) {
3827 // Test for the condition where skip block will be activated
3828 // because there are no non zero coefficients and make any
3829 // necessary adjustment for rate. Ignore if skip is coded at
3830 // segment level as the cost wont have been added in.
3831 if (cpi->common.mb_no_coeff_skip) {
3832 int mb_skip_allowed;
3833
3834 // Is Mb level skip allowed for this mb.
3835 mb_skip_allowed =
3836 !vp9_segfeature_active(xd, segment_id, SEG_LVL_EOB) ||
3837 vp9_get_segdata(xd, segment_id, SEG_LVL_EOB);
3838
3839 if (skippable) {
3840 mbmi->mb_skip_coeff = 1;
3841
3842 // Back out the coefficient coding costs
3843 rate2 -= (rate_y + rate_uv);
3844 // for best_yrd calculation
3845 rate_uv = 0;
3846
3847 if (mb_skip_allowed) {
3848 int prob_skip_cost;
3849
3850 // Cost the skip mb case
3851 vp9_prob skip_prob =
3852 vp9_get_pred_prob(cm, &x->e_mbd, PRED_MBSKIP);
3853
3854 if (skip_prob) {
3855 prob_skip_cost = vp9_cost_bit(skip_prob, 1);
3856 rate2 += prob_skip_cost;
3857 other_cost += prob_skip_cost;
3858 }
3859 }
3860 }
3861 // Add in the cost of the no skip flag.
3862 else {
3863 mbmi->mb_skip_coeff = 0;
3864 if (mb_skip_allowed) {
3865 int prob_skip_cost = vp9_cost_bit(
3866 vp9_get_pred_prob(cm, &x->e_mbd, PRED_MBSKIP), 0);
3867 rate2 += prob_skip_cost;
3868 other_cost += prob_skip_cost;
3869 }
3870 }
3871 }
3872
3873 // Calculate the final RD estimate for this mode.
3874 this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2);
3875 }
3876
3877 // Keep record of best intra distortion
3878 if ((mbmi->ref_frame == INTRA_FRAME) &&
3879 (this_rd < best_intra_rd)) {
3880 best_intra_rd = this_rd;
3881 *returnintra = distortion2;
3882 }
3883 #if CONFIG_COMP_INTERINTRA_PRED
3884 if ((mbmi->ref_frame == INTRA_FRAME) &&
3885 (this_mode <= TM_PRED) &&
3886 (this_rd < best_intra16_rd)) {
3887 best_intra16_rd = this_rd;
3888 best_intra16_mode = this_mode;
3889 best_intra16_uv_mode = (mbmi->txfm_size != TX_4X4 ?
3890 uv_intra_mode_8x8 : uv_intra_mode);
3891 }
3892 #endif
3893
3894
3895 if (!disable_skip && mbmi->ref_frame == INTRA_FRAME)
3896 for (i = 0; i < NB_PREDICTION_TYPES; ++i)
3897 best_pred_rd[i] = MIN(best_pred_rd[i], this_rd);
3898
3899 if (this_rd < best_overall_rd) {
3900 best_overall_rd = this_rd;
3901 #if CONFIG_PRED_FILTER
3902 best_filter_state = mbmi->pred_filter_enabled;
3903 #endif
3904 #if CONFIG_COMP_INTERINTRA_PRED
3905 is_best_interintra = (mbmi->second_ref_frame == INTRA_FRAME);
3906 #endif
3907 }
3908
3909 #if CONFIG_PRED_FILTER
3910 // Ignore modes where the prediction filter state doesn't
3911 // match the state signaled at the frame level
3912 if ((cm->pred_filter_mode == 2) ||
3913 (cm->pred_filter_mode ==
3914 mbmi->pred_filter_enabled)) {
3915 #endif
3916 // Did this mode help.. i.e. is it the new best mode
3917 if (this_rd < best_rd || x->skip) {
3918 if (!mode_excluded) {
3919 /*
3920 if (mbmi->second_ref_frame == INTRA_FRAME) {
3921 printf("rd %d best %d bestintra16 %d\n", this_rd, best_rd, best_intr a16_rd);
3922 }
3923 */
3924 // Note index of best mode so far
3925 best_mode_index = mode_index;
3926
3927 if (this_mode <= B_PRED) {
3928 if (mbmi->txfm_size != TX_4X4
3929 && this_mode != B_PRED
3930 && this_mode != I8X8_PRED)
3931 mbmi->uv_mode = uv_intra_mode_8x8;
3932 else
3933 mbmi->uv_mode = uv_intra_mode;
3934 /* required for left and above block mv */
3935 mbmi->mv[0].as_int = 0;
3936 }
3937
3938 other_cost += ref_costs[mbmi->ref_frame];
3939
3940 /* Calculate the final y RD estimate for this mode */
3941 best_yrd = RDCOST(x->rdmult, x->rddiv, (rate2 - rate_uv - other_cost),
3942 (distortion2 - distortion_uv));
3943
3944 *returnrate = rate2;
3945 *returndistortion = distortion2;
3946 best_rd = this_rd;
3947 vpx_memcpy(&best_mbmode, mbmi, sizeof(MB_MODE_INFO));
3948 vpx_memcpy(&best_partition, x->partition_info, sizeof(PARTITION_INFO)) ;
3949
3950 if ((this_mode == B_PRED)
3951 || (this_mode == I8X8_PRED)
3952 || (this_mode == SPLITMV))
3953 for (i = 0; i < 16; i++) {
3954 best_bmodes[i] = xd->block[i].bmi;
3955 }
3956 }
3957
3958 // Testing this mode gave rise to an improvement in best error score.
3959 // Lower threshold a bit for next time
3960 cpi->rd_thresh_mult[mode_index] =
3961 (cpi->rd_thresh_mult[mode_index] >= (MIN_THRESHMULT + 2)) ?
3962 cpi->rd_thresh_mult[mode_index] - 2 : MIN_THRESHMULT;
3963 cpi->rd_threshes[mode_index] =
3964 (cpi->rd_baseline_thresh[mode_index] >> 7) *
3965 cpi->rd_thresh_mult[mode_index];
3966 }
3967 // If the mode did not help improve the best error case then raise the
3968 // threshold for testing that mode next time around.
3969 else {
3970 cpi->rd_thresh_mult[mode_index] += 4;
3971
3972 if (cpi->rd_thresh_mult[mode_index] > MAX_THRESHMULT)
3973 cpi->rd_thresh_mult[mode_index] = MAX_THRESHMULT;
3974
3975 cpi->rd_threshes[mode_index] = (cpi->rd_baseline_thresh[mode_index] >> 7 ) * cpi->rd_thresh_mult[mode_index];
3976 }
3977
3978 /* keep record of best compound/single-only prediction */
3979 if (!disable_skip && mbmi->ref_frame != INTRA_FRAME) {
3980 int64_t single_rd, hybrid_rd;
3981 int single_rate, hybrid_rate;
3982
3983 if (cpi->common.comp_pred_mode == HYBRID_PREDICTION) {
3984 single_rate = rate2 - compmode_cost;
3985 hybrid_rate = rate2;
3986 } else {
3987 single_rate = rate2;
3988 hybrid_rate = rate2 + compmode_cost;
3989 }
3990
3991 single_rd = RDCOST(x->rdmult, x->rddiv, single_rate, distortion2);
3992 hybrid_rd = RDCOST(x->rdmult, x->rddiv, hybrid_rate, distortion2);
3993
3994 if (mbmi->second_ref_frame <= INTRA_FRAME &&
3995 single_rd < best_pred_rd[SINGLE_PREDICTION_ONLY]) {
3996 best_pred_rd[SINGLE_PREDICTION_ONLY] = single_rd;
3997 } else if (mbmi->second_ref_frame > INTRA_FRAME &&
3998 single_rd < best_pred_rd[COMP_PREDICTION_ONLY]) {
3999 best_pred_rd[COMP_PREDICTION_ONLY] = single_rd;
4000 }
4001 if (hybrid_rd < best_pred_rd[HYBRID_PREDICTION])
4002 best_pred_rd[HYBRID_PREDICTION] = hybrid_rd;
4003 }
4004
4005 /* keep record of best txfm size */
4006 if (!mode_excluded && this_rd != INT64_MAX) {
4007 for (i = 0; i < NB_TXFM_MODES; i++) {
4008 int64_t adj_rd;
4009 if (this_mode != B_PRED) {
4010 const int64_t txfm_mode_diff =
4011 txfm_cache[i] - txfm_cache[cm->txfm_mode];
4012 adj_rd = this_rd + txfm_mode_diff;
4013 } else {
4014 adj_rd = this_rd;
4015 }
4016 if (adj_rd < best_txfm_rd[i])
4017 best_txfm_rd[i] = adj_rd;
4018 }
4019 }
4020 #if CONFIG_PRED_FILTER
4021 }
4022 #endif
4023
4024 if (x->skip && !mode_excluded)
4025 break;
4026 }
4027
4028 #if CONFIG_PRED_FILTER
4029 // Update counts for prediction filter usage
4030 if (best_filter_state != 0)
4031 ++cpi->pred_filter_on_count;
4032 else
4033 ++cpi->pred_filter_off_count;
4034 #endif
4035 #if CONFIG_COMP_INTERINTRA_PRED
4036 ++cpi->interintra_select_count[is_best_interintra];
4037 #endif
4038
4039 // Reduce the activation RD thresholds for the best choice mode
4040 if ((cpi->rd_baseline_thresh[best_mode_index] > 0) &&
4041 (cpi->rd_baseline_thresh[best_mode_index] < (INT_MAX >> 2))) {
4042 int best_adjustment = (cpi->rd_thresh_mult[best_mode_index] >> 2);
4043
4044 cpi->rd_thresh_mult[best_mode_index] =
4045 (cpi->rd_thresh_mult[best_mode_index] >=
4046 (MIN_THRESHMULT + best_adjustment)) ?
4047 cpi->rd_thresh_mult[best_mode_index] - best_adjustment : MIN_THRESHMULT;
4048 cpi->rd_threshes[best_mode_index] =
4049 (cpi->rd_baseline_thresh[best_mode_index] >> 7) *
4050 cpi->rd_thresh_mult[best_mode_index];
4051 }
4052
4053 // This code force Altref,0,0 and skip for the frame that overlays a
4054 // an alrtef unless Altref is filtered. However, this is unsafe if
4055 // segment level coding of ref frame or mode is enabled for this
4056 // segment.
4057 if (!vp9_segfeature_active(xd, segment_id, SEG_LVL_REF_FRAME) &&
4058 !vp9_segfeature_active(xd, segment_id, SEG_LVL_MODE) &&
4059 cpi->is_src_frame_alt_ref &&
4060 (cpi->oxcf.arnr_max_frames == 0) &&
4061 (best_mbmode.mode != ZEROMV || best_mbmode.ref_frame != ALTREF_FRAME)) {
4062 mbmi->mode = ZEROMV;
4063 if (cm->txfm_mode != TX_MODE_SELECT)
4064 mbmi->txfm_size = cm->txfm_mode;
4065 else
4066 mbmi->txfm_size = TX_16X16;
4067 mbmi->ref_frame = ALTREF_FRAME;
4068 mbmi->mv[0].as_int = 0;
4069 mbmi->uv_mode = DC_PRED;
4070 mbmi->mb_skip_coeff =
4071 (cpi->common.mb_no_coeff_skip) ? 1 : 0;
4072 mbmi->partitioning = 0;
4073
4074 vpx_memset(best_pred_diff, 0, sizeof(best_pred_diff));
4075 vpx_memset(best_txfm_diff, 0, sizeof(best_txfm_diff));
4076 goto end;
4077 }
4078
4079 // macroblock modes
4080 vpx_memcpy(mbmi, &best_mbmode, sizeof(MB_MODE_INFO));
4081 if (best_mbmode.mode == B_PRED) {
4082 for (i = 0; i < 16; i++) {
4083 xd->mode_info_context->bmi[i].as_mode = best_bmodes[i].as_mode;
4084 xd->block[i].bmi.as_mode = xd->mode_info_context->bmi[i].as_mode;
4085 }
4086 }
4087
4088 if (best_mbmode.mode == I8X8_PRED)
4089 set_i8x8_block_modes(x, mode8x8);
4090
4091 if (best_mbmode.mode == SPLITMV) {
4092 for (i = 0; i < 16; i++)
4093 xd->mode_info_context->bmi[i].as_mv.first.as_int = best_bmodes[i].as_mv.fi rst.as_int;
4094 if (mbmi->second_ref_frame > 0)
4095 for (i = 0; i < 16; i++)
4096 xd->mode_info_context->bmi[i].as_mv.second.as_int = best_bmodes[i].as_mv .second.as_int;
4097
4098 vpx_memcpy(x->partition_info, &best_partition, sizeof(PARTITION_INFO));
4099
4100 mbmi->mv[0].as_int = x->partition_info->bmi[15].mv.as_int;
4101 mbmi->mv[1].as_int = x->partition_info->bmi[15].second_mv.as_int;
4102 }
4103
4104 for (i = 0; i < NB_PREDICTION_TYPES; ++i) {
4105 if (best_pred_rd[i] == INT64_MAX)
4106 best_pred_diff[i] = INT_MIN;
4107 else
4108 best_pred_diff[i] = best_rd - best_pred_rd[i];
4109 }
4110
4111 if (!x->skip) {
4112 for (i = 0; i < NB_TXFM_MODES; i++) {
4113 if (best_txfm_rd[i] == INT64_MAX)
4114 best_txfm_diff[i] = INT_MIN;
4115 else
4116 best_txfm_diff[i] = best_rd - best_txfm_rd[i];
4117 }
4118 } else {
4119 vpx_memset(best_txfm_diff, 0, sizeof(best_txfm_diff));
4120 }
4121
4122 end:
4123 store_coding_context(
4124 x, &x->mb_context[xd->mb_index], best_mode_index, &best_partition,
4125 &frame_best_ref_mv[xd->mode_info_context->mbmi.ref_frame],
4126 &frame_best_ref_mv[xd->mode_info_context->mbmi.second_ref_frame < 0 ?
4127 0 : xd->mode_info_context->mbmi.second_ref_frame],
4128 best_pred_diff, best_txfm_diff);
4129 }
4130
4131 #if CONFIG_SUPERBLOCKS
4132 void vp9_rd_pick_intra_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
4133 int *returnrate,
4134 int *returndist) {
4135 VP9_COMMON *cm = &cpi->common;
4136 MACROBLOCKD *xd = &x->e_mbd;
4137 int rate_y, rate_uv;
4138 int rate_y_tokenonly, rate_uv_tokenonly;
4139 int error_y, error_uv;
4140 int dist_y, dist_uv;
4141 int y_skip, uv_skip;
4142 int64_t txfm_cache[NB_TXFM_MODES];
4143
4144 xd->mode_info_context->mbmi.txfm_size = TX_8X8;
4145
4146 error_y = rd_pick_intra_sby_mode(cpi, x, &rate_y, &rate_y_tokenonly,
4147 &dist_y, &y_skip, txfm_cache);
4148 error_uv = rd_pick_intra_sbuv_mode(cpi, x, &rate_uv, &rate_uv_tokenonly,
4149 &dist_uv, &uv_skip);
4150
4151 if (cpi->common.mb_no_coeff_skip && y_skip && uv_skip) {
4152 *returnrate = rate_y + rate_uv - rate_y_tokenonly - rate_uv_tokenonly +
4153 vp9_cost_bit(vp9_get_pred_prob(cm, xd, PRED_MBSKIP), 1);
4154 *returndist = dist_y + (dist_uv >> 2);
4155 } else {
4156 *returnrate = rate_y + rate_uv;
4157 if (cpi->common.mb_no_coeff_skip)
4158 *returnrate += vp9_cost_bit(vp9_get_pred_prob(cm, xd, PRED_MBSKIP), 0);
4159 *returndist = dist_y + (dist_uv >> 2);
4160 }
4161 }
4162 #endif
4163
4164 void vp9_rd_pick_intra_mode(VP9_COMP *cpi, MACROBLOCK *x,
4165 int *returnrate, int *returndist) {
4166 VP9_COMMON *cm = &cpi->common;
4167 MACROBLOCKD *xd = &x->e_mbd;
4168 MB_MODE_INFO * mbmi = &x->e_mbd.mode_info_context->mbmi;
4169 int64_t error4x4, error16x16;
4170 #if CONFIG_COMP_INTRA_PRED
4171 int64_t error4x4d;
4172 int rate4x4d, dist4x4d;
4173 #endif
4174 int rate4x4, rate16x16 = 0, rateuv, rateuv8x8;
4175 int dist4x4 = 0, dist16x16 = 0, distuv = 0, distuv8x8 = 0;
4176 int rate;
4177 int rate4x4_tokenonly = 0;
4178 int rate16x16_tokenonly = 0;
4179 int rateuv_tokenonly = 0, rateuv8x8_tokenonly = 0;
4180 int64_t error8x8;
4181 int rate8x8_tokenonly=0;
4182 int rate8x8, dist8x8;
4183 int mode16x16;
4184 int mode8x8[2][4];
4185 int dist;
4186 int modeuv, modeuv8x8, uv_intra_skippable, uv_intra_skippable_8x8;
4187 int y_intra16x16_skippable = 0;
4188 int64_t txfm_cache[NB_TXFM_MODES];
4189 TX_SIZE txfm_size_16x16;
4190 int i;
4191
4192 mbmi->ref_frame = INTRA_FRAME;
4193 rd_pick_intra_mbuv_mode(cpi, x, &rateuv, &rateuv_tokenonly, &distuv,
4194 &uv_intra_skippable);
4195 modeuv = mbmi->uv_mode;
4196 if (cpi->common.txfm_mode != ONLY_4X4) {
4197 rd_pick_intra_mbuv_mode_8x8(cpi, x, &rateuv8x8, &rateuv8x8_tokenonly,
4198 &distuv8x8, &uv_intra_skippable_8x8);
4199 modeuv8x8 = mbmi->uv_mode;
4200 } else {
4201 uv_intra_skippable_8x8 = uv_intra_skippable;
4202 rateuv8x8 = rateuv;
4203 distuv8x8 = distuv;
4204 rateuv8x8_tokenonly = rateuv_tokenonly;
4205 modeuv8x8 = modeuv;
4206 }
4207
4208 // current macroblock under rate-distortion optimization test loop
4209 error16x16 = rd_pick_intra16x16mby_mode(cpi, x, &rate16x16,
4210 &rate16x16_tokenonly, &dist16x16,
4211 &y_intra16x16_skippable, txfm_cache);
4212 mode16x16 = mbmi->mode;
4213 txfm_size_16x16 = mbmi->txfm_size;
4214
4215 // FIXME(rbultje) support transform-size selection
4216 mbmi->txfm_size = (cm->txfm_mode == ONLY_4X4) ? TX_4X4 : TX_8X8;
4217 error8x8 = rd_pick_intra8x8mby_modes(cpi, x, &rate8x8, &rate8x8_tokenonly,
4218 &dist8x8, error16x16);
4219 mode8x8[0][0]= xd->mode_info_context->bmi[0].as_mode.first;
4220 mode8x8[0][1]= xd->mode_info_context->bmi[2].as_mode.first;
4221 mode8x8[0][2]= xd->mode_info_context->bmi[8].as_mode.first;
4222 mode8x8[0][3]= xd->mode_info_context->bmi[10].as_mode.first;
4223 #if CONFIG_COMP_INTRA_PRED
4224 mode8x8[1][0] = xd->mode_info_context->bmi[0].as_mode.second;
4225 mode8x8[1][1] = xd->mode_info_context->bmi[2].as_mode.second;
4226 mode8x8[1][2] = xd->mode_info_context->bmi[8].as_mode.second;
4227 mode8x8[1][3] = xd->mode_info_context->bmi[10].as_mode.second;
4228 #endif
4229
4230 error4x4 = rd_pick_intra4x4mby_modes(cpi, x,
4231 &rate4x4, &rate4x4_tokenonly,
4232 &dist4x4, error16x16,
4233 #if CONFIG_COMP_INTRA_PRED
4234 0,
4235 #endif
4236 0);
4237 #if CONFIG_COMP_INTRA_PRED
4238 error4x4d = rd_pick_intra4x4mby_modes(cpi, x,
4239 &rate4x4d, &rate4x4_tokenonly,
4240 &dist4x4d, error16x16, 1, 0);
4241 #endif
4242
4243 mbmi->mb_skip_coeff = 0;
4244 if (cpi->common.mb_no_coeff_skip &&
4245 y_intra16x16_skippable && uv_intra_skippable_8x8) {
4246 mbmi->mb_skip_coeff = 1;
4247 mbmi->mode = mode16x16;
4248 mbmi->uv_mode = modeuv;
4249 rate = rateuv8x8 + rate16x16 - rateuv8x8_tokenonly - rate16x16_tokenonly +
4250 vp9_cost_bit(vp9_get_pred_prob(cm, xd, PRED_MBSKIP), 1);
4251 dist = dist16x16 + (distuv8x8 >> 2);
4252 mbmi->txfm_size = txfm_size_16x16;
4253 memset(x->mb_context[xd->mb_index].txfm_rd_diff, 0,
4254 sizeof(x->mb_context[xd->mb_index].txfm_rd_diff));
4255 } else if (error8x8 > error16x16) {
4256 if (error4x4 < error16x16) {
4257 rate = rateuv;
4258 #if CONFIG_COMP_INTRA_PRED
4259 rate += (error4x4d < error4x4) ? rate4x4d : rate4x4;
4260 if (error4x4d >= error4x4) // FIXME save original modes etc.
4261 error4x4 = rd_pick_intra4x4mby_modes(cpi, x, &rate4x4,
4262 &rate4x4_tokenonly,
4263 &dist4x4, error16x16, 0,
4264 cpi->update_context);
4265 #else
4266 rate += rate4x4;
4267 #endif
4268 mbmi->mode = B_PRED;
4269 mbmi->txfm_size = TX_4X4;
4270 dist = dist4x4 + (distuv >> 2);
4271 memset(x->mb_context[xd->mb_index].txfm_rd_diff, 0,
4272 sizeof(x->mb_context[xd->mb_index].txfm_rd_diff));
4273 } else {
4274 mbmi->txfm_size = txfm_size_16x16;
4275 mbmi->mode = mode16x16;
4276 rate = rate16x16 + rateuv8x8;
4277 dist = dist16x16 + (distuv8x8 >> 2);
4278 for (i = 0; i < NB_TXFM_MODES; i++) {
4279 x->mb_context[xd->mb_index].txfm_rd_diff[i] = error16x16 - txfm_cache[i] ;
4280 }
4281 }
4282 if (cpi->common.mb_no_coeff_skip)
4283 rate += vp9_cost_bit(vp9_get_pred_prob(cm, xd, PRED_MBSKIP), 0);
4284 } else {
4285 if (error4x4 < error8x8) {
4286 rate = rateuv;
4287 #if CONFIG_COMP_INTRA_PRED
4288 rate += (error4x4d < error4x4) ? rate4x4d : rate4x4;
4289 if (error4x4d >= error4x4) // FIXME save original modes etc.
4290 error4x4 = rd_pick_intra4x4mby_modes(cpi, x, &rate4x4,
4291 &rate4x4_tokenonly,
4292 &dist4x4, error16x16, 0,
4293 cpi->update_context);
4294 #else
4295 rate += rate4x4;
4296 #endif
4297 mbmi->mode = B_PRED;
4298 mbmi->txfm_size = TX_4X4;
4299 dist = dist4x4 + (distuv >> 2);
4300 memset(x->mb_context[xd->mb_index].txfm_rd_diff, 0,
4301 sizeof(x->mb_context[xd->mb_index].txfm_rd_diff));
4302 } else {
4303 // FIXME(rbultje) support transform-size selection
4304 mbmi->mode = I8X8_PRED;
4305 mbmi->txfm_size = (cm->txfm_mode == ONLY_4X4) ? TX_4X4 : TX_8X8;
4306 set_i8x8_block_modes(x, mode8x8);
4307 rate = rate8x8 + rateuv;
4308 dist = dist8x8 + (distuv >> 2);
4309 memset(x->mb_context[xd->mb_index].txfm_rd_diff, 0,
4310 sizeof(x->mb_context[xd->mb_index].txfm_rd_diff));
4311 }
4312 if (cpi->common.mb_no_coeff_skip)
4313 rate += vp9_cost_bit(vp9_get_pred_prob(cm, xd, PRED_MBSKIP), 0);
4314 }
4315
4316 *returnrate = rate;
4317 *returndist = dist;
4318 }
4319
4320 #if CONFIG_SUPERBLOCKS
4321 int64_t vp9_rd_pick_inter_mode_sb(VP9_COMP *cpi, MACROBLOCK *x,
4322 int recon_yoffset, int recon_uvoffset,
4323 int *returnrate, int *returndistortion) {
4324 VP9_COMMON *cm = &cpi->common;
4325 MACROBLOCKD *xd = &x->e_mbd;
4326 MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
4327 MB_PREDICTION_MODE this_mode;
4328 MV_REFERENCE_FRAME ref_frame;
4329 unsigned char segment_id = xd->mode_info_context->mbmi.segment_id;
4330 int comp_pred, i;
4331 int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];
4332 int_mv frame_best_ref_mv[MAX_REF_FRAMES];
4333 int_mv mv_search_ref[MAX_REF_FRAMES];
4334 int frame_mdcounts[4][4];
4335 unsigned char *y_buffer[4];
4336 unsigned char *u_buffer[4];
4337 unsigned char *v_buffer[4];
4338 static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG,
4339 VP9_ALT_FLAG };
4340 int idx_list[4] = { 0, cpi->common.lst_fb_idx, cpi->common.gld_fb_idx,
4341 cpi->common.alt_fb_idx };
4342 int mdcounts[4];
4343 int near_sadidx[8] = { 0, 1, 2, 3, 4, 5, 6, 7 };
4344 int saddone = 0;
4345 int64_t best_rd = INT64_MAX;
4346 int64_t best_yrd = INT64_MAX;
4347 int64_t best_txfm_rd[NB_TXFM_MODES];
4348 int64_t best_txfm_diff[NB_TXFM_MODES];
4349 int64_t best_pred_diff[NB_PREDICTION_TYPES];
4350 int64_t best_pred_rd[NB_PREDICTION_TYPES];
4351 MB_MODE_INFO best_mbmode;
4352 int mode_index, best_mode_index = 0;
4353 unsigned int ref_costs[MAX_REF_FRAMES];
4354 #if CONFIG_COMP_INTERINTRA_PRED
4355 int is_best_interintra = 0;
4356 int64_t best_intra16_rd = INT64_MAX;
4357 int best_intra16_mode = DC_PRED, best_intra16_uv_mode = DC_PRED;
4358 #endif
4359 int64_t best_overall_rd = INT64_MAX;
4360 int rate_uv_4x4 = 0, rate_uv_8x8 = 0, rate_uv_tokenonly_4x4 = 0,
4361 rate_uv_tokenonly_8x8 = 0;
4362 int dist_uv_4x4 = 0, dist_uv_8x8 = 0, uv_skip_4x4 = 0, uv_skip_8x8 = 0;
4363 MB_PREDICTION_MODE mode_uv_4x4 = NEARESTMV, mode_uv_8x8 = NEARESTMV;
4364 int switchable_filter_index = 0;
4365
4366 x->skip = 0;
4367 xd->mode_info_context->mbmi.segment_id = segment_id;
4368 estimate_ref_frame_costs(cpi, segment_id, ref_costs);
4369 vpx_memset(&best_mbmode, 0, sizeof(best_mbmode));
4370
4371 for (i = 0; i < NB_PREDICTION_TYPES; ++i)
4372 best_pred_rd[i] = INT64_MAX;
4373 for (i = 0; i < NB_TXFM_MODES; i++)
4374 best_txfm_rd[i] = INT64_MAX;
4375
4376 for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) {
4377 if (cpi->ref_frame_flags & flag_list[ref_frame]) {
4378 setup_buffer_inter(cpi, x, idx_list[ref_frame], ref_frame, BLOCK_32X32,
4379 recon_yoffset, recon_uvoffset, frame_mv[NEARESTMV],
4380 frame_mv[NEARMV], frame_best_ref_mv, mv_search_ref,
4381 frame_mdcounts, y_buffer, u_buffer, v_buffer);
4382 }
4383 frame_mv[NEWMV][ref_frame].as_int = INVALID_MV;
4384 frame_mv[ZEROMV][ref_frame].as_int = 0;
4385 }
4386
4387 mbmi->mode = DC_PRED;
4388 if (cm->txfm_mode == ONLY_4X4 || cm->txfm_mode == TX_MODE_SELECT) {
4389 mbmi->txfm_size = TX_4X4;
4390 rd_pick_intra_sbuv_mode(cpi, x, &rate_uv_4x4, &rate_uv_tokenonly_4x4,
4391 &dist_uv_4x4, &uv_skip_4x4);
4392 mode_uv_4x4 = mbmi->uv_mode;
4393 }
4394 if (cm->txfm_mode != ONLY_4X4) {
4395 mbmi->txfm_size = TX_8X8;
4396 rd_pick_intra_sbuv_mode(cpi, x, &rate_uv_8x8, &rate_uv_tokenonly_8x8,
4397 &dist_uv_8x8, &uv_skip_8x8);
4398 mode_uv_8x8 = mbmi->uv_mode;
4399 }
4400
4401 for (mode_index = 0; mode_index < MAX_MODES;
4402 mode_index += (!switchable_filter_index)) {
4403 int mode_excluded = 0;
4404 int64_t this_rd = INT64_MAX;
4405 int disable_skip = 0;
4406 int other_cost = 0;
4407 int compmode_cost = 0;
4408 int rate2 = 0, rate_y = 0, rate_uv = 0;
4409 int distortion2 = 0, distortion_y = 0, distortion_uv = 0;
4410 int skippable;
4411 int64_t txfm_cache[NB_TXFM_MODES];
4412 #if CONFIG_COMP_INTERINTRA_PRED
4413 int compmode_interintra_cost = 0;
4414 #endif
4415
4416 // Test best rd so far against threshold for trying this mode.
4417 if (best_rd <= cpi->rd_threshes[mode_index] ||
4418 cpi->rd_threshes[mode_index] == INT_MAX) {
4419 continue;
4420 }
4421
4422 this_mode = vp9_mode_order[mode_index].mode;
4423 ref_frame = vp9_mode_order[mode_index].ref_frame;
4424 if (!(ref_frame == INTRA_FRAME ||
4425 (cpi->ref_frame_flags & flag_list[ref_frame]))) {
4426 continue;
4427 }
4428 mbmi->ref_frame = ref_frame;
4429 mbmi->second_ref_frame = vp9_mode_order[mode_index].second_ref_frame;
4430 comp_pred = mbmi->second_ref_frame > INTRA_FRAME;
4431 mbmi->mode = this_mode;
4432 mbmi->uv_mode = DC_PRED;
4433 #if CONFIG_COMP_INTRA_PRED
4434 mbmi->second_mode = (MB_PREDICTION_MODE)(DC_PRED - 1);
4435 mbmi->second_uv_mode = (MB_PREDICTION_MODE)(DC_PRED - 1);
4436 #endif
4437 #if CONFIG_COMP_INTERINTRA_PRED
4438 mbmi->interintra_mode = (MB_PREDICTION_MODE)(DC_PRED - 1);
4439 mbmi->interintra_uv_mode = (MB_PREDICTION_MODE)(DC_PRED - 1);
4440 #endif
4441 if (cpi->common.mcomp_filter_type == SWITCHABLE &&
4442 this_mode >= NEARESTMV && this_mode <= SPLITMV) {
4443 mbmi->interp_filter =
4444 vp9_switchable_interp[switchable_filter_index++];
4445 if (switchable_filter_index == VP9_SWITCHABLE_FILTERS)
4446 switchable_filter_index = 0;
4447 } else {
4448 mbmi->interp_filter = cpi->common.mcomp_filter_type;
4449 }
4450 vp9_setup_interp_filters(xd, mbmi->interp_filter, &cpi->common);
4451
4452 // if (!(cpi->ref_frame_flags & flag_list[ref_frame]))
4453 // continue;
4454
4455 if (this_mode == I8X8_PRED || this_mode == B_PRED || this_mode == SPLITMV)
4456 continue;
4457 // if (vp9_mode_order[mode_index].second_ref_frame == INTRA_FRAME)
4458 // continue;
4459
4460 if (comp_pred) {
4461 int second_ref;
4462
4463 if (ref_frame == ALTREF_FRAME) {
4464 second_ref = LAST_FRAME;
4465 } else {
4466 second_ref = ref_frame + 1;
4467 }
4468 if (!(cpi->ref_frame_flags & flag_list[second_ref]))
4469 continue;
4470 mbmi->second_ref_frame = second_ref;
4471
4472 xd->second_pre.y_buffer = y_buffer[second_ref];
4473 xd->second_pre.u_buffer = u_buffer[second_ref];
4474 xd->second_pre.v_buffer = v_buffer[second_ref];
4475 mode_excluded = cm->comp_pred_mode == SINGLE_PREDICTION_ONLY;
4476 } else {
4477 // mbmi->second_ref_frame = vp9_mode_order[mode_index].second_ref_frame;
4478 if (ref_frame != INTRA_FRAME) {
4479 if (mbmi->second_ref_frame != INTRA_FRAME)
4480 mode_excluded = cm->comp_pred_mode == COMP_PREDICTION_ONLY;
4481 #if CONFIG_COMP_INTERINTRA_PRED
4482 else
4483 mode_excluded = !cm->use_interintra;
4484 #endif
4485 }
4486 }
4487
4488 xd->pre.y_buffer = y_buffer[ref_frame];
4489 xd->pre.u_buffer = u_buffer[ref_frame];
4490 xd->pre.v_buffer = v_buffer[ref_frame];
4491 vpx_memcpy(mdcounts, frame_mdcounts[ref_frame], sizeof(mdcounts));
4492
4493 // If the segment reference frame feature is enabled....
4494 // then do nothing if the current ref frame is not allowed..
4495 if (vp9_segfeature_active(xd, segment_id, SEG_LVL_REF_FRAME) &&
4496 !vp9_check_segref(xd, segment_id, ref_frame)) {
4497 continue;
4498 // If the segment mode feature is enabled....
4499 // then do nothing if the current mode is not allowed..
4500 } else if (vp9_segfeature_active(xd, segment_id, SEG_LVL_MODE) &&
4501 (this_mode != vp9_get_segdata(xd, segment_id, SEG_LVL_MODE))) {
4502 continue;
4503 // Disable this drop out case if either the mode or ref frame
4504 // segment level feature is enabled for this segment. This is to
4505 // prevent the possibility that we end up unable to pick any mode.
4506 } else if (!vp9_segfeature_active(xd, segment_id, SEG_LVL_REF_FRAME) &&
4507 !vp9_segfeature_active(xd, segment_id, SEG_LVL_MODE)) {
4508 // Only consider ZEROMV/ALTREF_FRAME for alt ref frame,
4509 // unless ARNR filtering is enabled in which case we want
4510 // an unfiltered alternative
4511 if (cpi->is_src_frame_alt_ref && (cpi->oxcf.arnr_max_frames == 0)) {
4512 if (this_mode != ZEROMV || ref_frame != ALTREF_FRAME) {
4513 continue;
4514 }
4515 }
4516 }
4517
4518 if (ref_frame == INTRA_FRAME) {
4519 vp9_build_intra_predictors_sby_s(xd);
4520 super_block_yrd(cpi, x, &rate_y, &distortion_y,
4521 &skippable, txfm_cache);
4522 if (mbmi->txfm_size == TX_4X4) {
4523 rate_uv = rate_uv_4x4;
4524 distortion_uv = dist_uv_4x4;
4525 skippable = skippable && uv_skip_4x4;
4526 mbmi->uv_mode = mode_uv_4x4;
4527 } else {
4528 rate_uv = rate_uv_8x8;
4529 distortion_uv = dist_uv_8x8;
4530 skippable = skippable && uv_skip_8x8;
4531 mbmi->uv_mode = mode_uv_8x8;
4532 }
4533
4534 rate2 = rate_y + x->mbmode_cost[cm->frame_type][mbmi->mode] + rate_uv;
4535 distortion2 = distortion_y + distortion_uv;
4536 } else {
4537 #if CONFIG_COMP_INTERINTRA_PRED
4538 if (mbmi->second_ref_frame == INTRA_FRAME) {
4539 if (best_intra16_mode == DC_PRED - 1) continue;
4540 mbmi->interintra_mode = best_intra16_mode;
4541 #if SEPARATE_INTERINTRA_UV
4542 mbmi->interintra_uv_mode = best_intra16_uv_mode;
4543 #else
4544 mbmi->interintra_uv_mode = best_intra16_mode;
4545 #endif
4546 }
4547 #endif
4548 this_rd = handle_inter_mode(cpi, x, BLOCK_32X32,
4549 &saddone, near_sadidx, mdcounts, txfm_cache,
4550 &rate2, &distortion2, &skippable,
4551 &compmode_cost,
4552 #if CONFIG_COMP_INTERINTRA_PRED
4553 &compmode_interintra_cost,
4554 #endif
4555 &rate_y, &distortion_y,
4556 &rate_uv, &distortion_uv,
4557 &mode_excluded, &disable_skip, recon_yoffset,
4558 mode_index, frame_mv, frame_best_ref_mv,
4559 mv_search_ref);
4560 if (this_rd == INT64_MAX)
4561 continue;
4562 }
4563
4564 #if CONFIG_COMP_INTERINTRA_PRED
4565 if (cpi->common.use_interintra) {
4566 rate2 += compmode_interintra_cost;
4567 }
4568 #endif
4569 if (cpi->common.comp_pred_mode == HYBRID_PREDICTION) {
4570 rate2 += compmode_cost;
4571 }
4572
4573 // Estimate the reference frame signaling cost and add it
4574 // to the rolling cost variable.
4575 rate2 += ref_costs[xd->mode_info_context->mbmi.ref_frame];
4576
4577 if (!disable_skip) {
4578 // Test for the condition where skip block will be activated
4579 // because there are no non zero coefficients and make any
4580 // necessary adjustment for rate. Ignore if skip is coded at
4581 // segment level as the cost wont have been added in.
4582 if (cpi->common.mb_no_coeff_skip) {
4583 int mb_skip_allowed;
4584
4585 // Is Mb level skip allowed for this mb.
4586 mb_skip_allowed =
4587 !vp9_segfeature_active(xd, segment_id, SEG_LVL_EOB) ||
4588 vp9_get_segdata(xd, segment_id, SEG_LVL_EOB);
4589
4590 if (skippable) {
4591 // Back out the coefficient coding costs
4592 rate2 -= (rate_y + rate_uv);
4593 // for best_yrd calculation
4594 rate_uv = 0;
4595
4596 if (mb_skip_allowed) {
4597 int prob_skip_cost;
4598
4599 // Cost the skip mb case
4600 vp9_prob skip_prob =
4601 vp9_get_pred_prob(cm, xd, PRED_MBSKIP);
4602
4603 if (skip_prob) {
4604 prob_skip_cost = vp9_cost_bit(skip_prob, 1);
4605 rate2 += prob_skip_cost;
4606 other_cost += prob_skip_cost;
4607 }
4608 }
4609 }
4610 // Add in the cost of the no skip flag.
4611 else if (mb_skip_allowed) {
4612 int prob_skip_cost = vp9_cost_bit(vp9_get_pred_prob(cm, xd,
4613 PRED_MBSKIP), 0);
4614 rate2 += prob_skip_cost;
4615 other_cost += prob_skip_cost;
4616 }
4617 }
4618
4619 // Calculate the final RD estimate for this mode.
4620 this_rd = RDCOST(x->rdmult, x->rddiv, rate2, distortion2);
4621 }
4622
4623 #if 0
4624 // Keep record of best intra distortion
4625 if ((xd->mode_info_context->mbmi.ref_frame == INTRA_FRAME) &&
4626 (this_rd < best_intra_rd)) {
4627 best_intra_rd = this_rd;
4628 *returnintra = distortion2;
4629 }
4630 #endif
4631 #if CONFIG_COMP_INTERINTRA_PRED
4632 if ((mbmi->ref_frame == INTRA_FRAME) &&
4633 (this_mode <= TM_PRED) &&
4634 (this_rd < best_intra16_rd)) {
4635 best_intra16_rd = this_rd;
4636 best_intra16_mode = this_mode;
4637 best_intra16_uv_mode = (mbmi->txfm_size != TX_4X4 ?
4638 mode_uv_8x8 : mode_uv_4x4);
4639 }
4640 #endif
4641
4642 if (!disable_skip && mbmi->ref_frame == INTRA_FRAME)
4643 for (i = 0; i < NB_PREDICTION_TYPES; ++i)
4644 best_pred_rd[i] = MIN(best_pred_rd[i], this_rd);
4645
4646 if (this_rd < best_overall_rd) {
4647 best_overall_rd = this_rd;
4648 #if CONFIG_COMP_INTERINTRA_PRED
4649 is_best_interintra = (mbmi->second_ref_frame == INTRA_FRAME);
4650 #endif
4651 }
4652
4653 // Did this mode help.. i.e. is it the new best mode
4654 if (this_rd < best_rd || x->skip) {
4655 if (!mode_excluded) {
4656 // Note index of best mode so far
4657 best_mode_index = mode_index;
4658
4659 if (this_mode <= B_PRED) {
4660 /* required for left and above block mv */
4661 mbmi->mv[0].as_int = 0;
4662 }
4663
4664 other_cost += ref_costs[xd->mode_info_context->mbmi.ref_frame];
4665
4666 /* Calculate the final y RD estimate for this mode */
4667 best_yrd = RDCOST(x->rdmult, x->rddiv, (rate2 - rate_uv - other_cost),
4668 (distortion2 - distortion_uv));
4669
4670 *returnrate = rate2;
4671 *returndistortion = distortion2;
4672 best_rd = this_rd;
4673 vpx_memcpy(&best_mbmode, mbmi, sizeof(MB_MODE_INFO));
4674 }
4675 #if 0
4676 // Testing this mode gave rise to an improvement in best error score. Lowe r threshold a bit for next time
4677 cpi->rd_thresh_mult[mode_index] = (cpi->rd_thresh_mult[mode_index] >= (MIN _THRESHMULT + 2)) ? cpi->rd_thresh_mult[mode_index] - 2 : MIN_THRESHMULT;
4678 cpi->rd_threshes[mode_index] = (cpi->rd_baseline_thresh[mode_index] >> 7) * cpi->rd_thresh_mult[mode_index];
4679 #endif
4680 }
4681 // If the mode did not help improve the best error case then raise the thres hold for testing that mode next time around.
4682 else {
4683 #if 0
4684 cpi->rd_thresh_mult[mode_index] += 4;
4685
4686 if (cpi->rd_thresh_mult[mode_index] > MAX_THRESHMULT)
4687 cpi->rd_thresh_mult[mode_index] = MAX_THRESHMULT;
4688
4689 cpi->rd_threshes[mode_index] = (cpi->rd_baseline_thresh[mode_index] >> 7) * cpi->rd_thresh_mult[mode_index];
4690 #endif
4691 }
4692
4693 /* keep record of best compound/single-only prediction */
4694 if (!disable_skip && mbmi->ref_frame != INTRA_FRAME) {
4695 int single_rd, hybrid_rd, single_rate, hybrid_rate;
4696
4697 if (cpi->common.comp_pred_mode == HYBRID_PREDICTION) {
4698 single_rate = rate2 - compmode_cost;
4699 hybrid_rate = rate2;
4700 } else {
4701 single_rate = rate2;
4702 hybrid_rate = rate2 + compmode_cost;
4703 }
4704
4705 single_rd = RDCOST(x->rdmult, x->rddiv, single_rate, distortion2);
4706 hybrid_rd = RDCOST(x->rdmult, x->rddiv, hybrid_rate, distortion2);
4707
4708 if (mbmi->second_ref_frame <= INTRA_FRAME &&
4709 single_rd < best_pred_rd[SINGLE_PREDICTION_ONLY]) {
4710 best_pred_rd[SINGLE_PREDICTION_ONLY] = single_rd;
4711 } else if (mbmi->second_ref_frame > INTRA_FRAME &&
4712 single_rd < best_pred_rd[COMP_PREDICTION_ONLY]) {
4713 best_pred_rd[COMP_PREDICTION_ONLY] = single_rd;
4714 }
4715 if (hybrid_rd < best_pred_rd[HYBRID_PREDICTION])
4716 best_pred_rd[HYBRID_PREDICTION] = hybrid_rd;
4717 }
4718
4719 /* keep record of best txfm size */
4720 if (!mode_excluded && this_rd != INT64_MAX) {
4721 for (i = 0; i < NB_TXFM_MODES; i++) {
4722 int64_t adj_rd;
4723 if (this_mode != B_PRED) {
4724 adj_rd = this_rd + txfm_cache[i] - txfm_cache[cm->txfm_mode];
4725 } else {
4726 adj_rd = this_rd;
4727 }
4728 if (adj_rd < best_txfm_rd[i])
4729 best_txfm_rd[i] = adj_rd;
4730 }
4731 }
4732
4733 if (x->skip && !mode_excluded)
4734 break;
4735 }
4736
4737 #if CONFIG_COMP_INTERINTRA_PRED
4738 ++cpi->interintra_select_count[is_best_interintra];
4739 // if (is_best_interintra) printf("best_interintra\n");
4740 #endif
4741
4742 // TODO(rbultje) integrate with RD thresholding
4743 #if 0
4744 // Reduce the activation RD thresholds for the best choice mode
4745 if ((cpi->rd_baseline_thresh[best_mode_index] > 0) &&
4746 (cpi->rd_baseline_thresh[best_mode_index] < (INT_MAX >> 2))) {
4747 int best_adjustment = (cpi->rd_thresh_mult[best_mode_index] >> 2);
4748
4749 cpi->rd_thresh_mult[best_mode_index] =
4750 (cpi->rd_thresh_mult[best_mode_index] >= (MIN_THRESHMULT + best_adjustment )) ?
4751 cpi->rd_thresh_mult[best_mode_index] - best_adjustment : MIN_THRESHMULT;
4752 cpi->rd_threshes[best_mode_index] =
4753 (cpi->rd_baseline_thresh[best_mode_index] >> 7) * cpi->rd_thresh_mult[best _mode_index];
4754 }
4755 #endif
4756
4757 // This code forces Altref,0,0 and skip for the frame that overlays a
4758 // an alrtef unless Altref is filtered. However, this is unsafe if
4759 // segment level coding of ref frame or mode is enabled for this
4760 // segment.
4761 if (!vp9_segfeature_active(xd, segment_id, SEG_LVL_REF_FRAME) &&
4762 !vp9_segfeature_active(xd, segment_id, SEG_LVL_MODE) &&
4763 cpi->is_src_frame_alt_ref &&
4764 (cpi->oxcf.arnr_max_frames == 0) &&
4765 (best_mbmode.mode != ZEROMV || best_mbmode.ref_frame != ALTREF_FRAME)) {
4766 mbmi->mode = ZEROMV;
4767 mbmi->ref_frame = ALTREF_FRAME;
4768 mbmi->second_ref_frame = INTRA_FRAME;
4769 mbmi->mv[0].as_int = 0;
4770 mbmi->uv_mode = DC_PRED;
4771 mbmi->mb_skip_coeff = (cpi->common.mb_no_coeff_skip) ? 1 : 0;
4772 mbmi->partitioning = 0;
4773 mbmi->txfm_size = cm->txfm_mode == TX_MODE_SELECT ?
4774 TX_16X16 : cm->txfm_mode;
4775
4776 vpx_memset(best_txfm_diff, 0, sizeof(best_txfm_diff));
4777 vpx_memset(best_pred_diff, 0, sizeof(best_pred_diff));
4778 goto end;
4779 }
4780
4781 // macroblock modes
4782 vpx_memcpy(mbmi, &best_mbmode, sizeof(MB_MODE_INFO));
4783
4784 for (i = 0; i < NB_PREDICTION_TYPES; ++i) {
4785 if (best_pred_rd[i] == INT64_MAX)
4786 best_pred_diff[i] = INT_MIN;
4787 else
4788 best_pred_diff[i] = best_rd - best_pred_rd[i];
4789 }
4790
4791 if (!x->skip) {
4792 for (i = 0; i < NB_TXFM_MODES; i++) {
4793 if (best_txfm_rd[i] == INT64_MAX)
4794 best_txfm_diff[i] = INT_MIN;
4795 else
4796 best_txfm_diff[i] = best_rd - best_txfm_rd[i];
4797 }
4798 } else {
4799 vpx_memset(best_txfm_diff, 0, sizeof(best_txfm_diff));
4800 }
4801
4802 end:
4803 store_coding_context(x, &x->sb_context[0], best_mode_index, NULL,
4804 &frame_best_ref_mv[mbmi->ref_frame],
4805 &frame_best_ref_mv[mbmi->second_ref_frame < 0 ?
4806 0 : mbmi->second_ref_frame],
4807 best_pred_diff, best_txfm_diff);
4808
4809 return best_rd;
4810 }
4811 #endif
4812
4813 void vp9_pick_mode_inter_macroblock(VP9_COMP *cpi, MACROBLOCK *x,
4814 int recon_yoffset,
4815 int recon_uvoffset,
4816 int *totalrate, int *totaldist) {
4817 MACROBLOCKD *const xd = &x->e_mbd;
4818 MB_MODE_INFO * mbmi = &x->e_mbd.mode_info_context->mbmi;
4819 int rate, distortion;
4820 int64_t intra_error = 0;
4821 unsigned char *segment_id = &mbmi->segment_id;
4822
4823 if (xd->segmentation_enabled)
4824 x->encode_breakout = cpi->segment_encode_breakout[*segment_id];
4825 else
4826 x->encode_breakout = cpi->oxcf.encode_breakout;
4827
4828 // if (cpi->sf.RD)
4829 // For now this codebase is limited to a single rd encode path
4830 {
4831 int zbin_mode_boost_enabled = cpi->zbin_mode_boost_enabled;
4832
4833 rd_pick_inter_mode(cpi, x, recon_yoffset, recon_uvoffset, &rate,
4834 &distortion, &intra_error);
4835
4836 /* restore cpi->zbin_mode_boost_enabled */
4837 cpi->zbin_mode_boost_enabled = zbin_mode_boost_enabled;
4838 }
4839 // else
4840 // The non rd encode path has been deleted from this code base
4841 // to simplify development
4842 // vp9_pick_inter_mode
4843
4844 // Store metrics so they can be added in to totals if this mode is picked
4845 x->mb_context[xd->mb_index].distortion = distortion;
4846 x->mb_context[xd->mb_index].intra_error = intra_error;
4847
4848 *totalrate = rate;
4849 *totaldist = distortion;
4850 }
OLDNEW
« libvpx.gyp ('K') | « source/libvpx/vp9/encoder/vp9_rdopt.h ('k') | source/libvpx/vp9/encoder/vp9_sad_c.c » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698