sjero.net Git - linphone/blob - linphone/mediastreamer2/plugins/msx264/src/msx264.c

   1 /*
   2 mediastreamer2 x264 plugin
   3 Copyright (C) 2006  Simon MORLAT (simon.morlat@linphone.org)
   4
   5 This program is free software; you can redistribute it and/or
   6 modify it under the terms of the GNU General Public License
   7 as published by the Free Software Foundation; either version 2
   8 of the License, or (at your option) any later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU General Public License for more details.
  14
  15 You should have received a copy of the GNU General Public License
  16 along with this program; if not, write to the Free Software
  17 Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  18 */
  19
  20 #include "mediastreamer2/msfilter.h"
  21 #include "mediastreamer2/msticker.h"
  22 #include "mediastreamer2/msvideo.h"
  23 #include "mediastreamer2/rfc3984.h"
  24
  25 #include "ortp/b64.h"
  26
  27 #include <x264.h>
  28
  29 #ifdef HAVE_LIBAVCODEC_AVCODEC_H
  30 #include <libavcodec/avcodec.h>
  31 #include <libswscale/swscale.h>
  32 #else
  33 #include <ffmpeg/avcodec.h>
  34 #include <ffmpeg/swscale.h>
  35 #endif
  36
  37 #define REMOVE_PREVENTING_BYTES 1
  38
  39 typedef struct _EncData{
  40         x264_t *enc;
  41         MSVideoSize vsize;
  42         int bitrate;
  43         float fps;
  44         int mode;
  45         uint64_t framenum;
  46         Rfc3984Context packer;
  47         int keyframe_int;
  48         bool_t generate_keyframe;
  49 }EncData;
  50
  51
  52 static void enc_init(MSFilter *f){
  53         EncData *d=ms_new(EncData,1);
  54         d->enc=NULL;
  55         d->bitrate=384000;
  56         d->vsize=MS_VIDEO_SIZE_CIF;
  57         d->fps=30;
  58         d->keyframe_int=10; /*10 seconds */
  59         d->mode=0;
  60         d->framenum=0;
  61         d->generate_keyframe=FALSE;
  62         f->data=d;
  63 }
  64
  65 static void enc_uninit(MSFilter *f){
  66         EncData *d=(EncData*)f->data;
  67         ms_free(d);
  68 }
  69
  70 static void enc_preprocess(MSFilter *f){
  71         EncData *d=(EncData*)f->data;
  72         x264_param_t params;
  73
  74         rfc3984_init(&d->packer);
  75         rfc3984_set_mode(&d->packer,d->mode);
  76         rfc3984_enable_stap_a(&d->packer,FALSE);
  77
  78         x264_param_default(&params);
  79         params.i_threads=1;
  80         params.i_sync_lookahead=0;
  81         params.i_width=d->vsize.width;
  82         params.i_height=d->vsize.height;
  83         params.i_fps_num=(int)d->fps;
  84         params.i_fps_den=1;
  85         params.i_slice_max_size=ms_get_payload_max_size()-100; /*-100 security margin*/
  86         /*params.i_level_idc=30;*/
  87
  88         params.rc.i_rc_method = X264_RC_ABR;
  89         params.rc.i_bitrate=(int)( ( ((float)d->bitrate)*0.8)/1000.0);
  90         params.rc.f_rate_tolerance=0.1;
  91         params.rc.i_vbv_max_bitrate=(int) (((float)d->bitrate)*0.9/1000.0);
  92         params.rc.i_vbv_buffer_size=params.rc.i_vbv_max_bitrate;
  93         params.rc.f_vbv_buffer_init=0.5;
  94         params.rc.i_lookahead=0;
  95         /*enable this by config ?*/
  96         /*
  97         params.i_keyint_max = (int)d->fps*d->keyframe_int;
  98         params.i_keyint_min = (int)d->fps;
  99         */
 100         params.b_repeat_headers=1;
 101         params.b_cabac=0;//disable cabac to be baseline
 102         params.i_bframe=0;/*no B frames*/
 103         d->enc=x264_encoder_open(&params);
 104         if (d->enc==NULL) ms_error("Fail to create x264 encoder.");
 105         d->framenum=0;
 106 }
 107
 108 static void x264_nals_to_msgb(x264_nal_t *xnals, int num_nals, MSQueue * nalus){
 109         int i;
 110         mblk_t *m;
 111         /*int bytes;*/
 112         for (i=0;i<num_nals;++i){
 113                 m=allocb(xnals[i].i_payload+10,0);
 114
 115                 memcpy(m->b_wptr,xnals[i].p_payload+4,xnals[i].i_payload-4);
 116                 m->b_wptr+=xnals[i].i_payload-4;
 117                 if (xnals[i].i_type==7) {
 118                         ms_message("A SPS is being sent.");
 119                 }else if (xnals[i].i_type==8) {
 120                         ms_message("A PPS is being sent.");
 121                 }
 122                 ms_queue_put(nalus,m);
 123         }
 124 }
 125
 126 static void enc_process(MSFilter *f){
 127         EncData *d=(EncData*)f->data;
 128         uint32_t ts=f->ticker->time*90LL;
 129         mblk_t *im;
 130         MSPicture pic;
 131         MSQueue nalus;
 132         ms_queue_init(&nalus);
 133         while((im=ms_queue_get(f->inputs[0]))!=NULL){
 134                 if (yuv_buf_init_from_mblk(&pic,im)==0){
 135                         x264_picture_t xpic;
 136                         x264_picture_t oxpic;
 137                         x264_nal_t *xnals=NULL;
 138                         int num_nals=0;
 139
 140                         /*send I frame 2 seconds and 4 seconds after the beginning */
 141                         if (d->framenum==(int)d->fps*2 || d->framenum==(int)d->fps*4)
 142                                 d->generate_keyframe=TRUE;
 143
 144                         if (d->generate_keyframe){
 145                                 xpic.i_type=X264_TYPE_IDR;
 146                                 d->generate_keyframe=FALSE;
 147                         }else xpic.i_type=X264_TYPE_AUTO;
 148                         xpic.i_qpplus1=0;
 149                         xpic.i_pts=d->framenum;
 150                         xpic.param=NULL;
 151                         xpic.img.i_csp=X264_CSP_I420;
 152                         xpic.img.i_plane=3;
 153                         xpic.img.i_stride[0]=pic.strides[0];
 154                         xpic.img.i_stride[1]=pic.strides[1];
 155                         xpic.img.i_stride[2]=pic.strides[2];
 156                         xpic.img.i_stride[3]=0;
 157                         xpic.img.plane[0]=pic.planes[0];
 158                         xpic.img.plane[1]=pic.planes[1];
 159                         xpic.img.plane[2]=pic.planes[2];
 160                         xpic.img.plane[3]=0;
 161                         if (x264_encoder_encode(d->enc,&xnals,&num_nals,&xpic,&oxpic)>=0){
 162                                 x264_nals_to_msgb(xnals,num_nals,&nalus);
 163                                 rfc3984_pack(&d->packer,&nalus,f->outputs[0],ts);
 164                                 d->framenum++;
 165                         }else{
 166                                 ms_error("x264_encoder_encode() error.");
 167                         }
 168                 }
 169                 freemsg(im);
 170         }
 171 }
 172
 173 static void enc_postprocess(MSFilter *f){
 174         EncData *d=(EncData*)f->data;
 175         rfc3984_uninit(&d->packer);
 176         if (d->enc!=NULL){
 177                 x264_encoder_close(d->enc);
 178                 d->enc=NULL;
 179         }
 180 }
 181
 182 static int enc_set_br(MSFilter *f, void *arg){
 183         EncData *d=(EncData*)f->data;
 184         d->bitrate=*(int*)arg;
 185
 186         if (d->bitrate>=1024000){
 187                 d->vsize=MS_VIDEO_SIZE_VGA;
 188                 d->fps=25;
 189         }else if (d->bitrate>=512000){
 190                 d->vsize=MS_VIDEO_SIZE_VGA;
 191                 d->fps=15;
 192         }else if (d->bitrate>=384000){
 193                 d->vsize=MS_VIDEO_SIZE_CIF;
 194                 d->fps=30;
 195         }else if (d->bitrate>=256000){
 196                 d->vsize=MS_VIDEO_SIZE_CIF;
 197                 d->fps=15;
 198         }else if (d->bitrate>=128000){
 199                 d->vsize=MS_VIDEO_SIZE_CIF;
 200                 d->fps=15;
 201         }else if (d->bitrate>=64000){
 202                 d->vsize=MS_VIDEO_SIZE_CIF;
 203                 d->fps=10;
 204         }else if (d->bitrate>=32000){
 205                 d->vsize=MS_VIDEO_SIZE_QCIF;
 206                 d->fps=10;
 207         }else{
 208                 d->vsize=MS_VIDEO_SIZE_QCIF;
 209                 d->fps=5;
 210         }
 211         ms_message("bitrate set to %i",d->bitrate);
 212         return 0;
 213 }
 214
 215 static int enc_set_fps(MSFilter *f, void *arg){
 216         EncData *d=(EncData*)f->data;
 217         d->fps=*(float*)arg;
 218         return 0;
 219 }
 220
 221 static int enc_get_fps(MSFilter *f, void *arg){
 222         EncData *d=(EncData*)f->data;
 223         *(float*)arg=d->fps;
 224         return 0;
 225 }
 226
 227 static int enc_get_vsize(MSFilter *f, void *arg){
 228         EncData *d=(EncData*)f->data;
 229         *(MSVideoSize*)arg=d->vsize;
 230         return 0;
 231 }
 232
 233 static int enc_set_vsize(MSFilter *f, void *arg){
 234         EncData *d=(EncData*)f->data;
 235         d->vsize=*(MSVideoSize*)arg;
 236         return 0;
 237 }
 238
 239 static int enc_add_fmtp(MSFilter *f, void *arg){
 240         EncData *d=(EncData*)f->data;
 241         const char *fmtp=(const char *)arg;
 242         char value[12];
 243         if (fmtp_get_value(fmtp,"packetization-mode",value,sizeof(value))){
 244                 d->mode=atoi(value);
 245                 ms_message("packetization-mode set to %i",d->mode);
 246         }
 247         return 0;
 248 }
 249
 250 static int enc_req_vfu(MSFilter *f, void *arg){
 251         EncData *d=(EncData*)f->data;
 252         d->generate_keyframe=TRUE;
 253         return 0;
 254 }
 255
 256
 257 static MSFilterMethod enc_methods[]={
 258         {       MS_FILTER_SET_FPS       ,       enc_set_fps     },
 259         {       MS_FILTER_SET_BITRATE   ,       enc_set_br      },
 260         {       MS_FILTER_GET_FPS       ,       enc_get_fps     },
 261         {       MS_FILTER_GET_VIDEO_SIZE,       enc_get_vsize   },
 262         {       MS_FILTER_SET_VIDEO_SIZE,       enc_set_vsize   },
 263         {       MS_FILTER_ADD_FMTP      ,       enc_add_fmtp    },
 264         {       MS_FILTER_REQ_VFU       ,       enc_req_vfu     },
 265         {       0       ,                       NULL            }
 266 };
 267
 268 static MSFilterDesc x264_enc_desc={
 269         .id=MS_FILTER_PLUGIN_ID,
 270         .name="MSX264Enc",
 271         .text="A H264 encoder based on x264 project (with multislicing enabled)",
 272         .category=MS_FILTER_ENCODER,
 273         .enc_fmt="H264",
 274         .ninputs=1,
 275         .noutputs=1,
 276         .init=enc_init,
 277         .preprocess=enc_preprocess,
 278         .process=enc_process,
 279         .postprocess=enc_postprocess,
 280         .uninit=enc_uninit,
 281         .methods=enc_methods
 282 };
 283
 284 typedef struct _DecData{
 285         mblk_t *yuv_msg;
 286         mblk_t *sps,*pps;
 287         Rfc3984Context unpacker;
 288         MSPicture outbuf;
 289         struct SwsContext *sws_ctx;
 290         AVCodecContext av_context;
 291         unsigned int packet_num;
 292         uint8_t *bitstream;
 293         int bitstream_size;
 294 }DecData;
 295
 296 static void ffmpeg_init(){
 297         static bool_t done=FALSE;
 298         if (!done){
 299                 avcodec_init();
 300                 avcodec_register_all();
 301                 done=TRUE;
 302         }
 303 }
 304
 305 static void dec_open(DecData *d){
 306         AVCodec *codec;
 307         int error;
 308         codec=avcodec_find_decoder(CODEC_ID_H264);
 309         if (codec==NULL) ms_fatal("Could not find H264 decoder in ffmpeg.");
 310         avcodec_get_context_defaults(&d->av_context);
 311         error=avcodec_open(&d->av_context,codec);
 312         if (error!=0){
 313                 ms_fatal("avcodec_open() failed.");
 314         }
 315 }
 316
 317 static void dec_init(MSFilter *f){
 318         DecData *d=(DecData*)ms_new(DecData,1);
 319         ffmpeg_init();
 320         d->yuv_msg=NULL;
 321         d->sps=NULL;
 322         d->pps=NULL;
 323         d->sws_ctx=NULL;
 324         rfc3984_init(&d->unpacker);
 325         d->packet_num=0;
 326         dec_open(d);
 327         d->outbuf.w=0;
 328         d->outbuf.h=0;
 329         d->bitstream_size=65536;
 330         d->bitstream=ms_malloc0(d->bitstream_size);
 331         f->data=d;
 332 }
 333
 334 static void dec_reinit(DecData *d){
 335         avcodec_close(&d->av_context);
 336         dec_open(d);
 337 }
 338
 339 static void dec_uninit(MSFilter *f){
 340         DecData *d=(DecData*)f->data;
 341         rfc3984_uninit(&d->unpacker);
 342         avcodec_close(&d->av_context);
 343         if (d->yuv_msg) freemsg(d->yuv_msg);
 344         if (d->sps) freemsg(d->sps);
 345         if (d->pps) freemsg(d->pps);
 346         ms_free(d->bitstream);
 347         ms_free(d);
 348 }
 349
 350 static mblk_t *get_as_yuvmsg(MSFilter *f, DecData *s, AVFrame *orig){
 351         AVCodecContext *ctx=&s->av_context;
 352
 353         if (s->outbuf.w!=ctx->width || s->outbuf.h!=ctx->height){
 354                 if (s->sws_ctx!=NULL){
 355                         sws_freeContext(s->sws_ctx);
 356                         s->sws_ctx=NULL;
 357                         freemsg(s->yuv_msg);
 358                         s->yuv_msg=NULL;
 359                 }
 360                 ms_message("Getting yuv picture of %ix%i",ctx->width,ctx->height);
 361                 s->yuv_msg=yuv_buf_alloc(&s->outbuf,ctx->width,ctx->height);
 362                 s->outbuf.w=ctx->width;
 363                 s->outbuf.h=ctx->height;
 364                 s->sws_ctx=sws_getContext(ctx->width,ctx->height,ctx->pix_fmt,
 365                         ctx->width,ctx->height,PIX_FMT_YUV420P,SWS_FAST_BILINEAR,
 366                         NULL, NULL, NULL);
 367         }
 368         if (sws_scale(s->sws_ctx,orig->data,orig->linesize, 0,
 369                                         ctx->height, s->outbuf.planes, s->outbuf.strides)<0){
 370                 ms_error("%s: error in sws_scale().",f->desc->name);
 371         }
 372         return dupmsg(s->yuv_msg);
 373 }
 374
 375 static void update_sps(DecData *d, mblk_t *sps){
 376         if (d->sps)
 377                 freemsg(d->sps);
 378         d->sps=dupb(sps);
 379 }
 380
 381 static void update_pps(DecData *d, mblk_t *pps){
 382         if (d->pps)
 383                 freemsg(d->pps);
 384         if (pps) d->pps=dupb(pps);
 385         else d->pps=NULL;
 386 }
 387
 388 static bool_t check_sps_pps_change(DecData *d, mblk_t *sps, mblk_t *pps){
 389         bool_t ret1=FALSE,ret2=FALSE;
 390         if (d->sps){
 391                 if (sps){
 392                         ret1=(msgdsize(sps)!=msgdsize(d->sps)) || (memcmp(d->sps->b_rptr,sps->b_rptr,msgdsize(sps))!=0);
 393                         if (ret1) {
 394                                 update_sps(d,sps);
 395                                 ms_message("SPS changed !");
 396                                 update_pps(d,NULL);
 397                         }
 398                 }
 399         }else if (sps) {
 400                 ms_message("Receiving first SPS");
 401                 update_sps(d,sps);
 402         }
 403         if (d->pps){
 404                 if (pps){
 405                         ret2=(msgdsize(pps)!=msgdsize(d->pps)) || (memcmp(d->pps->b_rptr,pps->b_rptr,msgdsize(pps))!=0);
 406                         if (ret2) {
 407                                 update_sps(d,pps);
 408                                 ms_message("PPS changed ! %i,%i",msgdsize(pps),msgdsize(d->pps));
 409                         }
 410                 }
 411         }else if (pps) {
 412                 ms_message("Receiving first PPS");
 413                 update_pps(d,pps);
 414         }
 415         return ret1 || ret2;
 416 }
 417
 418 static void enlarge_bitstream(DecData *d, int new_size){
 419         d->bitstream_size=new_size;
 420         d->bitstream=ms_realloc(d->bitstream,d->bitstream_size);
 421 }
 422
 423 static int nalusToFrame(DecData *d, MSQueue *naluq, bool_t *new_sps_pps){
 424         mblk_t *im;
 425         uint8_t *dst=d->bitstream,*src,*end;
 426         int nal_len;
 427         bool_t start_picture=TRUE;
 428         uint8_t nalu_type;
 429         *new_sps_pps=FALSE;
 430         end=d->bitstream+d->bitstream_size;
 431         while((im=ms_queue_get(naluq))!=NULL){
 432                 src=im->b_rptr;
 433                 nal_len=im->b_wptr-src;
 434                 if (dst+nal_len+100>end){
 435                         int pos=dst-d->bitstream;
 436                         enlarge_bitstream(d, d->bitstream_size+nal_len+100);
 437                         dst=d->bitstream+pos;
 438                         end=d->bitstream+d->bitstream_size;
 439                 }
 440                 nalu_type=(*src) & ((1<<5)-1);
 441                 if (nalu_type==7)
 442                         *new_sps_pps=check_sps_pps_change(d,im,NULL) || *new_sps_pps;
 443                 if (nalu_type==8)
 444                         *new_sps_pps=check_sps_pps_change(d,NULL,im) || *new_sps_pps;
 445                 if (start_picture || nalu_type==7/*SPS*/ || nalu_type==8/*PPS*/ ){
 446                         *dst++=0;
 447                         start_picture=FALSE;
 448                 }
 449                 /*prepend nal marker*/
 450                 *dst++=0;
 451                 *dst++=0;
 452                 *dst++=1;
 453                 *dst++=*src++;
 454                 while(src<(im->b_wptr-3)){
 455                         if (src[0]==0 && src[1]==0 && src[2]<3){
 456                                 *dst++=0;
 457                                 *dst++=0;
 458                                 *dst++=3;
 459                                 src+=2;
 460                         }
 461                         *dst++=*src++;
 462                 }
 463                 *dst++=*src++;
 464                 *dst++=*src++;
 465                 *dst++=*src++;
 466                 freemsg(im);
 467         }
 468         return dst-d->bitstream;
 469 }
 470
 471 static void dec_process(MSFilter *f){
 472         DecData *d=(DecData*)f->data;
 473         mblk_t *im;
 474         MSQueue nalus;
 475         AVFrame orig;
 476         ms_queue_init(&nalus);
 477         while((im=ms_queue_get(f->inputs[0]))!=NULL){
 478                 /*push the sps/pps given in sprop-parameter-sets if any*/
 479                 if (d->packet_num==0 && d->sps && d->pps){
 480                         mblk_set_timestamp_info(d->sps,mblk_get_timestamp_info(im));
 481                         mblk_set_timestamp_info(d->pps,mblk_get_timestamp_info(im));
 482                         rfc3984_unpack(&d->unpacker,d->sps,&nalus);
 483                         rfc3984_unpack(&d->unpacker,d->pps,&nalus);
 484                         d->sps=NULL;
 485                         d->pps=NULL;
 486                 }
 487                 rfc3984_unpack(&d->unpacker,im,&nalus);
 488                 if (!ms_queue_empty(&nalus)){
 489                         int size;
 490                         uint8_t *p,*end;
 491                         bool_t need_reinit=FALSE;
 492
 493                         size=nalusToFrame(d,&nalus,&need_reinit);
 494                         if (need_reinit)
 495                                 dec_reinit(d);
 496                         p=d->bitstream;
 497                         end=d->bitstream+size;
 498                         while (end-p>0) {
 499                                 int len;
 500                                 int got_picture=0;
 501                                 AVPacket pkt;
 502                                 avcodec_get_frame_defaults(&orig);
 503                                 av_init_packet(&pkt);
 504                                 pkt.data = p;
 505                                 pkt.size = end-p;
 506                                 len=avcodec_decode_video2(&d->av_context,&orig,&got_picture,&pkt);
 507                                 if (len<=0) {
 508                                         ms_warning("ms_AVdecoder_process: error %i.",len);
 509                                         break;
 510                                 }
 511                                 if (got_picture) {
 512                                         ms_queue_put(f->outputs[0],get_as_yuvmsg(f,d,&orig));
 513                                 }
 514                                 p+=len;
 515                         }
 516                 }
 517                 d->packet_num++;
 518         }
 519 }
 520
 521 static int dec_add_fmtp(MSFilter *f, void *arg){
 522         DecData *d=(DecData*)f->data;
 523         const char *fmtp=(const char *)arg;
 524         char value[256];
 525         if (fmtp_get_value(fmtp,"sprop-parameter-sets",value,sizeof(value))){
 526                 char * b64_sps=value;
 527                 char * b64_pps=strchr(value,',');
 528                 if (b64_pps){
 529                         *b64_pps='\0';
 530                         ++b64_pps;
 531                         ms_message("Got sprop-parameter-sets : sps=%s , pps=%s",b64_sps,b64_pps);
 532                         d->sps=allocb(sizeof(value),0);
 533                         d->sps->b_wptr+=b64_decode(b64_sps,strlen(b64_sps),d->sps->b_wptr,sizeof(value));
 534                         d->pps=allocb(sizeof(value),0);
 535                         d->pps->b_wptr+=b64_decode(b64_pps,strlen(b64_pps),d->pps->b_wptr,sizeof(value));
 536                 }
 537         }
 538         return 0;
 539 }
 540
 541 static MSFilterMethod  h264_dec_methods[]={
 542         {       MS_FILTER_ADD_FMTP      ,       dec_add_fmtp    },
 543         {       0                       ,       NULL    }
 544 };
 545
 546 static MSFilterDesc h264_dec_desc={
 547         .id=MS_FILTER_PLUGIN_ID,
 548         .name="MSH264Dec",
 549         .text="A H264 decoder based on ffmpeg project.",
 550         .category=MS_FILTER_DECODER,
 551         .enc_fmt="H264",
 552         .ninputs=1,
 553         .noutputs=1,
 554         .init=dec_init,
 555         .process=dec_process,
 556         .uninit=dec_uninit,
 557         .methods=h264_dec_methods
 558 };
 559
 560 void libmsx264_init(void){
 561         ms_filter_register(&x264_enc_desc);
 562         ms_filter_register(&h264_dec_desc);
 563 }