Skip to content

Commit 2b4c38b

Browse files
torokati44torokati44
committed
video/external: Reorder frames decoded by OpenH264 manually
1 parent 41e4c65 commit 2b4c38b

File tree

4 files changed

+103
-43
lines changed

4 files changed

+103
-43
lines changed

‎core/src/display_object/video.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,7 @@ impl<'gc> Video<'gc> {
312312
codec: streamdef.codec,
313313
data: &read.movie.data()[*slice_start..*slice_end],
314314
frame_id,
315+
composition_time: None,
315316
};
316317
context
317318
.video
@@ -409,6 +410,7 @@ impl<'gc> TDisplayObject<'gc> for Video<'gc> {
409410
codec: streamdef.codec,
410411
data: &movie.data()[*frame_start..*frame_end],
411412
frame_id: *frame_id,
413+
composition_time: None,
412414
},
413415
);
414416

‎core/src/streams.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -947,6 +947,7 @@ impl<'gc> NetStream<'gc> {
947947
codec,
948948
data, //TODO: ScreenVideo's decoder wants the FLV header bytes
949949
frame_id,
950+
composition_time: Some(write.stream_time as i32),
950951
};
951952

952953
if let Err(e) = context
@@ -961,6 +962,7 @@ impl<'gc> NetStream<'gc> {
961962
codec,
962963
data, //TODO: ScreenVideo's decoder wants the FLV header bytes
963964
frame_id,
965+
composition_time: Some(write.stream_time as i32),
964966
};
965967

966968
match context.video.decode_video_stream_frame(
@@ -998,14 +1000,15 @@ impl<'gc> NetStream<'gc> {
9981000
Some(video_handle),
9991001
Some(codec),
10001002
FlvVideoPacket::AvcNalu {
1001-
composition_time_offset: _,
1003+
composition_time_offset,
10021004
data,
10031005
},
10041006
) => {
10051007
let encoded_frame = EncodedFrame {
10061008
codec,
10071009
data,
10081010
frame_id,
1011+
composition_time: Some(write.stream_time as i32 + composition_time_offset),
10091012
};
10101013

10111014
match context.video.decode_video_stream_frame(

‎video/external/src/decoder/openh264.rs‎

Lines changed: 95 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use core::slice;
2+
use std::collections::BinaryHeap;
23
use std::ffi::{c_int, c_uchar};
34
use std::fmt::Display;
45
use std::fs::File;
@@ -192,13 +193,42 @@ impl OpenH264Codec {
192193
}
193194
}
194195

196+
struct EnqueuedFrame {
197+
composition_time: i32,
198+
frame: DecodedFrame,
199+
}
200+
201+
impl PartialEq for EnqueuedFrame {
202+
fn eq(&self, other: &Self) -> bool {
203+
self.composition_time == other.composition_time
204+
}
205+
}
206+
207+
impl Eq for EnqueuedFrame {}
208+
209+
impl Ord for EnqueuedFrame {
210+
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
211+
// Note the reversal: BinaryHeap is a max-heap, but we always
212+
// want to pop the frame with the lowest timestamp.
213+
self.composition_time.cmp(&other.composition_time).reverse()
214+
}
215+
}
216+
217+
impl PartialOrd for EnqueuedFrame {
218+
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
219+
Some(self.cmp(other))
220+
}
221+
}
222+
195223
/// H264 video decoder.
196224
pub struct H264Decoder {
197225
/// How many bytes are used to store the length of the NALU (1, 2, 3, or 4).
198226
length_size: u8,
199227

200228
openh264: Arc<OpenH264>,
201229
decoder: *mut ISVCDecoder,
230+
231+
frame_reorder_queue: BinaryHeap<EnqueuedFrame>,
202232
}
203233

204234
struct OpenH264Data {
@@ -227,6 +257,7 @@ impl H264Decoder {
227257
length_size: 0,
228258
openh264,
229259
decoder,
260+
frame_reorder_queue: BinaryHeap::new(),
230261
}
231262
}
232263
}
@@ -366,55 +397,77 @@ impl VideoDecoder for H264Decoder {
366397
));
367398
}
368399
if dest_buf_info.iBufferStatus != 1 {
369-
return Err(Error::DecoderError(
370-
"No output frame produced by the decoder".into(),
371-
));
372-
}
373-
let buffer_info = dest_buf_info.UsrData.sSystemBuffer;
374-
if buffer_info.iFormat != videoFormatI420 as c_int {
375-
return Err(Error::DecoderError(
376-
format!("Unexpected output format: {}", buffer_info.iFormat).into(),
377-
));
400+
let ret = decoder_vtbl.FlushFrame.unwrap()(
401+
self.decoder,
402+
output.as_mut_ptr(),
403+
&mut dest_buf_info as *mut openh264_sys::SBufferInfo,
404+
);
405+
406+
if ret != 0 {
407+
return Err(Error::DecoderError(
408+
format!("Flushing failed with status code: {}", ret).into(),
409+
));
410+
}
378411
}
379412

380-
let mut yuv: Vec<u8> = Vec::with_capacity(
381-
buffer_info.iWidth as usize * buffer_info.iHeight as usize * 3 / 2,
382-
);
413+
if dest_buf_info.iBufferStatus == 1 {
414+
let buffer_info = dest_buf_info.UsrData.sSystemBuffer;
415+
if buffer_info.iFormat != videoFormatI420 as c_int {
416+
return Err(Error::DecoderError(
417+
format!("Unexpected output format: {}", buffer_info.iFormat).into(),
418+
));
419+
}
383420

384-
// Copying Y
385-
for i in 0..buffer_info.iHeight {
386-
yuv.extend_from_slice(slice::from_raw_parts(
387-
output[0].offset((i * buffer_info.iStride[0]) as isize),
388-
buffer_info.iWidth as usize,
389-
));
390-
}
421+
let mut yuv: Vec<u8> = Vec::with_capacity(
422+
buffer_info.iWidth as usize * buffer_info.iHeight as usize * 3 / 2,
423+
);
391424

392-
// Copying U
393-
for i in 0..buffer_info.iHeight / 2 {
394-
yuv.extend_from_slice(slice::from_raw_parts(
395-
output[1].offset((i * buffer_info.iStride[1]) as isize),
396-
buffer_info.iWidth as usize / 2,
397-
));
398-
}
425+
// Copying Y
426+
for i in 0..buffer_info.iHeight {
427+
yuv.extend_from_slice(slice::from_raw_parts(
428+
output[0].offset((i * buffer_info.iStride[0]) as isize),
429+
buffer_info.iWidth as usize,
430+
));
431+
}
399432

400-
// Copying V
401-
for i in 0..buffer_info.iHeight / 2 {
402-
yuv.extend_from_slice(slice::from_raw_parts(
403-
output[2].offset((i * buffer_info.iStride[1]) as isize),
404-
buffer_info.iWidth as usize / 2,
405-
));
433+
// Copying U
434+
for i in 0..buffer_info.iHeight / 2 {
435+
yuv.extend_from_slice(slice::from_raw_parts(
436+
output[1].offset((i * buffer_info.iStride[1]) as isize),
437+
buffer_info.iWidth as usize / 2,
438+
));
439+
}
440+
441+
// Copying V
442+
for i in 0..buffer_info.iHeight / 2 {
443+
yuv.extend_from_slice(slice::from_raw_parts(
444+
output[2].offset((i * buffer_info.iStride[1]) as isize),
445+
buffer_info.iWidth as usize / 2,
446+
));
447+
}
448+
449+
// TODO: Check whether frames are being squished/stretched, or cropped,
450+
// when encoded image size doesn't match declared video tag size.
451+
// NOTE: This will always use the BT.601 coefficients, which may or may
452+
// not be correct. So far I haven't seen anything to the contrary in FP.
453+
self.frame_reorder_queue.push(EnqueuedFrame {
454+
composition_time: encoded_frame
455+
.composition_time
456+
.ok_or(Error::DecoderError("No composition time provided".into()))?,
457+
frame: DecodedFrame::new(
458+
buffer_info.iWidth as u32,
459+
buffer_info.iHeight as u32,
460+
BitmapFormat::Yuv420p,
461+
yuv,
462+
),
463+
});
406464
}
407465

408-
// TODO: Check whether frames are being squished/stretched, or cropped,
409-
// when encoded image size doesn't match declared video tag size.
410-
// NOTE: This will always use the BT.601 coefficients, which may or may
411-
// not be correct. So far I haven't seen anything to the contrary in FP.
412-
Ok(DecodedFrame::new(
413-
buffer_info.iWidth as u32,
414-
buffer_info.iHeight as u32,
415-
BitmapFormat::Yuv420p,
416-
yuv,
417-
))
466+
if self.frame_reorder_queue.len() >= 3 {
467+
Ok(self.frame_reorder_queue.pop().unwrap().frame)
468+
} else {
469+
Err(Error::DecoderError("Not enough frames decoded yet".into()))
470+
}
418471
}
419472
}
420473
}

‎video/src/frame.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ pub struct EncodedFrame<'a> {
1212
/// A caller-specified frame ID. Frame IDs must be consistent between
1313
/// subsequent uses of the same data stream.
1414
pub frame_id: u32,
15+
16+
pub composition_time: Option<i32>,
1517
}
1618

1719
impl<'a> EncodedFrame<'a> {

0 commit comments

Comments
 (0)