Skip to content

Commit 958567b

Browse files
BrendonovichBrendonovich
committed
opus audio, stride correction
1 parent c81b818 commit 958567b

File tree

5 files changed

+109
-48
lines changed

5 files changed

+109
-48
lines changed

‎crates/media/src/data.rs

+3-1
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,9 @@ impl AudioInfo {
181181

182182
match self.channels {
183183
0 => unreachable!(),
184-
1 => frame.plane_data_mut(0)[0..data.len()].copy_from_slice(data),
184+
1 | _ if frame.is_packed() => {
185+
frame.plane_data_mut(0)[0..data.len()].copy_from_slice(data)
186+
}
185187
// cpal *always* returns interleaved data (i.e. the first sample from every channel, followed
186188
// by the second sample from every channel, et cetera). Many audio codecs work better/primarily
187189
// with planar data, so we de-interleave it here if there is more than one channel.

‎crates/media/src/encoders/mp4.rs

+60-38
Original file line numberDiff line numberDiff line change
@@ -81,45 +81,67 @@ impl MP4Encoder {
8181
};
8282

8383
let audio = if let Some(audio_config) = audio_config {
84-
let (mut audio_enc, audio_codec, output_format) = if cfg!(target_os = "macos") {
85-
let audio_codec = encoder::find_by_name("aac_at")
86-
.ok_or(MediaError::TaskLaunch("Could not find AAC codec".into()))?;
87-
let mut audio_ctx = context::Context::new_with_codec(audio_codec);
88-
audio_ctx.set_threading(Config::count(4));
89-
let mut audio_enc = audio_ctx.encoder().audio()?;
90-
91-
let output_format = ffmpeg::format::Sample::I16(format::sample::Type::Planar);
92-
93-
audio_enc.set_flags(ffmpeg::codec::Flags::QSCALE);
94-
audio_enc.set_quality(10 * FF_QP2LAMBDA as usize);
95-
96-
(audio_enc, audio_codec, output_format)
97-
} else {
98-
let audio_codec = encoder::find_by_name("aac")
99-
.ok_or(MediaError::TaskLaunch("Could not find AAC codec".into()))?;
100-
let mut audio_ctx = context::Context::new_with_codec(audio_codec);
101-
audio_ctx.set_threading(Config::count(4));
102-
let mut audio_enc = audio_ctx.encoder().audio()?;
103-
104-
audio_enc.set_bit_rate(128 * 1000);
105-
let output_format = ffmpeg::format::Sample::F32(format::sample::Type::Planar);
106-
107-
if !audio_codec
108-
.audio()
109-
.unwrap()
110-
.rates()
111-
.into_iter()
112-
.flatten()
113-
.any(|r| r == audio_config.rate())
114-
{
115-
return Err(MediaError::TaskLaunch(format!(
116-
"AAC Codec does not support sample rate {}",
117-
audio_config.rate()
118-
)));
119-
}
84+
let audio_codec = encoder::find_by_name("libopus")
85+
.ok_or(MediaError::TaskLaunch("Could not find Opus codec".into()))?;
86+
let mut audio_ctx = context::Context::new_with_codec(audio_codec);
87+
let mut audio_enc = audio_ctx.encoder().audio()?;
88+
89+
audio_enc.set_bit_rate(128 * 1000);
90+
let output_format = ffmpeg::format::Sample::F32(format::sample::Type::Packed);
91+
92+
if !audio_codec
93+
.audio()
94+
.unwrap()
95+
.rates()
96+
.into_iter()
97+
.flatten()
98+
.any(|r| r == audio_config.rate())
99+
{
100+
return Err(MediaError::TaskLaunch(format!(
101+
"AAC Codec does not support sample rate {}",
102+
audio_config.rate()
103+
)));
104+
}
120105

121-
(audio_enc, audio_codec, output_format)
122-
};
106+
// let (mut audio_enc, audio_codec, output_format) = if cfg!(target_os = "macos") {
107+
// let audio_codec = encoder::find_by_name("aac_at")
108+
// .ok_or(MediaError::TaskLaunch("Could not find AAC codec".into()))?;
109+
// let mut audio_ctx = context::Context::new_with_codec(audio_codec);
110+
// audio_ctx.set_threading(Config::count(4));
111+
// let mut audio_enc = audio_ctx.encoder().audio()?;
112+
113+
// let output_format = ffmpeg::format::Sample::I16(format::sample::Type::Planar);
114+
115+
// audio_enc.set_flags(ffmpeg::codec::Flags::QSCALE);
116+
// audio_enc.set_quality(10 * FF_QP2LAMBDA as usize);
117+
118+
// (audio_enc, audio_codec, output_format)
119+
// } else {
120+
// let audio_codec = encoder::find_by_name("aac")
121+
// .ok_or(MediaError::TaskLaunch("Could not find AAC codec".into()))?;
122+
// let mut audio_ctx = context::Context::new_with_codec(audio_codec);
123+
// audio_ctx.set_threading(Config::count(4));
124+
// let mut audio_enc = audio_ctx.encoder().audio()?;
125+
126+
// audio_enc.set_bit_rate(128 * 1000);
127+
// let output_format = ffmpeg::format::Sample::F32(format::sample::Type::Planar);
128+
129+
// if !audio_codec
130+
// .audio()
131+
// .unwrap()
132+
// .rates()
133+
// .into_iter()
134+
// .flatten()
135+
// .any(|r| r == audio_config.rate())
136+
// {
137+
// return Err(MediaError::TaskLaunch(format!(
138+
// "AAC Codec does not support sample rate {}",
139+
// audio_config.rate()
140+
// )));
141+
// }
142+
143+
// (audio_enc, audio_codec, output_format)
144+
// };
123145

124146
audio_enc.set_rate(audio_config.rate());
125147
audio_enc.set_format(output_format);

‎crates/media/src/feeds/audio.rs

+6-2
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,9 @@ impl F32Resampler {
415415
self.resampled_samples += self.resampled_frame.samples();
416416

417417
write_f32_ne_bytes(
418-
&self.resampled_frame.data(0)[0..self.resampled_frame.samples() * f32::BYTE_SIZE],
418+
&self.resampled_frame.data(0)[0..self.resampled_frame.samples()
419+
* f32::BYTE_SIZE
420+
* self.resampled_frame.channels() as usize],
419421
&mut self.buf,
420422
);
421423

@@ -431,7 +433,9 @@ impl F32Resampler {
431433
self.resampled_samples += self.resampled_frame.samples();
432434

433435
write_f32_ne_bytes(
434-
&self.resampled_frame.data(0)[0..self.resampled_frame.samples() * f32::BYTE_SIZE],
436+
&self.resampled_frame.data(0)[0..self.resampled_frame.samples()
437+
* f32::BYTE_SIZE
438+
* self.resampled_frame.channels() as usize],
435439
&mut self.buf,
436440
);
437441

‎crates/media/src/pipeline/audio_buffer.rs

+22-6
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,15 @@ impl AudioBuffer {
3939
if let Some(pts) = frame.pts() {
4040
self.current_pts = pts;
4141
}
42-
for channel in 0..self.config.channels {
43-
self.data[channel].extend(frame.plane_data(channel));
42+
if frame.is_planar() {
43+
for channel in 0..self.config.channels {
44+
self.data[channel].extend(frame.plane_data(channel));
45+
}
46+
} else {
47+
self.data[0].extend(
48+
&frame.data(0)
49+
[0..frame.samples() * frame.channels() as usize * frame.format().bytes()],
50+
);
4451
}
4552
}
4653

@@ -51,16 +58,25 @@ impl AudioBuffer {
5158

5259
let frame_size = self.frame_size * self.config.sample_size();
5360

54-
if self.len() < frame_size {
61+
if self.len() < frame_size * self.config.channels {
5562
return None;
5663
}
5764

5865
let mut frame = self.config.empty_frame(self.frame_size);
5966
frame.set_pts(Some(self.current_pts));
6067

61-
for channel in 0..self.config.channels {
62-
for (index, byte) in self.data[channel].drain(0..frame_size).enumerate() {
63-
frame.plane_data_mut(channel)[index] = byte;
68+
if frame.is_planar() {
69+
for channel in 0..self.config.channels {
70+
for (index, byte) in self.data[channel].drain(0..frame_size).enumerate() {
71+
frame.plane_data_mut(channel)[index] = byte;
72+
}
73+
}
74+
} else {
75+
for (index, byte) in self.data[0]
76+
.drain(0..frame_size * self.config.channels)
77+
.enumerate()
78+
{
79+
frame.plane_data_mut(0)[index] = byte;
6480
}
6581
}
6682

‎crates/rendering/src/decoder/avassetreader.rs

+18-1
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,24 @@ impl CachedFrame {
129129
let mut rgb_frame = frame::Video::empty();
130130
converter.run(&ffmpeg_frame, &mut rgb_frame).unwrap();
131131

132-
rgb_frame.data(0).to_vec()
132+
let slice = rgb_frame.data(0);
133+
let width = rgb_frame.width();
134+
let height = rgb_frame.height();
135+
let bytes_per_row = rgb_frame.stride(0);
136+
let row_length = width * 4;
137+
138+
let mut bytes = vec![0; (width * height * 4) as usize];
139+
140+
// TODO: allow for decoded frames to have stride, handle stride in shaders
141+
for i in 0..height as usize {
142+
bytes.as_mut_slice()[i * row_length as usize..(i + 1) * row_length as usize]
143+
.copy_from_slice(
144+
&slice
145+
[(i * bytes_per_row)..i * bytes_per_row + row_length as usize],
146+
)
147+
}
148+
149+
bytes
133150
};
134151

135152
let data = Arc::new(data);

0 commit comments

Comments
 (0)