| /* |
| * Copyright (c) 2023, Alliance for Open Media. All rights reserved |
| * |
| * This source code is subject to the terms of the BSD 3-Clause Clear License |
| * and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear |
| * License was not distributed with this source code in the LICENSE file, you |
| * can obtain it at www.aomedia.org/license/software-license/bsd-3-c-c. If the |
| * Alliance for Open Media Patent License 1.0 was not distributed with this |
| * source code in the PATENTS file, you can obtain it at |
| * www.aomedia.org/license/patent. |
| */ |
| #include "iamf/cli/rendering_mix_presentation_finalizer.h" |
| |
| #include <algorithm> |
| #include <cmath> |
| #include <cstddef> |
| #include <cstdint> |
| #include <cstring> |
| #include <functional> |
| #include <list> |
| #include <memory> |
| #include <string> |
| #include <utility> |
| #include <vector> |
| |
| #include "absl/base/nullability.h" |
| #include "absl/container/flat_hash_map.h" |
| #include "absl/container/flat_hash_set.h" |
| #include "absl/functional/any_invocable.h" |
| #include "absl/log/check.h" |
| #include "absl/log/log.h" |
| #include "absl/status/status.h" |
| #include "absl/strings/str_cat.h" |
| #include "absl/strings/string_view.h" |
| #include "absl/time/clock.h" |
| #include "absl/time/time.h" |
| #include "absl/types/span.h" |
| #include "iamf/cli/audio_element_with_data.h" |
| #include "iamf/cli/cli_util.h" |
| #include "iamf/cli/demixing_module.h" |
| #include "iamf/cli/loudness_calculator_base.h" |
| #include "iamf/cli/loudness_calculator_factory_base.h" |
| #include "iamf/cli/parameter_block_with_data.h" |
| #include "iamf/cli/renderer/audio_element_renderer_base.h" |
| #include "iamf/cli/renderer_factory.h" |
| #include "iamf/cli/sample_processor_base.h" |
| #include "iamf/common/utils/macros.h" |
| #include "iamf/common/utils/numeric_utils.h" |
| #include "iamf/common/utils/sample_processing_utils.h" |
| #include "iamf/common/utils/validation_utils.h" |
| #include "iamf/obu/audio_element.h" |
| #include "iamf/obu/codec_config.h" |
| #include "iamf/obu/mix_presentation.h" |
| #include "iamf/obu/param_definitions.h" |
| #include "iamf/obu/parameter_block.h" |
| #include "iamf/obu/types.h" |
| |
| namespace iamf_tools { |
| |
| namespace { |
| |
| using LayoutRenderingMetadata = |
| RenderingMixPresentationFinalizer::LayoutRenderingMetadata; |
| using SubmixRenderingMetadata = |
| RenderingMixPresentationFinalizer::SubmixRenderingMetadata; |
| |
| bool CanRenderAnyLayout( |
| const std::vector<SubmixRenderingMetadata>& rendering_metadata) { |
| for (auto& submix_rendering_metadata : rendering_metadata) { |
| for (auto& layout_rendering_metadata : |
| submix_rendering_metadata.layout_rendering_metadata) { |
| if (layout_rendering_metadata.can_render) { |
| return true; |
| } |
| } |
| } |
| return false; |
| } |
| |
| absl::Status CollectAudioElementsInSubMix( |
| const absl::flat_hash_map<uint32_t, AudioElementWithData>& audio_elements, |
| const std::vector<SubMixAudioElement>& sub_mix_audio_elements, |
| std::vector<const AudioElementWithData*>& audio_elements_in_sub_mix) { |
| audio_elements_in_sub_mix.reserve(sub_mix_audio_elements.size()); |
| for (const auto& audio_element : sub_mix_audio_elements) { |
| auto iter = audio_elements.find(audio_element.audio_element_id); |
| if (iter == audio_elements.end()) { |
| return absl::InvalidArgumentError(absl::StrCat( |
| "Audio Element with ID= ", audio_element.audio_element_id, |
| " not found")); |
| } |
| audio_elements_in_sub_mix.push_back(&iter->second); |
| } |
| |
| return absl::OkStatus(); |
| } |
| |
| absl::Status GetCommonCodecConfigPropertiesFromAudioElementIds( |
| const std::vector<const AudioElementWithData*>& audio_elements_in_sub_mix, |
| uint32_t& common_sample_rate, uint8_t& common_bit_depth, |
| uint32_t& common_num_samples_per_frame, bool& requires_resampling) { |
| absl::flat_hash_set<uint32_t> sample_rates; |
| absl::flat_hash_set<uint32_t> num_samples_per_frame; |
| absl::flat_hash_set<uint8_t> bit_depths; |
| |
| // Get all the bit-depths and sample_rates from each Audio Element. |
| for (const auto* audio_element : audio_elements_in_sub_mix) { |
| num_samples_per_frame.insert( |
| audio_element->codec_config->GetNumSamplesPerFrame()); |
| sample_rates.insert(audio_element->codec_config->GetOutputSampleRate()); |
| bit_depths.insert( |
| audio_element->codec_config->GetBitDepthToMeasureLoudness()); |
| } |
| |
| RETURN_IF_NOT_OK(GetCommonSampleRateAndBitDepth( |
| sample_rates, bit_depths, common_sample_rate, common_bit_depth, |
| requires_resampling)); |
| if (num_samples_per_frame.size() != 1) { |
| return absl::InvalidArgumentError( |
| "Audio elements in a submix must have the same number of samples per " |
| "frame."); |
| } |
| common_num_samples_per_frame = *num_samples_per_frame.begin(); |
| |
| return absl::OkStatus(); |
| } |
| |
| using AudioElementRenderingMetadata = |
| RenderingMixPresentationFinalizer::AudioElementRenderingMetadata; |
| |
| absl::Status InitializeRenderingMetadata( |
| const RendererFactoryBase& renderer_factory, |
| const std::vector<const AudioElementWithData*>& audio_elements_in_sub_mix, |
| const std::vector<SubMixAudioElement>& sub_mix_audio_elements, |
| const Layout& loudness_layout, const uint32_t common_sample_rate, |
| std::vector<AudioElementRenderingMetadata>& rendering_metadata_array) { |
| rendering_metadata_array.resize(audio_elements_in_sub_mix.size()); |
| |
| for (int i = 0; i < audio_elements_in_sub_mix.size(); i++) { |
| const auto& sub_mix_audio_element = *audio_elements_in_sub_mix[i]; |
| auto& rendering_metadata = rendering_metadata_array[i]; |
| rendering_metadata.audio_element = &(sub_mix_audio_element.obu); |
| rendering_metadata.codec_config = sub_mix_audio_element.codec_config; |
| rendering_metadata.renderer = renderer_factory.CreateRendererForLayout( |
| sub_mix_audio_element.obu.audio_substream_ids_, |
| sub_mix_audio_element.substream_id_to_labels, |
| rendering_metadata.audio_element->GetAudioElementType(), |
| sub_mix_audio_element.obu.config_, |
| sub_mix_audio_elements[i].rendering_config, loudness_layout, |
| static_cast<size_t>( |
| rendering_metadata.codec_config->GetNumSamplesPerFrame())); |
| |
| if (rendering_metadata.renderer == nullptr) { |
| return absl::UnknownError("Unable to create renderer."); |
| } |
| |
| const uint32_t output_sample_rate = |
| sub_mix_audio_element.codec_config->GetOutputSampleRate(); |
| if (common_sample_rate != output_sample_rate) { |
| // Theoretically, we would have to resample this audio element to the |
| // common sample rate. However, as of IAMF v1.1.0, the spec forbids |
| // multiple Codec Config OBUs. This case is not possible to occur with a |
| // single Codec Config OBU. |
| return absl::UnimplementedError( |
| absl::StrCat("OBUs with different sample rates not supported yet: (", |
| common_sample_rate, " != ", output_sample_rate, ").")); |
| } |
| } |
| |
| return absl::OkStatus(); |
| } |
| |
| absl::Status FlushUntilNonEmptyOrTimeout( |
| AudioElementRendererBase& audio_element_renderer, |
| std::vector<InternalSampleType>& rendered_samples) { |
| static const int kMaxNumTries = 500; |
| for (int i = 0; i < kMaxNumTries; i++) { |
| RETURN_IF_NOT_OK(audio_element_renderer.Flush(rendered_samples)); |
| if (!rendered_samples.empty()) { |
| // Usually samples will be ready right away. So avoid sleeping. |
| return absl::OkStatus(); |
| } |
| absl::SleepFor(absl::Milliseconds(10)); |
| } |
| return absl::DeadlineExceededError("Timed out waiting for samples."); |
| } |
| |
| absl::Status RenderLabeledFrameToLayout( |
| const LabeledFrame& labeled_frame, |
| const AudioElementRenderingMetadata& rendering_metadata, |
| std::vector<InternalSampleType>& rendered_samples) { |
| const auto num_time_ticks = |
| rendering_metadata.renderer->RenderLabeledFrame(labeled_frame); |
| if (!num_time_ticks.ok()) { |
| return num_time_ticks.status(); |
| } else if (*num_time_ticks > |
| static_cast<size_t>( |
| rendering_metadata.codec_config->GetNumSamplesPerFrame())) { |
| return absl::InvalidArgumentError("Too many samples in this frame"); |
| } else if (*num_time_ticks == 0) { |
| // This was an empty frame. |
| return absl::OkStatus(); |
| } |
| |
| return FlushUntilNonEmptyOrTimeout(*rendering_metadata.renderer, |
| rendered_samples); |
| } |
| |
| absl::Status GetParameterBlockLinearMixGainsPerTick( |
| uint32_t common_sample_rate, const ParameterBlockWithData& parameter_block, |
| const MixGainParamDefinition& mix_gain, |
| std::vector<float>& linear_mix_gain_per_tick) { |
| if (mix_gain.parameter_rate_ != common_sample_rate) { |
| // TODO(b/283281856): Support resampling parameter blocks. |
| return absl::UnimplementedError( |
| "Parameter blocks that require resampling are not supported yet."); |
| } |
| |
| const int16_t default_mix_gain = mix_gain.default_mix_gain_; |
| // Initialize to the default gain value. |
| std::fill(linear_mix_gain_per_tick.begin(), linear_mix_gain_per_tick.end(), |
| std::pow(10.0f, Q7_8ToFloat(default_mix_gain) / 20.0f)); |
| |
| InternalTimestamp cur_tick = parameter_block.start_timestamp; |
| // Process as many ticks as possible until all are found or the parameter |
| // block ends. |
| while (cur_tick < parameter_block.end_timestamp && |
| (cur_tick - parameter_block.start_timestamp) < |
| linear_mix_gain_per_tick.size()) { |
| RETURN_IF_NOT_OK(parameter_block.obu->GetLinearMixGain( |
| cur_tick - parameter_block.start_timestamp, |
| linear_mix_gain_per_tick[cur_tick - parameter_block.start_timestamp])); |
| cur_tick++; |
| } |
| return absl::OkStatus(); |
| } |
| |
| // Fills in the output `mix_gains` with the gain in Q7.8 format to apply at each |
| // tick. |
| // TODO(b/288073842): Consider improving computational efficiency instead of |
| // searching through all parameter blocks for each frame. |
| // TODO(b/379961928): Remove this function once the new |
| // `GetParameterBlockLinearMixGainsPerTick()` is in use. |
| absl::Status GetParameterBlockLinearMixGainsPerTick( |
| uint32_t common_sample_rate, InternalTimestamp start_timestamp, |
| InternalTimestamp end_timestamp, |
| const std::list<ParameterBlockWithData>& parameter_blocks, |
| const MixGainParamDefinition& mix_gain, |
| std::vector<float>& linear_mix_gain_per_tick) { |
| if (mix_gain.parameter_rate_ != common_sample_rate) { |
| // TODO(b/283281856): Support resampling parameter blocks. |
| return absl::UnimplementedError( |
| "Parameter blocks that require resampling are not supported yet."); |
| } |
| |
| const auto parameter_id = mix_gain.parameter_id_; |
| const int16_t default_mix_gain = mix_gain.default_mix_gain_; |
| |
| // Initialize to the default gain value. |
| std::fill(linear_mix_gain_per_tick.begin(), linear_mix_gain_per_tick.end(), |
| std::pow(10.0f, Q7_8ToFloat(default_mix_gain) / 20.0f)); |
| |
| InternalTimestamp cur_tick = start_timestamp; |
| |
| // Find the mix gain at each tick. May terminate early if there are samples to |
| // trim at the end. |
| while (cur_tick < end_timestamp && |
| (cur_tick - start_timestamp) < linear_mix_gain_per_tick.size()) { |
| // Find the parameter block that this tick occurs during. |
| const auto parameter_block_iter = std::find_if( |
| parameter_blocks.begin(), parameter_blocks.end(), |
| [cur_tick, parameter_id](const auto& parameter_block) { |
| return parameter_block.obu->parameter_id_ == parameter_id && |
| parameter_block.start_timestamp <= cur_tick && |
| cur_tick < parameter_block.end_timestamp; |
| }); |
| if (parameter_block_iter == parameter_blocks.end()) { |
| // Default mix gain will be used for this frame. Logic elsewhere validates |
| // the rest of the audio frames have consistent coverage. |
| break; |
| } |
| |
| // Process as many ticks as possible until all are found or the parameter |
| // block ends. |
| while (cur_tick < end_timestamp && |
| cur_tick < parameter_block_iter->end_timestamp && |
| (cur_tick - start_timestamp) < linear_mix_gain_per_tick.size()) { |
| RETURN_IF_NOT_OK(parameter_block_iter->obu->GetLinearMixGain( |
| cur_tick - parameter_block_iter->start_timestamp, |
| linear_mix_gain_per_tick[cur_tick - start_timestamp])); |
| cur_tick++; |
| } |
| } |
| |
| return absl::OkStatus(); |
| } |
| |
| absl::Status GetAndApplyMixGain( // NOLINT |
| uint32_t common_sample_rate, const ParameterBlockWithData& parameter_block, |
| const MixGainParamDefinition& mix_gain, int32_t num_channels, |
| std::vector<InternalSampleType>& rendered_samples) { |
| if (rendered_samples.size() % num_channels != 0) { |
| return absl::InvalidArgumentError(absl::StrCat( |
| "Expected an integer number of interlaced channels. " |
| "renderered_samples.size()= ", |
| rendered_samples.size(), ", num_channels= ", num_channels)); |
| } |
| |
| // Get the mix gain on a per tick basis from the parameter block. |
| std::vector<float> linear_mix_gain_per_tick(rendered_samples.size() / |
| num_channels); |
| RETURN_IF_NOT_OK(GetParameterBlockLinearMixGainsPerTick( |
| common_sample_rate, parameter_block, mix_gain, linear_mix_gain_per_tick)); |
| |
| if (!linear_mix_gain_per_tick.empty()) { |
| LOG_FIRST_N(INFO, 6) << " First tick in this frame has gain: " |
| << linear_mix_gain_per_tick.front(); |
| } |
| |
| for (int tick = 0; tick < linear_mix_gain_per_tick.size(); tick++) { |
| for (int channel = 0; channel < num_channels; channel++) { |
| // Apply the same mix gain to all `num_channels` associated with this |
| // tick. |
| rendered_samples[tick * num_channels + channel] *= |
| linear_mix_gain_per_tick[tick]; |
| } |
| } |
| |
| return absl::OkStatus(); |
| } |
| |
| // TODO(b/379961928): Remove once the new GetAndApplyMixGain is in use. |
| absl::Status GetAndApplyMixGain( |
| uint32_t common_sample_rate, InternalTimestamp start_timestamp, |
| InternalTimestamp end_timestamp, |
| const std::list<ParameterBlockWithData>& parameter_blocks, |
| const MixGainParamDefinition& mix_gain, int32_t num_channels, |
| std::vector<float>& linear_mix_gain_per_tick, |
| std::vector<InternalSampleType>& rendered_samples) { |
| if (rendered_samples.size() % num_channels != 0) { |
| return absl::InvalidArgumentError(absl::StrCat( |
| "Expected an integer number of interlaced channels. " |
| "renderered_samples.size()= ", |
| rendered_samples.size(), ", num_channels= ", num_channels)); |
| } |
| |
| // Get the mix gain on a per tick basis from the parameter block. |
| linear_mix_gain_per_tick.resize(rendered_samples.size() / num_channels, 0.0f); |
| RETURN_IF_NOT_OK(GetParameterBlockLinearMixGainsPerTick( |
| common_sample_rate, start_timestamp, end_timestamp, parameter_blocks, |
| mix_gain, linear_mix_gain_per_tick)); |
| |
| if (!linear_mix_gain_per_tick.empty()) { |
| LOG_FIRST_N(INFO, 6) << " First tick in this frame has gain: " |
| << linear_mix_gain_per_tick.front(); |
| } |
| |
| for (int tick = 0; tick < linear_mix_gain_per_tick.size(); tick++) { |
| for (int channel = 0; channel < num_channels; channel++) { |
| // Apply the same mix gain to all `num_channels` associated with this |
| // tick. |
| rendered_samples[tick * num_channels + channel] *= |
| linear_mix_gain_per_tick[tick]; |
| } |
| } |
| |
| return absl::OkStatus(); |
| } |
| |
| absl::Status MixAudioElements( |
| std::vector<std::vector<InternalSampleType>>& rendered_audio_elements, |
| std::vector<InternalSampleType>& rendered_samples) { |
| const size_t num_samples = rendered_audio_elements.empty() |
| ? 0 |
| : rendered_audio_elements.front().size(); |
| rendered_samples.reserve(num_samples); |
| |
| for (const auto& rendered_audio_element : rendered_audio_elements) { |
| if (rendered_audio_element.size() != num_samples) { |
| return absl::UnknownError( |
| "Expected all frames to have the same number of samples."); |
| } |
| } |
| |
| for (int i = 0; i < num_samples; i++) { |
| InternalSampleType mixed_sample = 0; |
| // Sum all audio elements for this tick. |
| for (const auto& rendered_audio_element : rendered_audio_elements) { |
| mixed_sample += rendered_audio_element[i]; |
| } |
| // Push the clipped result. |
| rendered_samples.push_back(mixed_sample); |
| } |
| |
| return absl::OkStatus(); |
| } |
| |
| // Fills in `valid_rendered_samples` which is a view backed by |
| // `rendered_samples` of the ticks actually rendered. |
| absl::Status RenderAllFramesForLayout( |
| int32_t num_channels, |
| const std::vector<SubMixAudioElement> sub_mix_audio_elements, |
| const MixGainParamDefinition& output_mix_gain, |
| const IdLabeledFrameMap& id_to_labeled_frame, |
| const std::vector<AudioElementRenderingMetadata>& rendering_metadata_array, |
| InternalTimestamp start_timestamp, InternalTimestamp end_timestamp, |
| const std::list<ParameterBlockWithData>& parameter_blocks, |
| const uint32_t common_sample_rate, |
| std::vector<std::vector<int32_t>>& rendered_samples, |
| absl::Span<const std::vector<int32_t>>& valid_rendered_samples) { |
| // Each audio element rendered individually with `element_mix_gain` applied. |
| std::vector<std::vector<InternalSampleType>> rendered_audio_elements( |
| sub_mix_audio_elements.size()); |
| std::vector<float> linear_mix_gain_per_tick; |
| for (int i = 0; i < sub_mix_audio_elements.size(); i++) { |
| const SubMixAudioElement& sub_mix_audio_element = sub_mix_audio_elements[i]; |
| const auto audio_element_id = sub_mix_audio_element.audio_element_id; |
| const auto& rendering_metadata = rendering_metadata_array[i]; |
| |
| if (id_to_labeled_frame.find(audio_element_id) != |
| id_to_labeled_frame.end()) { |
| const auto& labeled_frame = id_to_labeled_frame.at(audio_element_id); |
| // Render the frame to the specified `loudness_layout` and apply element |
| // mix gain. |
| RETURN_IF_NOT_OK(RenderLabeledFrameToLayout( |
| labeled_frame, rendering_metadata, rendered_audio_elements[i])); |
| } |
| RETURN_IF_NOT_OK(GetAndApplyMixGain( |
| common_sample_rate, start_timestamp, end_timestamp, parameter_blocks, |
| sub_mix_audio_element.element_mix_gain, num_channels, |
| linear_mix_gain_per_tick, rendered_audio_elements[i])); |
| } |
| |
| // Mix the audio elements. |
| std::vector<InternalSampleType> rendered_samples_internal; |
| RETURN_IF_NOT_OK( |
| MixAudioElements(rendered_audio_elements, rendered_samples_internal)); |
| |
| LOG_FIRST_N(INFO, 1) << " Applying output_mix_gain.default_mix_gain= " |
| << output_mix_gain.default_mix_gain_; |
| |
| RETURN_IF_NOT_OK( |
| GetAndApplyMixGain(common_sample_rate, start_timestamp, end_timestamp, |
| parameter_blocks, output_mix_gain, num_channels, |
| linear_mix_gain_per_tick, rendered_samples_internal)); |
| |
| // Convert the rendered samples to int32, clipping if needed. |
| size_t num_ticks = 0; |
| RETURN_IF_NOT_OK(ConvertInterleavedToTimeChannel( |
| absl::MakeConstSpan(rendered_samples_internal), num_channels, |
| absl::AnyInvocable<absl::Status(InternalSampleType, int32_t&) const>( |
| NormalizedFloatingPointToInt32<InternalSampleType>), |
| rendered_samples, num_ticks)); |
| valid_rendered_samples = |
| absl::MakeConstSpan(rendered_samples).first(num_ticks); |
| return absl::OkStatus(); |
| } |
| |
| absl::Status ValidateUserLoudness(const LoudnessInfo& user_loudness, |
| const uint32_t mix_presentation_id, |
| const int sub_mix_index, |
| const int layout_index, |
| const LoudnessInfo& output_loudness, |
| bool& loudness_matches_user_data) { |
| const std::string mix_presentation_sub_mix_layout_index = |
| absl::StrCat("Mix Presentation(ID ", mix_presentation_id, ")->sub_mixes[", |
| sub_mix_index, "]->layouts[", layout_index, "]: "); |
| if (output_loudness.integrated_loudness != |
| user_loudness.integrated_loudness) { |
| LOG(ERROR) << mix_presentation_sub_mix_layout_index |
| << "Computed integrated loudness different from " |
| << "user specification: " << output_loudness.integrated_loudness |
| << " vs " << user_loudness.integrated_loudness; |
| loudness_matches_user_data = false; |
| } |
| |
| if (output_loudness.digital_peak != user_loudness.digital_peak) { |
| LOG(ERROR) << mix_presentation_sub_mix_layout_index |
| << "Computed digital peak different from " |
| << "user specification: " << output_loudness.digital_peak |
| << " vs " << user_loudness.digital_peak; |
| loudness_matches_user_data = false; |
| } |
| |
| if (output_loudness.info_type & LoudnessInfo::kTruePeak) { |
| if (output_loudness.true_peak != user_loudness.true_peak) { |
| LOG(ERROR) << mix_presentation_sub_mix_layout_index |
| << "Computed true peak different from " |
| << "user specification: " << output_loudness.true_peak |
| << " vs " << user_loudness.true_peak; |
| loudness_matches_user_data = false; |
| } |
| } |
| |
| // Anchored loudness and layout extension are copied from the user input |
| // and do not need to be validated. |
| |
| return absl::OkStatus(); |
| } |
| |
| // Calculates the loudness of the rendered samples. These rendered samples are |
| // for a specific timestamp for a given submix and layout. If |
| // `validate_loudness` is true, then the user provided loudness values are |
| // validated against the computed values. |
| absl::Status UpdateLoudnessInfoForLayout( |
| bool validate_loudness, const LoudnessInfo& input_loudness, |
| const uint32_t mix_presentation_id, const int sub_mix_index, |
| const int layout_index, bool& loudness_matches_user_data, |
| std::unique_ptr<LoudnessCalculatorBase> loudness_calculator, |
| LoudnessInfo& output_calculated_loudness) { |
| // Copy the final loudness values back to the output OBU. |
| auto calculated_loudness_info = loudness_calculator->QueryLoudness(); |
| if (!calculated_loudness_info.ok()) { |
| return calculated_loudness_info.status(); |
| } |
| |
| if (validate_loudness) { |
| // Validate any user provided loudness values match computed values. |
| RETURN_IF_NOT_OK(ValidateUserLoudness( |
| input_loudness, mix_presentation_id, sub_mix_index, layout_index, |
| *calculated_loudness_info, loudness_matches_user_data)); |
| } |
| output_calculated_loudness = *calculated_loudness_info; |
| return absl::OkStatus(); |
| } |
| |
| // Generates rendering metadata for all layouts within a submix. This includes |
| // optionally creating a sample processor and/or a loudness calculator for each |
| // layout. |
| absl::Status GenerateRenderingMetadataForLayouts( |
| const RendererFactoryBase& renderer_factory, |
| const LoudnessCalculatorFactoryBase* loudness_calculator_factory, |
| const RenderingMixPresentationFinalizer::SampleProcessorFactory& |
| sample_processor_factory, |
| const DecodedUleb128 mix_presentation_id, |
| const MixPresentationSubMix& sub_mix, int sub_mix_index, |
| const std::vector<const AudioElementWithData*>& audio_elements_in_sub_mix, |
| uint32_t common_sample_rate, uint8_t rendering_bit_depth, |
| uint32_t common_num_samples_per_frame, |
| std::vector<LayoutRenderingMetadata>& output_layout_rendering_metadata) { |
| output_layout_rendering_metadata.resize(sub_mix.layouts.size()); |
| for (int layout_index = 0; layout_index < sub_mix.layouts.size(); |
| layout_index++) { |
| LayoutRenderingMetadata& layout_rendering_metadata = |
| output_layout_rendering_metadata[layout_index]; |
| const auto& layout = sub_mix.layouts[layout_index]; |
| |
| int32_t num_channels = 0; |
| auto can_render_status = MixPresentationObu::GetNumChannelsFromLayout( |
| layout.loudness_layout, num_channels); |
| layout_rendering_metadata.num_channels = num_channels; |
| |
| can_render_status.Update(InitializeRenderingMetadata( |
| renderer_factory, audio_elements_in_sub_mix, sub_mix.audio_elements, |
| layout.loudness_layout, common_sample_rate, |
| layout_rendering_metadata.audio_element_rendering_metadata)); |
| |
| if (!can_render_status.ok()) { |
| layout_rendering_metadata.can_render = false; |
| continue; |
| } else { |
| layout_rendering_metadata.can_render = true; |
| } |
| if (loudness_calculator_factory != nullptr) { |
| // Optionally create a loudness calculator. |
| layout_rendering_metadata.loudness_calculator = |
| loudness_calculator_factory->CreateLoudnessCalculator( |
| layout, common_num_samples_per_frame, common_sample_rate, |
| rendering_bit_depth); |
| } |
| // Optionally create a post-processor. |
| layout_rendering_metadata.sample_processor = sample_processor_factory( |
| mix_presentation_id, sub_mix_index, layout_index, |
| layout.loudness_layout, num_channels, common_sample_rate, |
| rendering_bit_depth, common_num_samples_per_frame); |
| |
| // Pre-allocate a buffer to store a frame's worth of rendered samples. |
| layout_rendering_metadata.rendered_samples.resize( |
| common_num_samples_per_frame, std::vector<int32_t>(num_channels, 0)); |
| } |
| |
| return absl::OkStatus(); |
| } |
| |
| // We generate one rendering metadata object for each submix. Once this |
| // metadata is generated, we will loop through it to render all submixes |
| // for a given timestamp. Within a submix, there can be many different audio |
| // elements and layouts that need to be rendered as well. Not all of these |
| // need to be rendered; only the ones that either have a wav writer or a |
| // loudness calculator. |
| absl::Status GenerateRenderingMetadataForSubmixes( |
| const RendererFactoryBase& renderer_factory, |
| absl::Nullable<const LoudnessCalculatorFactoryBase*> |
| loudness_calculator_factory, |
| const RenderingMixPresentationFinalizer::SampleProcessorFactory& |
| sample_processor_factory, |
| const absl::flat_hash_map<uint32_t, AudioElementWithData>& audio_elements, |
| const MixPresentationObu& mix_presentation_obu, |
| std::vector<SubmixRenderingMetadata>& output_rendering_metadata) { |
| const auto mix_presentation_id = mix_presentation_obu.GetMixPresentationId(); |
| output_rendering_metadata.resize(mix_presentation_obu.sub_mixes_.size()); |
| for (int sub_mix_index = 0; |
| sub_mix_index < mix_presentation_obu.sub_mixes_.size(); |
| ++sub_mix_index) { |
| SubmixRenderingMetadata& submix_rendering_metadata = |
| output_rendering_metadata[sub_mix_index]; |
| const auto& sub_mix = mix_presentation_obu.sub_mixes_[sub_mix_index]; |
| |
| // Pointers to audio elements in this sub mix; useful later. |
| std::vector<const AudioElementWithData*> audio_elements_in_sub_mix; |
| RETURN_IF_NOT_OK(CollectAudioElementsInSubMix( |
| audio_elements, sub_mix.audio_elements, audio_elements_in_sub_mix)); |
| |
| submix_rendering_metadata.audio_elements_in_sub_mix = |
| sub_mix.audio_elements; |
| submix_rendering_metadata.mix_gain = |
| std::make_unique<MixGainParamDefinition>(sub_mix.output_mix_gain); |
| |
| // Data common to all audio elements and layouts. |
| bool requires_resampling; |
| uint32_t common_num_samples_per_frame; |
| uint8_t rendering_bit_depth; |
| RETURN_IF_NOT_OK(GetCommonCodecConfigPropertiesFromAudioElementIds( |
| audio_elements_in_sub_mix, submix_rendering_metadata.common_sample_rate, |
| rendering_bit_depth, common_num_samples_per_frame, |
| requires_resampling)); |
| if (requires_resampling) { |
| // Detected multiple Codec Config OBUs with different sample rates or |
| // bit-depths. As of IAMF v1.1.0, multiple Codec Config OBUs in the same |
| // IA sequence are never permitted. The spec implies we would have to |
| // resample to a common sample rate and/or bit-depth. |
| return absl::UnimplementedError( |
| "This implementation does not support mixing Codec Config OBUs with " |
| "different sample rates or bit-depths."); |
| } |
| RETURN_IF_NOT_OK(GenerateRenderingMetadataForLayouts( |
| renderer_factory, loudness_calculator_factory, sample_processor_factory, |
| mix_presentation_id, sub_mix, sub_mix_index, audio_elements_in_sub_mix, |
| submix_rendering_metadata.common_sample_rate, rendering_bit_depth, |
| common_num_samples_per_frame, |
| submix_rendering_metadata.layout_rendering_metadata)); |
| } |
| return absl::OkStatus(); |
| } |
| |
| absl::Status FlushPostProcessors( |
| std::vector<SubmixRenderingMetadata>& rendering_metadata) { |
| for (auto& submix_rendering_metadata : rendering_metadata) { |
| for (auto& layout_rendering_metadata : |
| submix_rendering_metadata.layout_rendering_metadata) { |
| if (layout_rendering_metadata.sample_processor != nullptr) { |
| RETURN_IF_NOT_OK(layout_rendering_metadata.sample_processor->Flush()); |
| } |
| } |
| } |
| |
| return absl::OkStatus(); |
| } |
| |
| absl::Status FillLoudnessForMixPresentation( |
| bool validate_loudness, |
| std::vector<SubmixRenderingMetadata>& rendering_metadata, |
| MixPresentationObu& mix_presentation_obu) { |
| bool loudness_matches_user_data = true; |
| int submix_index = 0; |
| for (auto& submix_rendering_metadata : rendering_metadata) { |
| int layout_index = 0; |
| for (auto& layout_rendering_metadata : |
| submix_rendering_metadata.layout_rendering_metadata) { |
| if (layout_rendering_metadata.loudness_calculator != nullptr) { |
| RETURN_IF_NOT_OK(UpdateLoudnessInfoForLayout( |
| validate_loudness, |
| mix_presentation_obu.sub_mixes_[submix_index] |
| .layouts[layout_index] |
| .loudness, |
| mix_presentation_obu.GetMixPresentationId(), submix_index, |
| layout_index, loudness_matches_user_data, |
| std::move(layout_rendering_metadata.loudness_calculator), |
| mix_presentation_obu.sub_mixes_[submix_index] |
| .layouts[layout_index] |
| .loudness)); |
| } |
| layout_index++; |
| } |
| submix_index++; |
| } |
| if (!loudness_matches_user_data) { |
| return absl::InvalidArgumentError("Loudness does not match user data."); |
| } |
| return absl::OkStatus(); |
| } |
| |
| // Renders all submixes, layouts, and audio elements for a temporal unit. It |
| // then optionally writes the rendered samples to a wav file and/or calculates |
| // the loudness of the rendered samples. |
| absl::Status RenderWriteAndCalculateLoudnessForTemporalUnit( |
| const IdLabeledFrameMap& id_to_labeled_frame, |
| InternalTimestamp start_timestamp, InternalTimestamp end_timestamp, |
| const std::list<ParameterBlockWithData>& parameter_blocks, |
| std::vector<SubmixRenderingMetadata>& rendering_metadata) { |
| for (auto& submix_rendering_metadata : rendering_metadata) { |
| for (auto& layout_rendering_metadata : |
| submix_rendering_metadata.layout_rendering_metadata) { |
| if (!layout_rendering_metadata.can_render) { |
| continue; |
| } |
| if (submix_rendering_metadata.mix_gain == nullptr) { |
| return absl::InvalidArgumentError("Submix mix gain is null"); |
| } |
| |
| RETURN_IF_NOT_OK(RenderAllFramesForLayout( |
| layout_rendering_metadata.num_channels, |
| submix_rendering_metadata.audio_elements_in_sub_mix, |
| *submix_rendering_metadata.mix_gain, id_to_labeled_frame, |
| layout_rendering_metadata.audio_element_rendering_metadata, |
| start_timestamp, end_timestamp, parameter_blocks, |
| submix_rendering_metadata.common_sample_rate, |
| layout_rendering_metadata.rendered_samples, |
| layout_rendering_metadata.valid_rendered_samples)); |
| |
| // Calculate loudness based on the original rendered samples; we do not |
| // know what post-processing the end user will have. |
| if (layout_rendering_metadata.loudness_calculator != nullptr) { |
| RETURN_IF_NOT_OK( |
| layout_rendering_metadata.loudness_calculator |
| ->AccumulateLoudnessForSamples( |
| layout_rendering_metadata.valid_rendered_samples)); |
| } |
| |
| // Perform any post-processing. |
| if (layout_rendering_metadata.sample_processor != nullptr) { |
| RETURN_IF_NOT_OK(layout_rendering_metadata.sample_processor->PushFrame( |
| layout_rendering_metadata.valid_rendered_samples)); |
| } |
| } |
| } |
| return absl::OkStatus(); |
| } |
| |
| absl::StatusOr<const LayoutRenderingMetadata*> |
| GetRenderedSamplesAndPostProcessor( |
| const absl::flat_hash_map<DecodedUleb128, |
| std::vector<SubmixRenderingMetadata>>& |
| mix_presentation_id_to_sub_mix_rendering_metadata, |
| DecodedUleb128 mix_presentation_id, size_t sub_mix_index, |
| size_t layout_index) { |
| // Lookup the requested layout in the requested mix presentation. |
| const auto sub_mix_rendering_metadata_it = |
| mix_presentation_id_to_sub_mix_rendering_metadata.find( |
| mix_presentation_id); |
| const auto mix_presentation_id_error_message = |
| absl::StrCat(" Mix Presentation ID ", mix_presentation_id); |
| if (sub_mix_rendering_metadata_it == |
| mix_presentation_id_to_sub_mix_rendering_metadata.end()) { |
| return absl::NotFoundError( |
| absl::StrCat(mix_presentation_id_error_message, |
| " not found in rendering metadata.")); |
| } |
| |
| // Validate the sub mix and layout are in bounds, then retrieve it. |
| const auto& [unused_mix_presentation_id, sub_mix_rendering_metadatas] = |
| *sub_mix_rendering_metadata_it; |
| RETURN_IF_NOT_OK(Validate( |
| sub_mix_index, std::less<size_t>(), sub_mix_rendering_metadatas.size(), |
| absl::StrCat(mix_presentation_id_error_message, " sub_mix_index <"))); |
| RETURN_IF_NOT_OK(Validate( |
| layout_index, std::less<size_t>(), |
| sub_mix_rendering_metadatas[sub_mix_index] |
| .layout_rendering_metadata.size(), |
| absl::StrCat(mix_presentation_id_error_message, " layout_index <"))); |
| return &sub_mix_rendering_metadatas[sub_mix_index] |
| .layout_rendering_metadata[layout_index]; |
| } |
| |
| } // namespace |
| |
| absl::StatusOr<RenderingMixPresentationFinalizer> |
| RenderingMixPresentationFinalizer::Create( |
| absl::Nullable<const RendererFactoryBase*> renderer_factory, |
| absl::Nullable<const LoudnessCalculatorFactoryBase*> |
| loudness_calculator_factory, |
| const absl::flat_hash_map<uint32_t, AudioElementWithData>& audio_elements, |
| const SampleProcessorFactory& sample_processor_factory, |
| const std::list<MixPresentationObu>& mix_presentation_obus) { |
| const bool rendering_enabled = renderer_factory != nullptr; |
| if (!rendering_enabled) { |
| LOG(INFO) << "Rendering is safely disabled."; |
| } |
| if (loudness_calculator_factory == nullptr) { |
| LOG(INFO) << "Loudness calculator factory is null so loudness will not be " |
| "calculated."; |
| } |
| absl::flat_hash_map<DecodedUleb128, std::vector<SubmixRenderingMetadata>> |
| mix_presentation_id_to_rendering_metadata; |
| std::list<MixPresentationObu> mix_presentation_obus_to_render; |
| for (const auto& mix_presentation_obu : mix_presentation_obus) { |
| // Copy all mix presentation OBUs, so they can be echoed back, even when |
| // rendering is disabled. |
| mix_presentation_obus_to_render.emplace_back(mix_presentation_obu); |
| |
| // Fill in rendering metadata if rendering is enabled, and at least one |
| // layout can be rendered. |
| if (rendering_enabled) { |
| std::vector<SubmixRenderingMetadata> temp_sub_mix_rendering_metadata; |
| RETURN_IF_NOT_OK(GenerateRenderingMetadataForSubmixes( |
| *renderer_factory, loudness_calculator_factory, |
| sample_processor_factory, audio_elements, mix_presentation_obu, |
| temp_sub_mix_rendering_metadata)); |
| if (CanRenderAnyLayout(temp_sub_mix_rendering_metadata)) { |
| mix_presentation_id_to_rendering_metadata.emplace( |
| mix_presentation_obu.GetMixPresentationId(), |
| std::move(temp_sub_mix_rendering_metadata)); |
| } |
| } |
| } |
| |
| return RenderingMixPresentationFinalizer( |
| std::move(mix_presentation_id_to_rendering_metadata), |
| std::move(mix_presentation_obus_to_render)); |
| } |
| |
| absl::Status RenderingMixPresentationFinalizer::PushTemporalUnit( |
| const IdLabeledFrameMap& id_to_labeled_frame, |
| InternalTimestamp start_timestamp, InternalTimestamp end_timestamp, |
| const std::list<ParameterBlockWithData>& parameter_blocks) { |
| switch (state_) { |
| case kAcceptingTemporalUnits: |
| // Ok to push. |
| break; |
| case kFinalizePushTemporalUnitCalled: |
| return absl::FailedPreconditionError( |
| "PushTemporalUnit() should not be called after " |
| "FinalizePushingTemporalUnits() has been called."); |
| case kFlushedFinalizedMixPresentationObus: |
| return absl::FailedPreconditionError( |
| "PushTemporalUnit() should not be called after " |
| "GetFinalizedMixPresentationOBUs() has been called."); |
| } |
| for (auto& [mix_presentation_ids, sub_mix_rendering_metadata] : |
| mix_presentation_id_to_sub_mix_rendering_metadata_) { |
| RETURN_IF_NOT_OK(RenderWriteAndCalculateLoudnessForTemporalUnit( |
| id_to_labeled_frame, start_timestamp, end_timestamp, parameter_blocks, |
| sub_mix_rendering_metadata)); |
| } |
| return absl::OkStatus(); |
| } |
| |
| absl::StatusOr<absl::Span<const std::vector<int32_t>>> |
| RenderingMixPresentationFinalizer::GetPostProcessedSamplesAsSpan( |
| DecodedUleb128 mix_presentation_id, size_t sub_mix_index, |
| size_t layout_index) const { |
| const auto layout_rendering_metadata = GetRenderedSamplesAndPostProcessor( |
| mix_presentation_id_to_sub_mix_rendering_metadata_, mix_presentation_id, |
| sub_mix_index, layout_index); |
| if (!layout_rendering_metadata.ok()) { |
| return layout_rendering_metadata.status(); |
| } |
| // `absl::StatusOr<const T*> cannot hold a nullptr. |
| CHECK_NE(*layout_rendering_metadata, nullptr); |
| |
| // Prioritize returning the post-processed samples if a post-processor is |
| // available. Otherwise, return the rendered samples. |
| return (*layout_rendering_metadata)->sample_processor != nullptr |
| ? (*layout_rendering_metadata) |
| ->sample_processor->GetOutputSamplesAsSpan() |
| : (*layout_rendering_metadata)->valid_rendered_samples; |
| } |
| |
| absl::Status RenderingMixPresentationFinalizer::FinalizePushingTemporalUnits() { |
| switch (state_) { |
| case kAcceptingTemporalUnits: |
| state_ = kFinalizePushTemporalUnitCalled; |
| break; |
| case kFinalizePushTemporalUnitCalled: |
| case kFlushedFinalizedMixPresentationObus: |
| return absl::FailedPreconditionError( |
| "FinalizePushingTemporalUnits() should not be called twice."); |
| } |
| |
| for (auto& [mix_presentation_id, sub_mix_rendering_metadata] : |
| mix_presentation_id_to_sub_mix_rendering_metadata_) { |
| RETURN_IF_NOT_OK(FlushPostProcessors(sub_mix_rendering_metadata)); |
| } |
| return absl::OkStatus(); |
| } |
| |
| absl::StatusOr<std::list<MixPresentationObu>> |
| RenderingMixPresentationFinalizer::GetFinalizedMixPresentationObus( |
| bool validate_loudness) { |
| switch (state_) { |
| case kAcceptingTemporalUnits: |
| return absl::FailedPreconditionError( |
| "FinalizePushingTemporalUnits() should be called before " |
| "GetFinalizedMixPresentationOBUs()."); |
| case kFinalizePushTemporalUnitCalled: |
| // Ok to finalize. |
| break; |
| case kFlushedFinalizedMixPresentationObus: |
| return absl::FailedPreconditionError( |
| "GetFinalizedMixPresentationOBUs() should not be called twice."); |
| } |
| |
| // Finalize the OBUs in place. |
| for (auto& mix_presentation_obu : mix_presentation_obus_) { |
| const auto sub_mix_rendering_metadata_it = |
| mix_presentation_id_to_sub_mix_rendering_metadata_.find( |
| mix_presentation_obu.GetMixPresentationId()); |
| if (sub_mix_rendering_metadata_it == |
| mix_presentation_id_to_sub_mix_rendering_metadata_.end()) { |
| LOG(INFO) << "Rendering was disabled for Mix Presentation ID= " |
| << mix_presentation_obu.GetMixPresentationId() |
| << " echoing the input OBU."; |
| continue; |
| } |
| |
| RETURN_IF_NOT_OK(FillLoudnessForMixPresentation( |
| validate_loudness, sub_mix_rendering_metadata_it->second, |
| mix_presentation_obu)); |
| mix_presentation_obu.PrintObu(); |
| } |
| |
| // Flush the finalized OBUs and mark that this class should not use them |
| // again. |
| state_ = kFlushedFinalizedMixPresentationObus; |
| return std::move(mix_presentation_obus_); |
| } |
| |
| } // namespace iamf_tools |
