From a5607066484b2e9495064344410416f4697f3fbe Mon Sep 17 00:00:00 2001 From: Sil Vilerino Date: Fri, 10 Nov 2023 14:13:26 -0500 Subject: [PATCH] d3d12: Implement HEVC VUI Writer Reviewed-by: Jesse Natalie Part-of: --- src/gallium/drivers/d3d12/d3d12_video_enc.h | 1 + .../drivers/d3d12/d3d12_video_enc_hevc.cpp | 13 ++- ...2_video_encoder_bitstream_builder_hevc.cpp | 83 ++++++++++++++++++- ...d12_video_encoder_bitstream_builder_hevc.h | 1 + .../d3d12_video_encoder_nalu_writer_hevc.cpp | 71 +++++++++++++++- .../d3d12_video_encoder_nalu_writer_hevc.h | 43 ++++++++++ 6 files changed, 208 insertions(+), 4 deletions(-) diff --git a/src/gallium/drivers/d3d12/d3d12_video_enc.h b/src/gallium/drivers/d3d12/d3d12_video_enc.h index 699278c9f33..0444681a500 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_enc.h +++ b/src/gallium/drivers/d3d12/d3d12_video_enc.h @@ -276,6 +276,7 @@ struct D3D12EncodeConfiguration } m_encoderCodecSpecificStateDescAV1; struct pipe_h264_enc_seq_param m_encoderCodecSpecificSequenceStateDescH264; + struct pipe_h265_enc_seq_param m_encoderCodecSpecificSequenceStateDescH265; }; struct EncodedBitstreamResolvedMetadata diff --git a/src/gallium/drivers/d3d12/d3d12_video_enc_hevc.cpp b/src/gallium/drivers/d3d12/d3d12_video_enc_hevc.cpp index 849424ae000..fdfa45ef3dd 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_enc_hevc.cpp +++ b/src/gallium/drivers/d3d12/d3d12_video_enc_hevc.cpp @@ -752,6 +752,14 @@ d3d12_video_encoder_update_current_encoder_config_state_hevc(struct d3d12_video_ } pD3D12Enc->m_currentEncodeConfig.m_encoderCodecDesc = D3D12_VIDEO_ENCODER_CODEC_HEVC; + // Set Sequence information + if (memcmp(&pD3D12Enc->m_currentEncodeConfig.m_encoderCodecSpecificSequenceStateDescH265, + &hevcPic->seq, + sizeof(hevcPic->seq)) != 0) { + pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags |= d3d12_video_encoder_config_dirty_flag_sequence_info; + } + pD3D12Enc->m_currentEncodeConfig.m_encoderCodecSpecificSequenceStateDescH265 = hevcPic->seq; + // Set input format DXGI_FORMAT targetFmt = d3d12_convert_pipe_video_profile_to_dxgi_format(pD3D12Enc->base.profile); if (pD3D12Enc->m_currentEncodeConfig.m_encodeFormatInfo.Format != targetFmt) { @@ -956,7 +964,9 @@ d3d12_video_encoder_build_codec_headers_hevc(struct d3d12_video_encoder *pD3D12E bool isFirstFrame = (pD3D12Enc->m_fenceValue == 1); bool writeNewSPS = isFirstFrame // on first frame || ((pD3D12Enc->m_currentEncodeConfig.m_seqFlags & // also on resolution change - D3D12_VIDEO_ENCODER_SEQUENCE_CONTROL_FLAG_RESOLUTION_CHANGE) != 0); + D3D12_VIDEO_ENCODER_SEQUENCE_CONTROL_FLAG_RESOLUTION_CHANGE) != 0) + // Also on input format dirty flag for new SPS, VUI etc + || (pD3D12Enc->m_currentEncodeConfig.m_ConfigDirtyFlags & d3d12_video_encoder_config_dirty_flag_sequence_info); d3d12_video_bitstream_builder_hevc *pHEVCBitstreamBuilder = static_cast(pD3D12Enc->m_upBitstreamBuilder.get()); @@ -989,6 +999,7 @@ d3d12_video_encoder_build_codec_headers_hevc(struct d3d12_video_encoder *pD3D12E pHEVCBitstreamBuilder->build_sps( pHEVCBitstreamBuilder->get_latest_vps(), + pD3D12Enc->m_currentEncodeConfig.m_encoderCodecSpecificSequenceStateDescH265, active_seq_parameter_set_id, pD3D12Enc->m_currentEncodeConfig.m_currentResolution, pD3D12Enc->m_currentEncodeConfig.m_FrameCroppingCodecConfig, diff --git a/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_hevc.cpp b/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_hevc.cpp index f01b3df9dbf..5c9ce488c35 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_hevc.cpp +++ b/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_hevc.cpp @@ -354,6 +354,7 @@ d3d12_video_bitstream_builder_hevc::build_vps(const D3D12_VIDEO_ENCODER_PROFILE_ void d3d12_video_bitstream_builder_hevc::build_sps(const HevcVideoParameterSet& parentVPS, + const struct pipe_h265_enc_seq_param & seqData, uint8_t seq_parameter_set_id, const D3D12_VIDEO_ENCODER_PICTURE_RESOLUTION_DESC& encodeResolution, const D3D12_BOX& crop_window_upper_layer, @@ -437,7 +438,47 @@ d3d12_video_bitstream_builder_hevc::build_sps(const HevcVideoParameterSet& paren m_latest_sps.sps_temporal_mvp_enabled_flag = 0; m_latest_sps.strong_intra_smoothing_enabled_flag = 0; - m_latest_sps.vui_parameters_present_flag = 0; + m_latest_sps.vui_parameters_present_flag = seqData.vui_parameters_present_flag; + m_latest_sps.vui.aspect_ratio_idc = seqData.aspect_ratio_idc; + m_latest_sps.vui.sar_width = seqData.sar_width; + m_latest_sps.vui.sar_height = seqData.sar_height; + m_latest_sps.vui.video_format = seqData.video_format; + m_latest_sps.vui.video_full_range_flag = seqData.video_full_range_flag; + m_latest_sps.vui.colour_primaries = seqData.colour_primaries; + m_latest_sps.vui.transfer_characteristics = seqData.transfer_characteristics; + m_latest_sps.vui.matrix_coeffs = seqData.matrix_coefficients; + m_latest_sps.vui.chroma_sample_loc_type_top_field = seqData.chroma_sample_loc_type_top_field; + m_latest_sps.vui.chroma_sample_loc_type_bottom_field = seqData.chroma_sample_loc_type_bottom_field; + m_latest_sps.vui.def_disp_win_left_offset = seqData.def_disp_win_left_offset; + m_latest_sps.vui.def_disp_win_right_offset = seqData.def_disp_win_right_offset; + m_latest_sps.vui.def_disp_win_top_offset = seqData.def_disp_win_top_offset; + m_latest_sps.vui.def_disp_win_bottom_offset = seqData.def_disp_win_bottom_offset; + m_latest_sps.vui.num_units_in_tick = seqData.num_units_in_tick; + m_latest_sps.vui.time_scale = seqData.time_scale; + m_latest_sps.vui.num_ticks_poc_diff_one_minus1 = seqData.num_ticks_poc_diff_one_minus1; + m_latest_sps.vui.min_spatial_segmentation_idc = seqData.min_spatial_segmentation_idc; + m_latest_sps.vui.max_bytes_per_pic_denom = seqData.max_bytes_per_pic_denom; + m_latest_sps.vui.max_bits_per_min_cu_denom = seqData.max_bits_per_min_cu_denom; + m_latest_sps.vui.log2_max_mv_length_horizontal = seqData.log2_max_mv_length_horizontal; + m_latest_sps.vui.log2_max_mv_length_vertical = seqData.log2_max_mv_length_vertical; + m_latest_sps.vui.aspect_ratio_info_present_flag = seqData.vui_flags.aspect_ratio_info_present_flag; + m_latest_sps.vui.timing_info_present_flag = seqData.vui_flags.timing_info_present_flag; + m_latest_sps.vui.video_signal_type_present_flag = seqData.vui_flags.video_signal_type_present_flag; + m_latest_sps.vui.colour_description_present_flag = seqData.vui_flags.colour_description_present_flag; + m_latest_sps.vui.chroma_loc_info_present_flag = seqData.vui_flags.chroma_loc_info_present_flag; + m_latest_sps.vui.overscan_info_present_flag = seqData.vui_flags.overscan_info_present_flag; + m_latest_sps.vui.overscan_appropriate_flag = seqData.vui_flags.overscan_appropriate_flag; + m_latest_sps.vui.neutral_chroma_indication_flag = seqData.vui_flags.neutral_chroma_indication_flag; + m_latest_sps.vui.field_seq_flag = seqData.vui_flags.field_seq_flag; + m_latest_sps.vui.frame_field_info_present_flag = seqData.vui_flags.frame_field_info_present_flag; + m_latest_sps.vui.default_display_window_flag = seqData.vui_flags.default_display_window_flag; + m_latest_sps.vui.poc_proportional_to_timing_flag = seqData.vui_flags.poc_proportional_to_timing_flag; + m_latest_sps.vui.hrd_parameters_present_flag = seqData.vui_flags.hrd_parameters_present_flag; + m_latest_sps.vui.bitstream_restriction_flag = seqData.vui_flags.bitstream_restriction_flag; + m_latest_sps.vui.tiles_fixed_structure_flag = seqData.vui_flags.tiles_fixed_structure_flag; + m_latest_sps.vui.motion_vectors_over_pic_boundaries_flag = seqData.vui_flags.motion_vectors_over_pic_boundaries_flag; + m_latest_sps.vui.restricted_ref_pic_lists_flag = seqData.vui_flags.restricted_ref_pic_lists_flag; + m_latest_sps.sps_extension_flag = 0; // Print built SPS structure @@ -633,10 +674,48 @@ d3d12_video_bitstream_builder_hevc::print_sps(const HevcSeqParameterSet& SPS) debug_printf("sps_temporal_mvp_enabled_flag: %d\n", SPS.sps_temporal_mvp_enabled_flag); debug_printf("strong_intra_smoothing_enabled_flag: %d\n", SPS.strong_intra_smoothing_enabled_flag); debug_printf("vui_parameters_present_flag: %d\n", SPS.vui_parameters_present_flag); + debug_printf("aspect_ratio_info_present_flag: %d\n", SPS.vui.aspect_ratio_info_present_flag); + debug_printf("aspect_ratio_idc: %d\n", SPS.vui.aspect_ratio_idc); + debug_printf("sar_width: %d\n", SPS.vui.sar_width); + debug_printf("sar_height: %d\n", SPS.vui.sar_height); + debug_printf("overscan_info_present_flag: %d\n", SPS.vui.overscan_info_present_flag); + debug_printf("overscan_appropriate_flag: %d\n", SPS.vui.overscan_appropriate_flag); + debug_printf("video_signal_type_present_flag: %d\n", SPS.vui.video_signal_type_present_flag); + debug_printf("video_format: %d\n", SPS.vui.video_format); + debug_printf("video_full_range_flag: %d\n", SPS.vui.video_full_range_flag); + debug_printf("colour_description_present_flag: %d\n", SPS.vui.colour_description_present_flag); + debug_printf("colour_primaries: %d\n", SPS.vui.colour_primaries); + debug_printf("transfer_characteristics: %d\n", SPS.vui.transfer_characteristics); + debug_printf("matrix_coeffs: %d\n", SPS.vui.matrix_coeffs); + debug_printf("chroma_loc_info_present_flag: %d\n", SPS.vui.chroma_loc_info_present_flag); + debug_printf("chroma_sample_loc_type_top_field: %d\n", SPS.vui.chroma_sample_loc_type_top_field); + debug_printf("chroma_sample_loc_type_bottom_field: %d\n", SPS.vui.chroma_sample_loc_type_bottom_field); + debug_printf("neutral_chroma_indication_flag: %d\n", SPS.vui.neutral_chroma_indication_flag); + debug_printf("field_seq_flag: %d\n", SPS.vui.field_seq_flag); + debug_printf("frame_field_info_present_flag: %d\n", SPS.vui.frame_field_info_present_flag); + debug_printf("default_display_window_flag: %d\n", SPS.vui.default_display_window_flag); + debug_printf("def_disp_win_left_offset: %d\n", SPS.vui.def_disp_win_left_offset); + debug_printf("def_disp_win_right_offset: %d\n", SPS.vui.def_disp_win_right_offset); + debug_printf("def_disp_win_top_offset: %d\n", SPS.vui.def_disp_win_top_offset); + debug_printf("def_disp_win_bottom_offset: %d\n", SPS.vui.def_disp_win_bottom_offset); + debug_printf("timing_info_present_flag: %d\n", SPS.vui.timing_info_present_flag); + debug_printf("num_units_in_tick: %d\n", SPS.vui.num_units_in_tick); + debug_printf("time_scale: %d\n", SPS.vui.time_scale); + debug_printf("poc_proportional_to_timing_flag: %d\n", SPS.vui.poc_proportional_to_timing_flag); + debug_printf("num_ticks_poc_diff_one_minus1: %d\n", SPS.vui.num_ticks_poc_diff_one_minus1); + debug_printf("hrd_parameters_present_flag: %d\n", SPS.vui.hrd_parameters_present_flag); + debug_printf("bitstream_restriction_flag: %d\n", SPS.vui.bitstream_restriction_flag); + debug_printf("tiles_fixed_structure_flag: %d\n", SPS.vui.tiles_fixed_structure_flag); + debug_printf("motion_vectors_over_pic_boundaries_flag: %d\n", SPS.vui.motion_vectors_over_pic_boundaries_flag); + debug_printf("restricted_ref_pic_lists_flag: %d\n", SPS.vui.restricted_ref_pic_lists_flag); + debug_printf("min_spatial_segmentation_idc: %d\n", SPS.vui.min_spatial_segmentation_idc); + debug_printf("max_bytes_per_pic_denom: %d\n", SPS.vui.max_bytes_per_pic_denom); + debug_printf("max_bits_per_min_cu_denom: %d\n", SPS.vui.max_bits_per_min_cu_denom); + debug_printf("log2_max_mv_length_horizontal: %d\n", SPS.vui.log2_max_mv_length_horizontal); + debug_printf("log2_max_mv_length_vertical: %d\n", SPS.vui.log2_max_mv_length_vertical); debug_printf("sps_extension_flag: %d\n", SPS.sps_extension_flag); debug_printf("sps_extension_data_flag: %d\n", SPS.sps_extension_data_flag); - debug_printf("HevcSeqParameterSet values end\n--------------------------------------\n"); } void diff --git a/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_hevc.h b/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_hevc.h index e2487c132a4..06038d99445 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_hevc.h +++ b/src/gallium/drivers/d3d12/d3d12_video_encoder_bitstream_builder_hevc.h @@ -46,6 +46,7 @@ class d3d12_video_bitstream_builder_hevc : public d3d12_video_bitstream_builder_ HevcVideoParameterSet* pVPSStruct = nullptr); void build_sps(const HevcVideoParameterSet& parentVPS, + const struct pipe_h265_enc_seq_param & seqData, uint8_t seq_parameter_set_id, const D3D12_VIDEO_ENCODER_PICTURE_RESOLUTION_DESC& encodeResolution, const D3D12_BOX& crop_window_upper_layer, diff --git a/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_hevc.cpp b/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_hevc.cpp index f50b7600f5c..a14c71b1565 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_hevc.cpp +++ b/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_hevc.cpp @@ -282,7 +282,76 @@ d3d12_video_nalu_writer_hevc::write_sps_bytes(d3d12_video_encoder_bitstream *pBi pBitstream->put_bits(1, pSPS->sps_temporal_mvp_enabled_flag); pBitstream->put_bits(1, pSPS->strong_intra_smoothing_enabled_flag); pBitstream->put_bits(1, pSPS->vui_parameters_present_flag); - assert (pSPS->vui_parameters_present_flag == 0); + + pBitstream->put_bits(1, pSPS->vui.aspect_ratio_info_present_flag); + if (pSPS->vui.aspect_ratio_info_present_flag) { + pBitstream->put_bits(8, pSPS->vui.aspect_ratio_idc); + if (pSPS->vui.aspect_ratio_idc == 255) { + pBitstream->put_bits(16, pSPS->vui.sar_width); + pBitstream->put_bits(16, pSPS->vui.sar_height); + } + } + + pBitstream->put_bits(1, pSPS->vui.overscan_info_present_flag); + if (pSPS->vui.overscan_info_present_flag) { + pBitstream->put_bits(1, pSPS->vui.overscan_appropriate_flag); + } + + pBitstream->put_bits(1, pSPS->vui.video_signal_type_present_flag); + if (pSPS->vui.video_signal_type_present_flag) { + pBitstream->put_bits(3, pSPS->vui.video_format); + pBitstream->put_bits(1, pSPS->vui.video_full_range_flag); + pBitstream->put_bits(1, pSPS->vui.colour_description_present_flag); + if (pSPS->vui.colour_description_present_flag) { + pBitstream->put_bits(8, pSPS->vui.colour_primaries); + pBitstream->put_bits(8, pSPS->vui.transfer_characteristics); + pBitstream->put_bits(8, pSPS->vui.matrix_coeffs); + } + } + + pBitstream->put_bits(1, pSPS->vui.chroma_loc_info_present_flag); + if (pSPS->vui.chroma_loc_info_present_flag) { + pBitstream->exp_Golomb_ue(pSPS->vui.chroma_sample_loc_type_top_field); + pBitstream->exp_Golomb_ue(pSPS->vui.chroma_sample_loc_type_bottom_field); + } + + pBitstream->put_bits(1, pSPS->vui.neutral_chroma_indication_flag); + pBitstream->put_bits(1, pSPS->vui.field_seq_flag); + pBitstream->put_bits(1, pSPS->vui.frame_field_info_present_flag); + pBitstream->put_bits(1, pSPS->vui.default_display_window_flag); + if (pSPS->vui.default_display_window_flag) { + pBitstream->exp_Golomb_ue(pSPS->vui.def_disp_win_left_offset); + pBitstream->exp_Golomb_ue(pSPS->vui.def_disp_win_right_offset); + pBitstream->exp_Golomb_ue(pSPS->vui.def_disp_win_top_offset); + pBitstream->exp_Golomb_ue(pSPS->vui.def_disp_win_bottom_offset); + } + + pBitstream->put_bits(1, pSPS->vui.timing_info_present_flag); + if (pSPS->vui.timing_info_present_flag) { + pBitstream->put_bits(16, pSPS->vui.num_units_in_tick >> 16); + pBitstream->put_bits(16, pSPS->vui.num_units_in_tick & 0xffff); + pBitstream->put_bits(16, pSPS->vui.time_scale >> 16); + pBitstream->put_bits(16, pSPS->vui.time_scale & 0xffff); + pBitstream->put_bits(1, pSPS->vui.poc_proportional_to_timing_flag); + if (pSPS->vui.poc_proportional_to_timing_flag) { + pBitstream->exp_Golomb_ue(pSPS->vui.num_ticks_poc_diff_one_minus1); + } + + assert(pSPS->vui.hrd_parameters_present_flag == 0); + pBitstream->put_bits(1, 0); // hrd_parameters_present_flag = 0 until implementing HRD params + } + + pBitstream->put_bits(1, pSPS->vui.bitstream_restriction_flag); + if (pSPS->vui.bitstream_restriction_flag) { + pBitstream->put_bits(1, pSPS->vui.tiles_fixed_structure_flag); + pBitstream->put_bits(1, pSPS->vui.motion_vectors_over_pic_boundaries_flag); + pBitstream->put_bits(1, pSPS->vui.restricted_ref_pic_lists_flag); + pBitstream->exp_Golomb_ue(pSPS->vui.min_spatial_segmentation_idc); + pBitstream->exp_Golomb_ue(pSPS->vui.max_bytes_per_pic_denom); + pBitstream->exp_Golomb_ue(pSPS->vui.max_bits_per_min_cu_denom); + pBitstream->exp_Golomb_ue(pSPS->vui.log2_max_mv_length_horizontal); + pBitstream->exp_Golomb_ue(pSPS->vui.log2_max_mv_length_vertical); + } // pSps_extension_flag pBitstream->put_bits(1, 0); diff --git a/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_hevc.h b/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_hevc.h index ad52f96b079..6359143cc0c 100644 --- a/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_hevc.h +++ b/src/gallium/drivers/d3d12/d3d12_video_encoder_nalu_writer_hevc.h @@ -146,6 +146,48 @@ struct HEVCReferencePictureSet { }; }; +struct HEVCVideoUsabilityInfo { + uint8_t aspect_ratio_info_present_flag; + uint8_t aspect_ratio_idc; + int32_t sar_width; + int32_t sar_height; + uint8_t overscan_info_present_flag; + uint8_t overscan_appropriate_flag; + uint8_t video_signal_type_present_flag; + uint8_t video_format; + uint8_t video_full_range_flag; + uint8_t colour_description_present_flag; + uint8_t colour_primaries; + uint8_t transfer_characteristics; + uint8_t matrix_coeffs; + uint8_t chroma_loc_info_present_flag; + int32_t chroma_sample_loc_type_top_field; + int32_t chroma_sample_loc_type_bottom_field; + uint8_t neutral_chroma_indication_flag; + uint8_t field_seq_flag; + uint8_t frame_field_info_present_flag; + uint8_t default_display_window_flag; + int32_t def_disp_win_left_offset; + int32_t def_disp_win_right_offset; + int32_t def_disp_win_top_offset; + int32_t def_disp_win_bottom_offset; + uint8_t timing_info_present_flag; + uint32_t num_units_in_tick; + uint32_t time_scale; + uint8_t poc_proportional_to_timing_flag; + uint32_t num_ticks_poc_diff_one_minus1; + uint8_t hrd_parameters_present_flag; + uint8_t bitstream_restriction_flag; + uint8_t tiles_fixed_structure_flag; + uint8_t motion_vectors_over_pic_boundaries_flag; + uint8_t restricted_ref_pic_lists_flag; + uint32_t min_spatial_segmentation_idc; + uint32_t max_bytes_per_pic_denom; + uint32_t max_bits_per_min_cu_denom; + uint32_t log2_max_mv_length_horizontal; + uint32_t log2_max_mv_length_vertical; +}; + struct HevcSeqParameterSet { HEVCNaluHeader nalu; uint8_t sps_video_parameter_set_id; @@ -200,6 +242,7 @@ struct HevcSeqParameterSet { uint8_t sps_temporal_mvp_enabled_flag; uint8_t strong_intra_smoothing_enabled_flag; uint8_t vui_parameters_present_flag; + HEVCVideoUsabilityInfo vui; uint8_t sps_extension_flag; uint8_t sps_extension_data_flag; };