技术背景
我们在做Windows平台流数据转发的时候，除了常规的RTSP转RTMP推送外，还有个场景就是，好多开发者希望拉取的RTSP流，做二次视频分析，或者加动态水印等，处理后的数据，再二次编码推送到RTMP服务或轻量级RTSP服务。
技术实现
本文就以Windows平台拉取RTSP流，回调yuv数据到上层，处理后的数据，二次投递到RTMP服务和轻量级RTSP服务，然后叠加动态水印，并实现处理后的数据实时录像功能，废话不多说，先上图：
上图拉取了RTSP流，然后左侧窗体显示，添加动态水印后，再在右侧预览，并把数据重新投递到推送端，考虑到编码性能，我们可选硬编码。
先说RTSP拉流，其他接口不表，这里主要是设置下video frame callback：
video_frame_call_back_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, IntPtr.Zero, video_frame_call_back_);回调上来的video数据，投递到推送端，当然如果需要二次处理的话，处理后再丢给推送端：
public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame)
{
if (frame == IntPtr.Zero)
return;
NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));
if (publisher_wrapper_ != null) {
publisher_wrapper_.post_i420_layer_image(publisher_wrapper_.get_external_video_layer_index(), video_frame.plane0_, video_frame.stride0_, video_frame.plane1_, video_frame.stride1_,
video_frame.plane2_, video_frame.stride2_,
video_frame.width_, video_frame.height_);
}
}audio的也是：
audio_pcm_frame_call_back_ = new SP_SDKAudioPCMFrameCallBack(SetAudioPCMFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetAudioPCMFrameCallBack(player_handle_, IntPtr.Zero, audio_pcm_frame_call_back_);回调上来的audio数据：
public void SetAudioPCMFrameCallBack(IntPtr handle, IntPtr user_data,
UInt32 status, IntPtr data, UInt32 size,
Int32 sample_rate, Int32 channel, Int32 per_channel_sample_number)
{
if (data == IntPtr.Zero || size == 0)
return;
if (publisher_wrapper_ != null)
publisher_wrapper_.post_audio_pcm_data(data, size, 0, sample_rate, channel, per_channel_sample_number);
}由于处理后的数据，需要重新编码，我们播放端会把原始视频宽高回调上来：
video_size_call_back_ = new SP_SDKVideoSizeCallBack(SP_SDKVideoSizeHandle);
NTSmartPlayerSDK.NT_SP_SetVideoSizeCallBack(player_handle_, IntPtr.Zero, video_size_call_back_);拿到视频宽高后，我们可以把宽高投递到推送端，便于推送端推送相同分辨率出去，当然，二次编码，也可以设置其他期望的分辨率：
private void PlaybackWindowResized(Int32 width, Int32 height)
{
width_ = width;
height_ = height;
if (publisher_wrapper_ != null)
publisher_wrapper_.SetResolution(width, height);
}设置文字水印字体、字号：
private void btn_set_font_Click(object sender, EventArgs e)
{
FontDialog font_dlg = new FontDialog();
DialogResult result = font_dlg.ShowDialog();
if (result == DialogResult.OK)
{
// 获取用户所选字体
Font selectedFont = font_dlg.Font;
btn_set_font.Text = "" + selectedFont.Name + ", " + selectedFont.Size + "pt";
selected_osd_font_ = new Font(selectedFont.Name, selectedFont.Size, FontStyle.Regular, GraphicsUnit.Point);
}
}如果需要添加文字水印：
private async void btn_text_osd_Click(object sender, EventArgs e) {
Bitmap bitmap = null;
try
{
string format = "yyyy-MM-dd HH:mm:ss.fff";
StringBuilder sb = new StringBuilder();
sb.Append("施工单位：上海视沃信息科技有限公司(daniusdk.com)");
sb.Append("rn");
sb.Append("施工时间：");
sb.Append(DateTime.Now.DayOfWeek.ToString());
sb.Append(" ");
sb.Append(DateTime.Now.ToString(format));
sb.Append("rn");
sb.Append("当前位置：上海市");
string str = sb.ToString();
bitmap = GenerateBitmap(str);
}
catch (Exception )
{
return;
}
if (null == bitmap)
return;
int x = 0;
int y = 200;
UpdateLayerRegion(publisher_wrapper_.get_text_layer_index(), x, y, bitmap);
publisher_wrapper_.enable_layer(publisher_wrapper_.get_text_layer_index(), true);
await Task.Delay(30);
publisher_wrapper_.post_argb8888_layer_image(publisher_wrapper_.get_text_layer_index(), bitmap);
}开始推送RTMP：
private void btn_publish_rtmp_Click(object sender, EventArgs e)
{
if (!OpenPublisherHandle())
return;
SetCommonOptionToPublisherSDK();
String url = "rtmp://192.168.0.108:1935/hls/stream1";
//String url = "rtmp://192.168.2.154:1935/live/stream1";
if (url.Length publisher_wrapper接口封装：
public bool StartPublisher(String url)
{
if (is_empty_handle() || is_rtmp_publishing())
return false;
if (!String.IsNullOrEmpty(url))
NTSmartPublisherSDK.NT_PB_SetURL(handle_, url, IntPtr.Zero);
if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPublisher(handle_, IntPtr.Zero))
{
try_close_handle();
return false;
}
shared_lock_.EnterWriteLock();
try
{
handle_reference_count_++;
is_rtmp_publishing_ = true;
}
finally
{
shared_lock_.ExitWriteLock();
}
return true;
}
public void StopPublisher()
{
if (is_empty_handle() || !is_rtmp_publishing())
return;
shared_lock_.EnterWriteLock();
try
{
is_rtmp_publishing_ = false;
handle_reference_count_--;
}
finally
{
shared_lock_.ExitWriteLock();
}
NTSmartPublisherSDK.NT_PB_StopPublisher(handle_);
try_close_handle();
}对应的SetCommonOptionToPublisherSDK()实现：
private void SetCommonOptionToPublisherSDK()
{
if (publisher_wrapper_.is_empty_handle())
return;
if (publisher_wrapper_.handle_reference_count() > 0)
return;
publisher_wrapper_.config_layers(true);
publisher_wrapper_.SetFrameRate((uint)video_fps_);
int cur_video_codec_id = (int)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264;
bool is_h264_encoder = true;
bool is_hw_encoder = false;
if (btn_check_video_hardware_encoder_.Checked)
{
is_hw_encoder = true;
}
Int32 cur_sel_encoder_id = 0;
Int32 cur_sel_gpu = 0;
if (is_hw_encoder)
{
int cur_sel_hw = combobox_video_encoders_.SelectedIndex;
if (cur_sel_hw >= 0)
{
cur_sel_encoder_id = Convert.ToInt32(combobox_video_encoders_.SelectedValue);
cur_sel_gpu = -1;
int cur_sel_hw_dev = combobox_video_hardware_encoder_devices_.SelectedIndex;
if (cur_sel_hw_dev >= 0)
{
cur_sel_gpu = Convert.ToInt32(combobox_video_hardware_encoder_devices_.SelectedValue);
}
}
else
{
is_hw_encoder = false;
}
}
if (!is_hw_encoder)
{
if ((int)NTCommonMediaDefine.NT_MEDIA_CODEC_ID.NT_MEDIA_CODEC_ID_H264 == cur_video_codec_id)
{
cur_sel_encoder_id = btn_check_openh264_encoder_.Checked ? 1 : 0;
}
}
publisher_wrapper_.SetVideoEncoder((int)(is_hw_encoder ? 1 : 0), (int)cur_sel_encoder_id, (uint)cur_video_codec_id, (int)cur_sel_gpu);
publisher_wrapper_.SetVideoQualityV2(publisher_wrapper_.CalVideoQuality(width_, height_, is_h264_encoder));
publisher_wrapper_.SetVideoBitRate(publisher_wrapper_.CalBitRate(video_fps_, width_, height_));
publisher_wrapper_.SetVideoMaxBitRate(publisher_wrapper_.CalMaxKBitRate(video_fps_, width_, height_, false));
publisher_wrapper_.SetVideoKeyFrameInterval(key_frame_interval_);
if (is_h264_encoder)
{
publisher_wrapper_.SetVideoEncoderProfile(3);
}
publisher_wrapper_.SetVideoEncoderSpeed(publisher_wrapper_.CalVideoEncoderSpeed(width_, height_, is_h264_encoder));
publisher_wrapper_.SetPublisherAudioCodecType(1); //1: AAC 2: Speex
}图层配置实现：
public bool config_layers(bool is_add_rgbx_zero_layer)
{
if (video_option_ != (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)
return false;
if (is_empty_handle())
return false;
int w = video_width_;
int h = video_height_;
if ((w & 0x1) != 0)
--w;
if ((h & 0x1) != 0)
--h;
if (w 0;
}推送端二次录像：
private void btn_start_recorder_Click(object sender, EventArgs e)
{
if (!OpenPublisherHandle())
return;
SetCommonOptionToPublisherSDK();
if (!publisher_wrapper_.StartRecorder())
{
MessageBox.Show("调用StartRecorder失败..");
return;
}
btn_start_recorder.Enabled = false;
btn_stop_recorder.Enabled = true;
}
private void btn_pause_recorder_Click(object sender, EventArgs e)
{
String btn_pause_rec_text = btn_pause_recorder.Text;
if ("暂停录像" == btn_pause_rec_text)
{
UInt32 ret = publisher_wrapper_.PauseRecorder(true);
if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
{
MessageBox.Show("暂停录像失败, 请重新尝试!");
return;
}
else if (NTBaseCodeDefine.NT_ERC_OK == ret)
{
btn_pause_recorder.Text = "恢复录像";
}
}
else
{
UInt32 ret = publisher_wrapper_.PauseRecorder(false);
if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
{
MessageBox.Show("恢复录像失败, 请重新尝试!");
return;
}
else if (NTBaseCodeDefine.NT_ERC_OK == ret)
{
btn_pause_recorder.Text = "暂停录像";
}
}
}
private void btn_stop_recorder_Click(object sender, EventArgs e)
{
if (publisher_wrapper_.is_recording()) {
publisher_wrapper_.StopRecorder();
btn_start_recorder.Enabled = true;
btn_stop_recorder.Enabled = false;
}
}启动轻量级RTSP服务：
private void btn_rtsp_service_Click(object sender, EventArgs e)
{
if(publisher_wrapper_.IsRTSPSerivceRunning())
{
publisher_wrapper_.StopRtspService();
btn_rtsp_service.Text = "启动RTSP服务";
btn_rtsp_stream.Enabled = false;
}
else
{
if(publisher_wrapper_.StartRtspService())
{
btn_rtsp_service.Text = "停止RTSP服务";
btn_rtsp_stream.Enabled = true;
}
}
}发布RTSP流：
private void btn_rtsp_stream_Click(object sender, EventArgs e)
{
if (publisher_wrapper_.is_rtsp_publishing())
{
publisher_wrapper_.StopRtspStream();
btn_rtsp_stream.Text = "发布RTSP流";
btn_get_rtsp_session_numbers.Enabled = false;
btn_rtsp_service.Enabled = true;
}
else
{
if (!OpenPublisherHandle())
return;
SetCommonOptionToPublisherSDK();
if (!publisher_wrapper_.StartRtspStream())
{
MessageBox.Show("调用StartRtspStream失败..");
return;
}
btn_rtsp_stream.Text = "停止RTSP流";
btn_get_rtsp_session_numbers.Enabled = true;
btn_rtsp_service.Enabled = false;
}
}获取RTSP session会话数：
private void btn_get_rtsp_session_numbers_Click(object sender, EventArgs e)
{
if (publisher_wrapper_.is_rtsp_publishing())
{
int session_numbers = publisher_wrapper_.GetRtspSessionNumbers();
MessageBox.Show(session_numbers.ToString(), "当前RTSP连接会话数");
}
}叠加后的数据，本地预览：
private void btn_preview_Click(object sender, EventArgs e)
{
if (publisher_wrapper_.is_previewing())
{
publisher_wrapper_.StopPreview();
btn_preview.Text = "开始预览";
}
else
{
if (!OpenPublisherHandle())
return;
SetCommonOptionToPublisherSDK();
if (!publisher_wrapper_.StartPreview())
{
MessageBox.Show("调用StartPreview失败..");
return;
}
btn_preview.Text = "停止预览";
}
}对应的预览封装实现：
public bool StartPreview()
{
if (is_empty_handle() || is_previewing())
return false;
video_preview_image_callback_ = new NT_PB_SDKVideoPreviewImageCallBack(SDKVideoPreviewImageCallBack);
NTSmartPublisherSDK.NT_PB_SetVideoPreviewImageCallBack(handle_, (int)NTSmartPublisherDefine.NT_PB_E_IMAGE_FORMAT.NT_PB_E_IMAGE_FORMAT_RGB32, IntPtr.Zero, video_preview_image_callback_);
if (NTBaseCodeDefine.NT_ERC_OK != NTSmartPublisherSDK.NT_PB_StartPreview(handle_, 0x800000, IntPtr.Zero))
{
try_close_handle();
return false;
}
shared_lock_.EnterWriteLock();
try
{
handle_reference_count_++;
is_previewing_ = true;
}
finally
{
shared_lock_.ExitWriteLock();
}
return true;
}
public void StopPreview()
{
if (is_empty_handle() || !is_previewing())
return;
shared_lock_.EnterWriteLock();
try
{
服务器托管网 is_previewing_ = false;
handle_reference_count_--;
}
finally
{
shared_lock_.ExitWriteLock();
}
NTSmartPublisherSDK.NT_PB_StopPreview(handle_);
try_close_handle();
if (render_wnd_ != null)
render_wnd_.Invalidate();
}预览数据回调：
//预览数据回调
public void SDKVideoPreviewImageCallBack(IntPtr handle, IntPtr user_data, IntPtr image)
{
NT_PB_Image pb_image = (NT_PB_Image)Marshal.PtrToStructure(image, typeof(NT_PB_Image));
NT_VideoFrame pVideoFrame = new NT_VideoFrame();
pVideoFrame.width_ = pb_image.width_;
pVideoFrame.height_ = pb_image.height_;
pVideoFrame.stride_ = pb_image.stride_[0];
Int32 argb_size = pb_image.stride_[0] * pb_image.height_;
pVideoFrame.plane_ = Marshal.AllocHGlobal(argb_size);
CopyMemory(pVideoFrame.plane_, pb_image.plane_[0], (UInt32)argb_size);
if (sync_invoke_ != null)
{
System.ComponentModel.ISynchronizeInvoke sync_invoke_target = sync_invoke_.Target as System.ComponentModel.ISynchronizeInvoke;
if (sync_invoke_target != null)
{
if (sync_invoke_target.InvokeRequired)
{
sync_invoke_target.B服务器托管网eginInvoke(set_video_preview_image_callback_, new object[] { pVideoFrame });
}
else
{
set_video_preview_image_callback_(pVideoFrame);
}
}
}
}总结
以上就是RTSP流二次编辑（如增加动态水印）或视频分析（视觉算法处理）后，再录像、转推至RTMP或轻量级RTSP服务流程，经过二次处理后的流数据，配合我们的SmartPlayer，依然可以整体毫秒级的延迟体验，感兴趣的开发者，可以和我探讨。
服务器托管，北京服务器托管，服务器租用 http://www.fwqtg.net
机房租用，北京机房租用，IDC机房托管， http://www.fwqtg.net
需求列表： 1、是否需要数据抽取？ 2、是否有数据标准化的要求？ 3、是否需要建立数据资产？ 4、是否需要提供数据服务？ 5、是否需要对于数据服务器托管网进行数据建模（二次加工处理）？ 6、是否需要对于数据于模型进行权限管理？ 7、是否需要支持外部数据的导入？…
