第五十四周总结——WebRTC录制音频

云文档开发笔记-录制音频

我首先想到的就是使用WebRTC，如果使用WebRTC我们首先要请求麦克风权限。

window.navigator.mediaDevices.getUserMedia({audio: true
}).then(mediaStream => {beginRecord(mediaStream);
}).catch(err => {console.log(err)
});

这里输出的mediaStream其实就可以直接传递给audio标签的src属性使用。

<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Document</title>
</head>
<body><audio id="local-audio" autoplay controls>播放麦克风捕获的声音</audio><button id="playAudio">打开麦克风</button><script>document.getElementById('playAudio').addEventListener('click',()=>{window.navigator.mediaDevices.getUserMedia({audio: true}).then(mediaStream => {document.getElementById('local-audio').srcObject = mediaStream;}).catch(err => {console.log(err);});})</script>
</body>
</html>

我们看上面的代码，当我们点击打开麦克风的时候，就可以使audio标签一直捕获我们的声音播放，但是问题就是，如果没戴耳机的情况下会有回音，而且，我们要做的肯定也不是直接捕获播放，而是要收集起来，点击播放的时候再播放。

当我们获取得到mediaStream数据后我们也可以采取下面的方法直接播放
audioContext.createMediaStreamSource需要传入一个媒体流，然后对音频执行播放的操作。

let audioContext = new (window.AudioContext || window.webkitAudioContext);
let mediaNode = audioContext.createMediaStreamSource(mediaStream);
mediaNode.connect(audioContext.destination);

保存音频信息的话，我采用了PCM格式，音频的信息就相当于一连串的电信号变化，有许多[-1,1]之间的数字组成的波。如果需要播放就要转成PCM格式。

#mermaid-svg-frzFg34fyuj2DU82 {font-family:"trebuchet ms",verdana,arial,sans-serif;font-size:16px;fill:#333;}#mermaid-svg-frzFg34fyuj2DU82 .error-icon{fill:#552222;}#mermaid-svg-frzFg34fyuj2DU82 .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-frzFg34fyuj2DU82 .edge-thickness-normal{stroke-width:2px;}#mermaid-svg-frzFg34fyuj2DU82 .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-frzFg34fyuj2DU82 .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-frzFg34fyuj2DU82 .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-frzFg34fyuj2DU82 .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-frzFg34fyuj2DU82 .marker{fill:#333333;stroke:#333333;}#mermaid-svg-frzFg34fyuj2DU82 .marker.cross{stroke:#333333;}#mermaid-svg-frzFg34fyuj2DU82 svg{font-family:"trebuchet ms",verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-frzFg34fyuj2DU82 .label{font-family:"trebuchet ms",verdana,arial,sans-serif;color:#333;}#mermaid-svg-frzFg34fyuj2DU82 .cluster-label text{fill:#333;}#mermaid-svg-frzFg34fyuj2DU82 .cluster-label span{color:#333;}#mermaid-svg-frzFg34fyuj2DU82 .label text,#mermaid-svg-frzFg34fyuj2DU82 span{fill:#333;color:#333;}#mermaid-svg-frzFg34fyuj2DU82 .node rect,#mermaid-svg-frzFg34fyuj2DU82 .node circle,#mermaid-svg-frzFg34fyuj2DU82 .node ellipse,#mermaid-svg-frzFg34fyuj2DU82 .node polygon,#mermaid-svg-frzFg34fyuj2DU82 .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-frzFg34fyuj2DU82 .node .label{text-align:center;}#mermaid-svg-frzFg34fyuj2DU82 .node.clickable{cursor:pointer;}#mermaid-svg-frzFg34fyuj2DU82 .arrowheadPath{fill:#333333;}#mermaid-svg-frzFg34fyuj2DU82 .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-frzFg34fyuj2DU82 .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-frzFg34fyuj2DU82 .edgeLabel{background-color:#e8e8e8;text-align:center;}#mermaid-svg-frzFg34fyuj2DU82 .edgeLabel rect{opacity:0.5;background-color:#e8e8e8;fill:#e8e8e8;}#mermaid-svg-frzFg34fyuj2DU82 .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-frzFg34fyuj2DU82 .cluster text{fill:#333;}#mermaid-svg-frzFg34fyuj2DU82 .cluster span{color:#333;}#mermaid-svg-frzFg34fyuj2DU82 div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:"trebuchet ms",verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-frzFg34fyuj2DU82 :root{--mermaid-font-family:"trebuchet ms",verdana,arial,sans-serif;}

gerUserMedia传递mediaStream

AudioContext

使用onaudioprocess来监听音频信息

加头部文件

麦克风

webAudio解析数据

采集数据

采样PCM缓存

转换成WAV格式

我们利用audioContext.createScriptProcessor来创建缓存节点，

//创建AudioContext，将获取麦克风权限的stream传递给AudioContext
//并创建jsNode用来收集信息，将jsNode连接
//开始录音，调用该函数，将recorder函数返回的msg传递进去
beginRecord(mediaStream:MediaStream){let audioContext = new window.AudioContext;let mediaNode = audioContext.createMediaStreamSource(mediaStream);// 创建一个jsNodelet jsNode = this.createJSNode(audioContext);this.jsNodes = jsNode;// 需要连到扬声器消费掉outputBuffer，process回调才能触发// 并且由于不给outputBuffer设置内容，所以扬声器不会播放出声音jsNode.connect(audioContext.destination);jsNode.onaudioprocess = this.onAudioProcess;// 把mediaNode连接到jsNodemediaNode.connect(jsNode);
}//创建jsNode
createJSNode (audioContext:AudioContext) {const BUFFER_SIZE = 4096;const INPUT_CHANNEL_COUNT = 2;const OUTPUT_CHANNEL_COUNT = 2;// createJavaScriptNode已被废弃//@ts-ignorelet creator = audioContext.createScriptProcessor || audioContext.createJavaScriptNode;creator = creator.bind(audioContext);return creator(BUFFER_SIZE,INPUT_CHANNEL_COUNT, OUTPUT_CHANNEL_COUNT);
}

我们主要使用onaudioprocess来监听音频信息。

//收集录音信息，大概0.09s调用一次
onAudioProcess (event:any) {let audioBuffer = event.inputBuffer;//左声道let leftChannelData = audioBuffer.getChannelData(0);//右声道let rightChannelData = audioBuffer.getChannelData(1);leftDataList.push([...leftChannelData]);rightDataList.push([...rightChannelData]);
}

这里在全局定义leftChannelData和rightChannelData两个数组，来缓存音频信息，大概每0.09s调用一下onAudioProcess函数。

因为我们在上面向leftChannelData和rightChannelData中push的是数组，所以我们要先将leftChannelData和rightChannelData扁平化，合并成一个Float32Array数组。

//停止录音
stopRecord () {//合并左右声道let leftData = this.mergeArray(leftDataList),rightData = this.mergeArray(rightDataList);//交叉合并左右声道let allData = this.interleaveLeftAndRight(leftData, rightData);let wavBuffer = this.createWavFile(allData);return this.playRecord(wavBuffer);
}
//合并左声道和右声道
mergeArray (list:any[]) {let length = list.length * list[0].length;let data = new Float32Array(length),offset = 0;for (let i = 0; i < list.length; i++) {data.set(list[i], offset);offset += list[i].length;}return data;
}

我们将还要将左右声道交叉合并。

//交叉合并左右声道
interleaveLeftAndRight (left:Float32Array, right:Float32Array) {let totalLength = left.length + right.length;let data = new Float32Array(totalLength);for (let i = 0; i < left.length; i++) {let k = i * 2;data[k] = left[i];data[k + 1] = right[i];}return data;
}

此时我们就可以创建一个WAV文件了。

我们先写入WAV文件固定的头部。

createWavFile (audioData:Float32Array) {const WAV_HEAD_SIZE = 44;let buffer = new ArrayBuffer(audioData.length * 2 + WAV_HEAD_SIZE),view = new DataView(buffer);this.writeUTFBytes(view, 0, 'RIFF');view.setUint32(4, 44 + audioData.length * 2, true);this.writeUTFBytes(view, 8, 'WAVE');this.writeUTFBytes(view, 12, 'fmt ');view.setUint32(16, 16, true);view.setUint16(20, 1, true);view.setUint16(22, 2, true);view.setUint32(24, 44100, true);view.setUint32(28, 44100 * 2, true);view.setUint16(32, 2 * 2, true);view.setUint16(34, 16, true);this.writeUTFBytes(view, 36, 'data');view.setUint32(40, audioData.length * 2, true);// 写入PCM数据let length = audioData.length;let index = 44;let volume = 1;for (let i = 0; i < length; i++) {view.setInt16(index, audioData[i] * (0x7FFF * volume), true);index += 2;}return buffer;
}writeUTFBytes (view:DataView, offset:number, string:string) {var lng = string.length;for (var i = 0; i < lng; i++) { view.setUint8(offset + i, string.charCodeAt(i));}
}

最后写入刚刚录制的音频数据，我们采用16位二进制来表示声音的强弱，16位表示的范围是[-32768, +32767]，最大值是32767即0x7FFF，录音数据的取值范围是[-1, 1]。

function createWavFile (audioData) {// 写入wav头部，代码同上// 写入PCM数据let length = audioData.length;let index = 44;let volume = 1;for (let i = 0; i < length; i++) {view.setInt16(index, audioData[i] * (0x7FFF * volume), true);index += 2;}return buffer;
}

最后生成本地的blob url返回。

//返回src
playRecord (arrayBuffer:ArrayBuffer) {let blob = new Blob([new Uint8Array(arrayBuffer)]);let blobUrl = URL.createObjectURL(blob);return blobUrl;
}

完整代码

enum status {success = 200,error = 500
}let leftDataList:any[] = [];
let rightDataList:any[] = [];//录音
export class Audio{static instance:any;public mediaStreams:MediaStream | undefined;public jsNodes:ScriptProcessorNode | undefined;constructor(){}//初始化，单例模式static init():Audio{if(!this.instance){this.instance = new Audio();}return this.instance;}//获取麦克风权限recorder () {return new Promise<{code: status,msg: MediaStream}>((resolve,reject)=>{window.navigator.mediaDevices.getUserMedia({audio: true}).then(mediaStream => {this.mediaStreams = mediaStream;resolve({code: status.success,msg: mediaStream});}).catch(err => {reject({code: status.error,msg: err})});})}//创建AudioContext，将获取麦克风权限的stream传递给AudioContext//并创建jsNode用来收集信息，将jsNode连接//开始录音，调用该函数，将recorder函数返回的msg传递进去beginRecord(mediaStream:MediaStream){let audioContext = new window.AudioContext;let mediaNode = audioContext.createMediaStreamSource(mediaStream);// 创建一个jsNodelet jsNode = this.createJSNode(audioContext);this.jsNodes = jsNode;// 需要连到扬声器消费掉outputBuffer，process回调才能触发// 并且由于不给outputBuffer设置内容，所以扬声器不会播放出声音jsNode.connect(audioContext.destination);jsNode.onaudioprocess = this.onAudioProcess;// 把mediaNode连接到jsNodemediaNode.connect(jsNode);}//创建jsNodecreateJSNode (audioContext:AudioContext) {const BUFFER_SIZE = 4096;const INPUT_CHANNEL_COUNT = 2;const OUTPUT_CHANNEL_COUNT = 2;// createJavaScriptNode已被废弃//@ts-ignorelet creator = audioContext.createScriptProcessor || audioContext.createJavaScriptNode;creator = creator.bind(audioContext);return creator(BUFFER_SIZE,INPUT_CHANNEL_COUNT, OUTPUT_CHANNEL_COUNT);}//收集录音信息，大概0.09s调用一次onAudioProcess (event:any) {let audioBuffer = event.inputBuffer;//左声道let leftChannelData = audioBuffer.getChannelData(0);//右声道let rightChannelData = audioBuffer.getChannelData(1);leftDataList.push([...leftChannelData]);rightDataList.push([...rightChannelData]);}//停止录音stopRecord () {//合并左右声道let leftData = this.mergeArray(leftDataList),rightData = this.mergeArray(rightDataList);//交叉合并左右声道let allData = this.interleaveLeftAndRight(leftData, rightData);let wavBuffer = this.createWavFile(allData);return this.playRecord(wavBuffer);}//返回srcplayRecord (arrayBuffer:ArrayBuffer) {let blob = new Blob([new Uint8Array(arrayBuffer)]);let blobUrl = URL.createObjectURL(blob);return blobUrl;}//合并左声道和右声道mergeArray (list:any[]) {let length = list.length * list[0].length;let data = new Float32Array(length),offset = 0;for (let i = 0; i < list.length; i++) {data.set(list[i], offset);offset += list[i].length;}return data;}//交叉合并左右声道interleaveLeftAndRight (left:Float32Array, right:Float32Array) {let totalLength = left.length + right.length;let data = new Float32Array(totalLength);for (let i = 0; i < left.length; i++) {let k = i * 2;data[k] = left[i];data[k + 1] = right[i];}return data;}//将PCM数据转换成wavcreateWavFile (audioData:Float32Array) {const WAV_HEAD_SIZE = 44;let buffer = new ArrayBuffer(audioData.length * 2 + WAV_HEAD_SIZE),view = new DataView(buffer);this.writeUTFBytes(view, 0, 'RIFF');view.setUint32(4, 44 + audioData.length * 2, true);this.writeUTFBytes(view, 8, 'WAVE');this.writeUTFBytes(view, 12, 'fmt ');view.setUint32(16, 16, true);view.setUint16(20, 1, true);view.setUint16(22, 2, true);view.setUint32(24, 44100, true);view.setUint32(28, 44100 * 2, true);view.setUint16(32, 2 * 2, true);view.setUint16(34, 16, true);this.writeUTFBytes(view, 36, 'data');view.setUint32(40, audioData.length * 2, true);// 写入PCM数据let length = audioData.length;let index = 44;let volume = 1;for (let i = 0; i < length; i++) {view.setInt16(index, audioData[i] * (0x7FFF * volume), true);index += 2;}return buffer;}writeUTFBytes (view:DataView, offset:number, string:string) {var lng = string.length;for (var i = 0; i < lng; i++) { view.setUint8(offset + i, string.charCodeAt(i));}}}

在React中调用

import { useEffect, useRef } from 'react';
//Audio就是上面的完整代码
import { Audio } from '../../../utils/audio';export default function ContentAudio() {const audioRef = useRef(null);useEffect(() => {(async function fn(){let audio = Audio.init();//获取麦克风权限let recorder = await audio.recorder();//开始录音audio.beginRecord(recorder.msg);setTimeout(()=>{console.log('停止录音');let url = audio.stopRecord();//@ts-ignoreaudioRef.current.src = url},1000)})()}, [])return (<div><audio ref={audioRef} src="" id="audio" controls autoPlay></audio></div>)
}

参考资料

[1]前端webrtc基础 —— 录音篇
[2]如何实现前端录音功能