--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/webaudio/webrtc-integration.html Sat Oct 15 19:00:15 2011 -0700
@@ -0,0 +1,303 @@
+<html>
+
+<head>
+<link rel="stylesheet" href="style.css">
+</head>
+
+
+<body>
+
+<h1>MediaStream Integration</h1>
+
+<p>
+The following examples illustrate
+ <a href="http://dev.w3.org/2011/webrtc/editor/webrtc-20111004.html">WebRTC</a> integration with the
+ <a href="https://dvcs.w3.org/hg/audio/raw-file/tip/webaudio/specification.html">Web Audio API</a>. They are borrowed and modified from Robert O'Callahan's
+ <a href="http://hg.mozilla.org/users/rocallahan_mozilla.com/specs/raw-file/tip/StreamProcessing/StreamProcessing.html#examples">MediaStream Processing API</a>
+ proposal.
+</p>
+
+
+<ol>
+<li>Play video with processing effect applied to the audio track
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<video src="foo.webm" id="v" controls></video>
+<script>
+ var audioSource = context.createMediaElementSource(document.getElementById("v"));
+ var filter = context.createBiquadFilter();
+ audioSource.connect(filter);
+ filter.connect(context.destination);
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+
+<li>Play video with processing effects mixing in out-of-band audio tracks (in sync)
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<video src="foo.webm" id="v"></video>
+<audio src="back.webm" id="back"></audio>
+<script>
+ var videoSource = context.createMediaElementSource(document.getElementById("v"));
+ var audioSource = context.createMediaElementSource(document.getElementById("back"));
+ var effect = context.createJavaScriptNode();
+ effect.onaudioprocess = customAudioProcessor;
+ videoSource.connect(effect);
+ effect.connect(context.destination);
+ audioSource.connect(context.destination);
+
+ function startPlaying() {
+ document.getElementById("v").play();
+ document.getElementById("back").play();
+ }
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+<li>Capture microphone input and stream it out to a peer with a processing effect applied to the audio
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<script>
+ navigator.getUserMedia('audio', gotAudio);
+ function gotAudio(stream) {
+ var microphone = context.createMediaStreamSource(stream);
+ var filter = context.createBiquadFilter();
+ var peer = context.createMediaStreamDestination();
+ microphone.connect(filter);
+ filter.connect(peer);
+ peerConnection.addStream(peer.stream);
+ }
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+<li>Capture microphone input and visualize it as it is being streamed out to a peer and recorded
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<canvas id="c"></canvas>
+<script>
+ navigator.getUserMedia('audio', gotAudio);
+ var streamRecorder;
+ function gotAudio(stream) {
+ var microphone = context.createMediaStreamSource(stream);
+ var analyser = context.createAnalyser();
+ microphone.connect(analyser);
+ analyser.connect(context.destination);
+ requestAnimationFrame(drawAnimation);
+
+ streamRecorder = stream.record();
+ peerConnection.addStream(stream);
+ }
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+<li>Capture microphone input, visualize it, mix in another audio track and stream the result to a peer and record
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<canvas id="c"></canvas>
+<audio src="back.webm" id="back"></audio>
+<script>
+ navigator.getUserMedia('audio', gotAudio);
+ var streamRecorder;
+ function gotAudio(stream) {
+ var microphone = context.createMediaStreamSource(stream);
+ var backgroundMusic = context.createMediaElementSource(document.getElementById("back"));
+ var analyser = context.createAnalyser();
+ var mixedOutput = context.createMediaStreamDestination();
+ microphone.connect(analyser);
+ analyser.connect(mixedOutput);
+ backgroundMusic.connect(mixedOutput);
+ requestAnimationFrame(drawAnimation);
+
+ streamRecorder = mixedOutput.stream.record();
+ peerConnection.addStream(mixedOutput.stream);
+ }
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+<li>Receive audio streams from peers, mix them with spatialization effects, and play
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<audio id="out" autoplay></audio>
+<script>
+ peerConnection.onaddstream = function(event) {
+ var peerInput = context.createMediaStreamSource(event.stream);
+ var panner = context.createPanner();
+ panner.setPosition(x, y, z);
+ peerInput.connect(panner);
+ panner.connect(context.destination);
+ };
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+<li>Seamlessly chain from the end of one input stream to another
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<audio src="in1.webm" id="in1" preload></audio>
+<audio src="in2.webm" id="in2"></audio>
+<script>
+ var in1 = document.getElementById("in1");
+ var in2 = document.getElementById("in2");
+ in1.onloadeddata = function() {
+ in1.onended = function() { in2.play(); };
+ in1.play();
+ }
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+<li>Seamlessly switch from one input stream to another, e.g. to implement adaptive streaming
+
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<audio src="in1.webm" id="in1" preload></audio>
+<audio src="in2.webm" id="in2"></audio>
+<script>
+
+var in1 = document.getElementById("in1");
+var in2 = document.getElementById("in2");
+var source1 = context.createMediaElementSource(in1);
+var source2 = context.createMediaElementSource(in2);
+source1.connect(context.destination);
+source2.connect(context.destination);
+
+source1.play();
+
+function switchStreams() {
+ in2.currentTime = in1.currentTime + 10; // arbitrary, but we should be able to complete the seek within this time
+ var switchTime = context.currentTime + 10;
+ in1.pause(switchTime);
+ in2.play(switchTime);
+}
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+<li>Synthesize samples from JS data
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<script>
+ var processor = context.createJavaScriptNode(2048, 2, 2);
+ processor.onaudioprocess = customAudioProcess;
+ processor.connect(context.destination);
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+<li>Trigger a sound sample to be played through the effects graph ASAP but without causing any blocking
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<script>
+ var source = context.createBufferSource();
+ source.buffer = kickDrum909;
+ source.connect(context.destination);
+ source.noteOn(0);
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+<li>Trigger a sound sample to be played through the effects graph in five seconds
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<script>
+ var source = context.createBufferSource();
+ source.buffer = kickDrum909;
+ var effect = context.createConvolver();
+ effect.buffer = immenseCathedral;
+ source.connect(effect);
+ effect.connect(context.destination);
+ source.noteOn(context.currentTime + 5);
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+<br><br><br><br>
+<h2>
+Please note that examples 12, 13, and 14 involve no audio processing and thus should work with the
+ <a href="http://dev.w3.org/2011/webrtc/editor/webrtc-20111004.html">WebRTC API</a> as it is currently designed.
+</h2>
+
+
+
+<li>Capture video from a camera and analyze it (e.g. face recognition)
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<script>
+ navigator.getUserMedia('video', gotVideo);
+ function gotVideo(stream) {
+ stream.createWorkerProcessor(new Worker("face-recognizer.js"));
+ }
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+<li>Capture video, record it to a file and upload the file (e.g. Youtube)
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<script>
+ navigator.getUserMedia('video', gotVideo);
+ var streamRecorder;
+ function gotVideo(stream) {
+ streamRecorder = stream.record();
+ }
+ function stopRecording() {
+ streamRecorder.getRecordedData(gotData);
+ }
+ function gotData(blob) {
+ var x = new XMLHttpRequest();
+ x.open('POST', 'uploadMessage');
+ x.send(blob);
+ }
+</script>
+</code></pre></div></div>
+</div>
+</div>
+
+<li>Capture video from a canvas, record it to a file then upload
+
+<div class="block"><div class="blockTitleDiv"><span class="blockTitle">ECMAScript</span></div><div class="blockContent"><pre class="code"><code class="es-code">
+<canvas width="640" height="480" id="c"></canvas>
+<script>
+ var canvas = document.getElementById("c");
+ var streamRecorder = canvas.stream.record();
+ function stopRecording() {
+ streamRecorder.getRecordedData(gotData);
+ }
+ function gotData(blob) {
+ var x = new XMLHttpRequest();
+ x.open('POST', 'uploadMessage');
+ x.send(blob);
+ }
+ var frame = 0;
+ function updateCanvas() {
+ var ctx = canvas.getContext("2d");
+ ctx.clearRect(0, 0, 640, 480);
+ ctx.fillText("Frame " + frame, 0, 200);
+ ++frame;
+ }
+ setInterval(updateCanvas, 30);
+</script>
+</code></pre></div></div>
+</div>
+</div>
+</ol>
+
+
+</body>
+</html>