Source: https://www.shadertoy.com/js/cmRenderUtils.js?v=8 https://www.shadertoy.com/js/effect.js?v=8 Below are shadertoy's sound-specific code. question: this.mBuffer.getChannelData(0) or 1 are audio channels. How does it know when they are updated? answer: it doesn't! it just fills it all out at the start. Anyway, vibrato will be tricky to handle in a parallel manner... (the period is the area under the frequency curve? the integral of sine is cosine...) ...and the IIR lowpass filter is simply impossible to handle in a parallel way. EffectPass.prototype.MakeHeader_Sound = function( precission, supportDerivatives ) { var header = this.mPrecision; var headerlength = 3; if( this.mSupportsDerivatives ) { header += "#extension GL_OES_standard_derivatives : enable\n"; headerlength++; } header += "uniform float iChannelTime[4];\n" + "uniform float iBlockOffset;\n" + "uniform vec4 iDate;\n" + "uniform float iSampleRate;\n" + "uniform vec3 iChannelResolution[4];\n"; headerlength += 5; for( var i=0; i<this.mInputs.length; i++ ) { var inp = this.mInputs[i]; if( inp!=null && inp.mInfo.mType=="cubemap" ) header += "uniform samplerCube iChannel" + i + ";\n"; else header += "uniform sampler2D iChannel" + i + ";\n"; headerlength++; } this.mHeader = header; this.mHeaderLength = headerlength; } EffectPass.prototype.Create_Sound = function( wa, gl ) { this.MakeHeader(); this.mSoundPassFooter = "void main()" + "{" + "float t = iBlockOffset + (gl_FragCoord.x + gl_FragCoord.y*512.0)/44100.0;" + "vec2 y = mainSound( t );" + "vec2 v = floor((0.5+0.5*y)*65536.0);" + "vec2 vl = mod(v,256.0)/255.0;" + "vec2 vh = floor(v/256.0)/255.0;" + "gl_FragColor = vec4(vl.x,vh.x,vl.y,vh.y);" + "}"; this.mProgram = null; this.mSampleRate = 44100; this.mPlayTime = 60; this.mPlaySamples = this.mPlayTime*this.mSampleRate; this.mBuffer = wa.createBuffer( 2, this.mPlaySamples, this.mSampleRate ); this.mTextureDimensions = 512; this.mRenderTexture = createEmptyTextureNearest( gl, this.mTextureDimensions, this.mTextureDimensions ); this.mRenderFBO = createFBO( gl, this.mRenderTexture ); // ArrayBufferView pixels; this.mTmpBufferSamples = this.mTextureDimensions*this.mTextureDimensions; this.mData = new Uint8Array( this.mTmpBufferSamples*4 ); this.mPlayNode = null; } EffectPass.prototype.Destroy_Sound = function( wa, gl ) { if ( this.mPlayNode!=null ) this.mPlayNode.stop(); this.mPlayNode = null; this.mBuffer = null; this.mData = null; deleteFBO( gl, this.mRenderFBO ); deleteTexture( gl, this.mRenderTexture ); } EffectPass.prototype.NewShader_Sound = function( gl, shaderCode ) { var vsSource = "attribute vec2 pos; void main() { gl_Position = vec4(pos.xy,0.0,1.0); }"; var res = CreateShader( gl, vsSource, this.mHeader + shaderCode + this.mSoundPassFooter, false ); if ( res.mSuccess==false ) return res.mInfo; if ( this.mProgram != null ) gl.deleteProgram( this.mProgram ); this.mProgram = res.mProgram; // force sound to be regenerated this.mFrame = 0; return null; } EffectPass.prototype.Paint_Sound = function( wa, gl ) { gl.bindFramebuffer( gl.FRAMEBUFFER, this.mRenderFBO ); gl.viewport( 0, 0, this.mTextureDimensions, this.mTextureDimensions ); gl.useProgram( this.mProgram ); for ( var i=0; i<this.mInputs.length; i++ ) { var inp = this.mInputs[i]; gl.activeTexture( gl.TEXTURE0+i ); if ( inp==null ) { gl.bindTexture( gl.TEXTURE_2D, null ); } else if ( inp.mInfo.mType=="texture" ) { gl.bindTexture( gl.TEXTURE_2D, inp.loaded == false ? null : inp.globject ); } } var l2 = gl.getUniformLocation( this.mProgram, "iBlockOffset" ); var l9 = gl.getUniformLocation( this.mProgram, "iSampleRate" ); gl.uniform1f( l9, this.mSampleRate ); var ich0 = gl.getUniformLocation( this.mProgram, "iChannel0" ); if( ich0!=null ) gl.uniform1i( ich0, 0 ); var l1 = gl.getAttribLocation( this.mProgram, "pos" ); gl.bindBuffer( gl.ARRAY_BUFFER, this.mQuadVBO ); gl.vertexAttribPointer( l1, 2, gl.FLOAT, false, 0, 0); gl.enableVertexAttribArray( l1 ); var bufL = this.mBuffer.getChannelData(0); // Float32Array var bufR = this.mBuffer.getChannelData(1); // Float32Array var numBlocks = this.mPlaySamples / this.mTmpBufferSamples; for ( var j = 0; j < numBlocks; j++ ) { var off = j * this.mTmpBufferSamples; gl.uniform1f( l2, off/this.mSampleRate ); gl.drawArrays( gl.TRIANGLES, 0, 6 ); gl.readPixels( 0, 0, this.mTextureDimensions, this.mTextureDimensions, gl.RGBA, gl.UNSIGNED_BYTE, this.mData ); for ( var i=0; i<this.mTmpBufferSamples; i++ ) { bufL[off+i] = -1.0 + 2.0 * (this.mData[4 * i + 0] + 256.0 * this.mData[4 * i + 1]) / 65535.0; bufR[off+i] = -1.0 + 2.0 * (this.mData[4 * i + 2] + 256.0 * this.mData[4 * i + 3]) / 65535.0; } } gl.disableVertexAttribArray( l1 ); gl.useProgram( null ); gl.bindFramebuffer( gl.FRAMEBUFFER, null ); if ( this.mPlayNode != null ) { this.mPlayNode.disconnect(); this.mPlayNode.stop(); } this.mPlayNode = wa.createBufferSource(); this.mPlayNode.buffer = this.mBuffer; this.mPlayNode.connect( this.mGainNode ); this.mPlayNode.state = this.mPlayNode.noteOn; this.mPlayNode.start(0); } EffectPass.prototype.StopOutput_Sound = function( wa, gl ) { this.mPlayNode.disconnect(); } EffectPass.prototype.ResumeOutput_Sound = function( wa, gl ) { this.mPlayNode.connect( this.mGainNode ); } function Effect( vr, ac, gl, xres, yres, callback, obj, forceMuted, forcePaused ) { this.mSupportTextureFloat = (gl.getExtension('OES_texture_float') != null ); var precision = DetermineShaderPrecission( gl ); if ( ac!=null ) { this.mGainNode = ac.createGain(); this.mGainNode.connect( ac.destination); this.mGainNode.gain.value = (this.mForceMuted)?0.0:1.0; } this.mQuadVBO = createQuadVBO( gl ); } Effect.prototype.AddPass = function( passType ) { var shaderStr = "vec2 mainSound(float time)\n{\n return vec2( sin(6.2831*440.0*time)*exp(-3.0*time) );\n}"; var id = this.GetNumPasses(); this.mPasses[id].Create( passType, this.mAudioContext, this.mGLContext ); var res = this.mPasses[id].NewShader( this.mGLContext, shaderStr ); return { mId : id, mShader : shaderStr, mError : res }; } CreateShader = function( gl, tvs, tfs, nativeDebug ) { if( gl==null ) return {mSuccess:false, mInfo:"no GL"}; var vs = gl.createShader(gl.VERTEX_SHADER); gl.shaderSource(vs, tvs); gl.compileShader(vs); var fs = gl.createShader(gl.FRAGMENT_SHADER); gl.shaderSource(fs, tfs); gl.compileShader(fs); if (!gl.getShaderParameter(vs, gl.COMPILE_STATUS)) { var infoLog = gl.getShaderInfoLog(vs); return {mSuccess:false, mInfo:infoLog}; } if (!gl.getShaderParameter( fs, gl.COMPILE_STATUS)) { var infoLog = gl.getShaderInfoLog(fs); return {mSuccess:false, mInfo:infoLog}; } if( nativeDebug ) { var dbgext = gl.getExtension("WEBGL_debug_shaders"); if( dbgext != null ) { var hlsl = dbgext.getTranslatedShaderSource( fs ); console.log( "------------------------\nHLSL code\n------------------------\n" + hlsl + "\n------------------------\n" ); } } var tmpProgram = gl.createProgram(); gl.attachShader(tmpProgram, vs); gl.attachShader(tmpProgram, fs); gl.deleteShader(vs); gl.deleteShader(fs); gl.linkProgram(tmpProgram); if( !gl.getProgramParameter(tmpProgram,gl.LINK_STATUS) ) { var infoLog = gl.getProgramInfoLog(tmpProgram); gl.deleteProgram( tmpProgram ); return {mSuccess:false, mInfo:infoLog}; } return {mSuccess:true, mProgram:tmpProgram}; } function createGLTexture( gl, image, format, texture ) { if( gl==null ) return; //var texture = gl.createTexture(); gl.bindTexture( gl.TEXTURE_2D, texture); //gl.pixelStorei( gl.UNPACK_FLIP_Y_WEBGL, false ); gl.texImage2D( gl.TEXTURE_2D, 0, format, gl.RGBA, gl.UNSIGNED_BYTE, image); gl.texParameteri( gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri( gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR_MIPMAP_LINEAR); gl.texParameteri( gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.REPEAT); gl.texParameteri( gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.REPEAT); gl.generateMipmap(gl.TEXTURE_2D); gl.bindTexture(gl.TEXTURE_2D, null); } function CreateTextureRGB8LinearClamp( gl, xres, yres ) { gl.bindTexture( gl.TEXTURE_2D, id ); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, xres, yres, 0, gl.RGBA, gl.UNSIGNED_BYTE, null ); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.bindTexture(gl.TEXTURE_2D, null); return id; } function createGLTextureLinear( gl, image, texture ) { if( gl==null ) return; gl.bindTexture( gl.TEXTURE_2D, texture); gl.pixelStorei( gl.UNPACK_FLIP_Y_WEBGL, false ); gl.texImage2D( gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, image); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.bindTexture(gl.TEXTURE_2D, null); } function createAudioTexture( gl, texture ) { if( gl==null ) return; gl.bindTexture( gl.TEXTURE_2D, texture ); gl.texParameteri( gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR ); gl.texParameteri( gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR ); //gl.texParameteri( gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE ); //gl.texParameteri( gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE) ; gl.texImage2D( gl.TEXTURE_2D, 0, gl.LUMINANCE, 512, 2, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, null); gl.bindTexture( gl.TEXTURE_2D, null); } function deleteTexture( gl, tex ) { gl.deleteTexture( tex ); } function createQuadVBO( gl ) { var vertices = new Float32Array( [ -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0] ); var vbo = gl.createBuffer(); gl.bindBuffer( gl.ARRAY_BUFFER, vbo ); gl.bufferData( gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW ); gl.bindBuffer( gl.ARRAY_BUFFER, null ); return vbo; } function createFBO( gl, texture0 ) { var fbo = gl.createFramebuffer(); gl.bindFramebuffer( gl.FRAMEBUFFER, fbo ); gl.framebufferTexture2D( gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture0, 0 ); gl.bindFramebuffer( gl.FRAMEBUFFER, null ); return fbo; } function deleteFBO( gl, fbo ) { gl.deleteFramebuffer( fbo ); } function DetermineShaderPrecission( gl ) { var h1 = "#ifdef GL_ES\n" + "precision highp float;\n" + "#endif\n"; var h2 = "#ifdef GL_ES\n" + "precision mediump float;\n" + "#endif\n"; var h3 = "#ifdef GL_ES\n" + "precision lowp float;\n" + "#endif\n"; var vstr = "void main() { gl_Position = vec4(1.0); }\n"; var fstr = "void main() { gl_FragColor = vec4(1.0); }\n"; if( CreateShader( gl, vstr, h1 + fstr, false).mSuccess==true ) return h1; if( CreateShader( gl, vstr, h2 + fstr, false).mSuccess==true ) return h2; if( CreateShader( gl, vstr, h3 + fstr, false).mSuccess==true ) return h3; return ""; }