From a1ed3576296ecff4f7905d2fa1bf533766f11d2f Mon Sep 17 00:00:00 2001 From: Spencer Russell Date: Mon, 23 Jun 2014 02:10:35 -0400 Subject: [PATCH] refactored AudioNode to contain a AudioRenderer, tests passing --- src/AudioIO.jl | 56 +++++++----- src/nodes.jl | 208 +++++++++++++++++++------------------------ src/operators.jl | 11 +++ src/portaudio.jl | 16 ++-- src/sndfile.jl | 31 +++---- test/test_AudioIO.jl | 16 ++-- test/test_nodes.jl | 92 +++++++++---------- 7 files changed, 211 insertions(+), 219 deletions(-) diff --git a/src/AudioIO.jl b/src/AudioIO.jl index 1acf400..8b722a8 100644 --- a/src/AudioIO.jl +++ b/src/AudioIO.jl @@ -12,11 +12,12 @@ typealias AudioSample Float32 # A frame of audio, possibly multi-channel typealias AudioBuf Array{AudioSample} -# A node in the render tree -abstract AudioNode +# used as a type parameter for AudioNodes. Subtypes handle the actual DSP for +# each node +abstract AudioRenderer -# A stream of audio (for instance that writes to hardware) -# All AudioStream subtypes should have a mixer and info field +# A stream of audio (for instance that writes to hardware). All AudioStream +# subtypes should have a mixer and info field abstract AudioStream # An audio interface is usually a physical sound card, but could @@ -29,6 +30,31 @@ type DeviceInfo buf_size::Integer end +type AudioNode{T<:AudioRenderer} + active::Bool + end_cond::Condition + renderer::T + + function AudioNode(renderer::AudioRenderer) + new(true, Condition(), renderer) + end +end + +function render(node::AudioNode, input::AudioBuf, info::DeviceInfo) + # TODO: not sure if the compiler will infer that render() always returns an + # AudioBuf. Might need to help it + if node.active + result = render(node.renderer, input, info) + if length(result) < info.buf_size + node.active = false + notify(node.end_cond) + end + return result + else + return AudioSample[] + end +end + include("nodes.jl") include("portaudio.jl") include("sndfile.jl") @@ -38,8 +64,7 @@ include("operators.jl") # Play an AudioNode by adding it as an input to the root mixer node function play(node::AudioNode, stream::AudioStream) - activate(node) - add_input(stream.mixer, node) + push!(stream.root, node) return node end @@ -53,26 +78,13 @@ function play(node::AudioNode) end function stop(node::AudioNode) - deactivate(node) - node -end - -function activate(node::AudioNode) - node.active = true -end - -function deactivate(node::AudioNode) node.active = false - notify(node.deactivate_cond) -end - -function is_active(node::AudioNode) - node.active + notify(node.end_cond) end function Base.wait(node::AudioNode) - if is_active(node) - wait(node.deactivate_cond) + if node.active + wait(node.end_cond) end end diff --git a/src/nodes.jl b/src/nodes.jl index c38ecdc..f83a1ba 100644 --- a/src/nodes.jl +++ b/src/nodes.jl @@ -1,146 +1,120 @@ +#### NullNode #### + +type NullRenderer <: AudioRenderer end +typealias NullNode AudioNode{NullRenderer} +NullNode() = NullNode(NullRenderer()) +export NullNode + +function render(node::NullRenderer, device_input::AudioBuf, info::DeviceInfo) + # TODO: preallocate buffer + return zeros(info.buf_size) +end + #### SinOsc #### # Generates a sin tone at the given frequency -export SinOsc -type SinOsc <: AudioNode - active::Bool - deactivate_cond::Condition +type SinOscRenderer <: AudioRenderer freq::Real phase::FloatingPoint - function SinOsc(freq::Real) - new(false, Condition(), freq, 0.0) + function SinOscRenderer(freq::Real) + new(freq, 0.0) end end -function render(node::SinOsc, device_input::AudioBuf, info::DeviceInfo) - phase = AudioSample[1:info.buf_size] * 2pi * node.freq / info.sample_rate +typealias SinOsc AudioNode{SinOscRenderer} +SinOsc(freq::Real) = SinOsc(SinOscRenderer(freq)) +SinOsc() = SinOsc(440) +export SinOsc + +function render(node::SinOscRenderer, device_input::AudioBuf, info::DeviceInfo) + phase = AudioSample[0:(info.buf_size-1)] * 2pi * node.freq / info.sample_rate phase .+= node.phase - node.phase = phase[end] - return sin(phase), is_active(node) + node.phase = phase[end] + 2pi * node.freq / info.sample_rate + return sin(phase) end -#### Gain #### -export Gain -type Gain <: AudioNode - active::Bool - deactivate_cond::Condition - in_node::AudioNode - gain::Float32 - - function Gain(in_node::AudioNode, gain::Real) - new(false, Condition(), in_node, gain) - end -end - -function render(node::Gain, device_input::AudioBuf, info::DeviceInfo) - input, child_active = render(node.in_node, device_input, info) - # TODO: should we check the active flag of the input? - return input .* node.gain, is_active(node) -end - - #### AudioMixer #### # Mixes a set of inputs equally -# a convenience alias used in the array of mix inputs -typealias MaybeAudioNode Union(AudioNode, Nothing) -const MAX_MIXER_INPUTS = 32 +type MixRenderer <: AudioRenderer + inputs::Vector{AudioNode} +end +typealias AudioMixer AudioNode{MixRenderer} +AudioMixer{T<:AudioNode}(inputs::Vector{T}) = AudioMixer(MixRenderer(inputs)) +AudioMixer() = AudioMixer(AudioNode[]) export AudioMixer -type AudioMixer <: AudioNode - active::Bool - deactivate_cond::Condition - mix_inputs::Array{MaybeAudioNode} - function AudioMixer{T <: AudioNode}(mix_inputs::Array{T}) - input_array = Array(MaybeAudioNode, MAX_MIXER_INPUTS) - fill!(input_array, nothing) - for (i, node) in enumerate(mix_inputs) - input_array[i] = node - end - new(false, Condition(), input_array) - end - - function AudioMixer() - AudioMixer(AudioNode[]) - end -end - -# TODO: at some point we need to figure out what the general API is for wiring -# up AudioNodes to each other -function add_input(mixer::AudioMixer, in_node::AudioNode) - for (i, node) in enumerate(mixer.mix_inputs) - if node === nothing - mixer.mix_inputs[i] = in_node - return - end - end - error("Mixer input array is full") -end - -# removes the given node from the mix inputs. If the node isn't an input the -# function returns without error -function remove_input(mixer::AudioMixer, in_node::AudioNode) - for (i, node) in enumerate(mixer.mix_inputs) - if node === in_node - mixer.mix_inputs[i] = nothing - return - end - end - # not an error if we didn't find it -end - -function render(node::AudioMixer, device_input::AudioBuf, info::DeviceInfo) +function render(node::MixRenderer, device_input::AudioBuf, info::DeviceInfo) # TODO: we probably want to pre-allocate this buffer and share between # render calls. Unfortunately we don't know the right size when the object # is created, so maybe we check the size on every render call and only # re-allocate when the size changes? I suppose that's got to be cheaper # than the GC and allocation every frame + mix_buffer = zeros(AudioSample, info.buf_size) - for in_node in node.mix_inputs - if in_node !== nothing - in_buffer, active = render(in_node, device_input, info) - mix_buffer += in_buffer - if !active - remove_input(node, in_node) - end + n_inputs = length(node.inputs) + i = 1 + max_samples = 0 + while i <= n_inputs + rendered = render(node.inputs[i], device_input, info)::AudioBuf + nsamples = length(rendered) + max_samples = max(max_samples, nsamples) + mix_buffer[1:nsamples] .+= rendered + if nsamples < info.buf_size + deleteat!(node.inputs, i) + n_inputs -= 1 + else + i += 1 end end - return mix_buffer, is_active(node) + return mix_buffer[1:max_samples] end +Base.push!(mixer::AudioMixer, node::AudioNode) = push!(mixer.renderer.inputs, node) + +#### Gain #### +type GainRenderer <: AudioRenderer + in_node::AudioNode + gain::Float32 +end + +function render(node::GainRenderer, device_input::AudioBuf, info::DeviceInfo) + input = render(node.in_node, device_input, info) + return input .* node.gain +end + +typealias Gain AudioNode{GainRenderer} +Gain(in_node::AudioNode, gain::Real) = Gain(GainRenderer(in_node, gain)) +export Gain + + #### Array Player #### # Plays a AudioBuf by rendering it out piece-by-piece -export ArrayPlayer -type ArrayPlayer <: AudioNode - active::Bool - deactivate_cond::Condition +type ArrayRenderer <: AudioRenderer arr::AudioBuf arr_index::Int - function ArrayPlayer(arr::AudioBuf) - new(false, Condition(), arr, 1) - end + ArrayRenderer(arr::AudioBuf) = new(arr, 1) end -function render(node::ArrayPlayer, device_input::AudioBuf, info::DeviceInfo) +typealias ArrayPlayer AudioNode{ArrayRenderer} +ArrayPlayer(arr::AudioBuf) = ArrayPlayer(ArrayRenderer(arr)) +export ArrayPlayer + +function render(node::ArrayRenderer, device_input::AudioBuf, info::DeviceInfo) # TODO: this should remove itself from the render tree when playback is # complete i = node.arr_index range_end = min(i + info.buf_size-1, length(node.arr)) output = node.arr[i:range_end] - if length(output) < info.buf_size - # we're finished with the array, pad with zeros and deactivate - output = vcat(output, zeros(AudioSample, info.buf_size - length(output))) - deactivate(node) - end node.arr_index = range_end + 1 - return output, is_active(node) + return output end # Allow users to play a raw array by wrapping it in an ArrayPlayer @@ -171,22 +145,22 @@ function play{T <: Unsigned}(arr::Array{T}, args...) play(arr, args...) end -#### AudioInput #### - -# Renders incoming audio input from the hardware - -export AudioInput -type AudioInput <: AudioNode - active::Bool - deactivate_cond::Condition - channel::Int - - function AudioInput(channel::Int) - new(false, Condition(), channel) - end -end - -function render(node::AudioInput, device_input::AudioBuf, info::DeviceInfo) - @assert size(device_input, 1) == info.buf_size - return device_input[:, node.channel], is_active(node) -end +##### AudioInput #### +# +## Renders incoming audio input from the hardware +# +#export AudioInput +#type AudioInput <: AudioNode +# active::Bool +# deactivate_cond::Condition +# channel::Int +# +# function AudioInput(channel::Int) +# new(false, Condition(), channel) +# end +#end +# +#function render(node::AudioInput, device_input::AudioBuf, info::DeviceInfo) +# @assert size(device_input, 1) == info.buf_size +# return device_input[:, node.channel], is_active(node) +#end diff --git a/src/operators.jl b/src/operators.jl index 864c64e..fa3ca7a 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -1,2 +1,13 @@ *(node::AudioNode, coef::Real) = Gain(node, coef) *(coef::Real, node::AudioNode) = Gain(node, coef) +# multiplying by silence gives silence +*(in1::NullNode, in2::NullNode) = in1 +*(in1::AudioNode, in2::NullNode) = in2 +*(in1::NullNode, in2::AudioNode) = in1 + + ++(in1::AudioNode, in2::AudioNode) = AudioMixer([in1, in2]) +# adding silence has no effect ++(in1::NullNode, in2::NullNode) = in1 ++(in1::AudioNode, in2::NullNode) = in1 ++(in1::NullNode, in2::AudioNode) = in2 diff --git a/src/portaudio.jl b/src/portaudio.jl index 18ea73d..54e52dc 100644 --- a/src/portaudio.jl +++ b/src/portaudio.jl @@ -35,13 +35,13 @@ portaudio_inited = false ################## Types #################### type PortAudioStream <: AudioStream - mixer::AudioMixer + root::AudioMixer info::DeviceInfo function PortAudioStream(sample_rate::Int=44100, buf_size::Int=1024) init_portaudio() - mixer = AudioMixer() - stream = new(mixer, DeviceInfo(sample_rate, buf_size)) + root = AudioMixer() + stream = new(root, DeviceInfo(sample_rate, buf_size)) # we need to start up the stream with the portaudio library open_portaudio_stream(stream) return stream @@ -91,8 +91,14 @@ function portaudio_task(jl_filedesc::Integer, stream::PortAudioStream) jl_rawfd = RawFD(jl_filedesc) try while true - # assume the root mixer is always active - buffer::AudioBuf, _::Bool = render(stream.mixer, buffer, stream.info) + # assume the root is always active + rendered = render(stream.root, buffer, stream.info) + for i in 1:length(rendered) + buffer[i] = rendered[i] + end + for i in (length(rendered)+1):length(buffer) + buffer[i] = 0.0 + end # wake the C code so it knows we've given it some more data synchronize_buffer(buffer) diff --git a/src/sndfile.jl b/src/sndfile.jl index f7409fb..e2b3a29 100644 --- a/src/sndfile.jl +++ b/src/sndfile.jl @@ -88,7 +88,6 @@ end # through an arbitrary render chain and returns the result as a vector function Base.read(file::AudioFile, nframes::Integer = file.sfinfo.frames, dtype::Type = Int16) - arr = [] @assert file.sfinfo.channels <= 2 if file.sfinfo.channels == 2 arr = zeros(dtype, 2, nframes) @@ -114,11 +113,7 @@ function Base.read(file::AudioFile, nframes::Integer = file.sfinfo.frames, file.filePtr, arr, nframes) end - if nread == 0 - return Nothing - end - - return arr + return arr[1:nread] end function Base.write{T}(file::AudioFile, frames::Array{T}) @@ -144,38 +139,34 @@ function Base.write{T}(file::AudioFile, frames::Array{T}) end end -type FilePlayer <: AudioNode - active::Bool - deactivate_cond::Condition +type FileRenderer <: AudioRenderer file::AudioFile - function FilePlayer(file::AudioFile) - node = new(false, Condition(), file) + function FileRenderer(file::AudioFile) finalizer(node, node -> close(node.file)) return node end - - function FilePlayer(path::String) - return FilePlayer(af_open(path)) - end end -function render(node::FilePlayer, device_input::AudioBuf, info::DeviceInfo) +typealias FilePlayer AudioNode{FileRenderer} +FilePlayer(file::AudioFile) = FilePlayer(FileRenderer(file)) +FilePlayer(path::String) = FilePlayer(af_open(path)) + +function render(node::FileRenderer, device_input::AudioBuf, info::DeviceInfo) @assert node.file.sfinfo.samplerate == info.sample_rate audio = read(node.file, info.buf_size, AudioSample) if audio == Nothing - deactivate(node) - return zeros(AudioSample, info.buf_size), is_active(node) + return AudioSample[] end # if the file is stereo, mix the two channels together if node.file.sfinfo.channels == 2 - return (audio[1, :] / 2) + (audio[2, :] / 2), is_active(node) + return (audio[1, :] / 2) + (audio[2, :] / 2) end - return audio, is_active(node) + return audio end function play(filename::String, args...) diff --git a/test/test_AudioIO.jl b/test/test_AudioIO.jl index 0e89108..a292e75 100644 --- a/test/test_AudioIO.jl +++ b/test/test_AudioIO.jl @@ -5,20 +5,22 @@ const TEST_SAMPLERATE = 44100 const TEST_BUF_SIZE = 1024 type TestAudioStream <: AudioIO.AudioStream - mixer::AudioMixer + root::AudioIO.AudioMixer info::AudioIO.DeviceInfo function TestAudioStream() - mixer = AudioMixer() - new(mixer, AudioIO.DeviceInfo(TEST_SAMPLERATE, TEST_BUF_SIZE)) + root = AudioMixer() + new(root, AudioIO.DeviceInfo(TEST_SAMPLERATE, TEST_BUF_SIZE)) end end # render the stream and return the next block of audio. This is used in testing # to simulate the audio callback that's normally called by the device. function process(stream::TestAudioStream) + out_array = zeros(AudioIO.AudioSample, stream.info.buf_size) in_array = zeros(AudioIO.AudioSample, stream.info.buf_size) - out_array, _ = AudioIO.render(stream.mixer, in_array, stream.info) + rendered = AudioIO.render(stream.root, in_array, stream.info) + out_array[1:length(rendered)] = rendered return out_array end @@ -59,18 +61,12 @@ player = play(ui8, test_stream) @test_approx_eq(process(test_stream)[1:255], convert(AudioIO.AudioBuf, linspace(-1.0, 1.0, 255))) - info("Testing AudioNode Stopping...") test_stream = TestAudioStream() node = SinOsc(440) -@test !node.active play(node, test_stream) -@test node.active process(test_stream) stop(node) -@test !node.active -# give the render task a chance to clean up -process(test_stream) @test process(test_stream) == zeros(AudioIO.AudioSample, TEST_BUF_SIZE) info("Testing wav file write/read") diff --git a/test/test_nodes.jl b/test/test_nodes.jl index 16bcc7d..93442e0 100644 --- a/test/test_nodes.jl +++ b/test/test_nodes.jl @@ -1,23 +1,25 @@ using Base.Test using AudioIO +import AudioIO.AudioSample +import AudioIO.AudioBuf +import AudioIO.AudioRenderer +import AudioIO.AudioNode +import AudioIO.DeviceInfo +import AudioIO.render -test_info = AudioIO.DeviceInfo(44100, 512) -dev_input = zeros(AudioIO.AudioSample, test_info.buf_size) +test_info = DeviceInfo(44100, 512) +dev_input = zeros(AudioSample, test_info.buf_size) # A TestNode just renders out 1:buf_size each frame -type TestNode <: AudioIO.AudioNode - active::Bool - deactivate_cond::Condition +type TestRenderer <: AudioRenderer end - function TestNode() - return new(false, Condition()) - end -end +typealias TestNode AudioNode{TestRenderer} +TestNode() = TestNode(TestRenderer()) -function AudioIO.render(node::TestNode, - device_input::AudioIO.AudioBuf, - info::AudioIO.DeviceInfo) - return AudioIO.AudioSample[1:info.buf_size], node.active +function render(node::TestRenderer, + device_input::AudioBuf, + info::DeviceInfo) + return AudioSample[1:info.buf_size] end #### AudioMixer Tests #### @@ -27,64 +29,64 @@ end info("Testing AudioMixer...") mix = AudioMixer() -render_output, active = AudioIO.render(mix, dev_input, test_info) -@test render_output == zeros(AudioIO.AudioSample, test_info.buf_size) +render_output = render(mix, dev_input, test_info) +@test render_output == AudioSample[] testnode = TestNode() mix = AudioMixer([testnode]) -render_output, active = AudioIO.render(mix, dev_input, test_info) -@test render_output == AudioIO.AudioSample[1:test_info.buf_size] +render_output = render(mix, dev_input, test_info) +@test render_output == AudioSample[1:test_info.buf_size] test1 = TestNode() test2 = TestNode() mix = AudioMixer([test1, test2]) -render_output, active = AudioIO.render(mix, dev_input, test_info) +render_output = render(mix, dev_input, test_info) # make sure the two inputs are being added together -@test render_output == 2 * AudioIO.AudioSample[1:test_info.buf_size] +@test render_output == 2 * AudioSample[1:test_info.buf_size] # now we'll stop one of the inputs and make sure it gets removed -# TODO: this test should depend on the render output, not on the internals of -# the mixer stop(test1) -AudioIO.render(mix, dev_input, test_info) -@test !in(test1, mix.mix_inputs) +render_output = render(mix, dev_input, test_info) +# make sure the two inputs are being added together +@test render_output == AudioSample[1:test_info.buf_size] stop(mix) -render_output, active = AudioIO.render(mix, dev_input, test_info) -@test !active +render_output = render(mix, dev_input, test_info) +@test render_output == AudioSample[] info("Testing SinOSC...") freq = 440 -t = linspace(1 / test_info.sample_rate, - test_info.buf_size / test_info.sample_rate, +t = linspace(0, + (test_info.buf_size-1) / test_info.sample_rate, test_info.buf_size) -test_vect = convert(AudioIO.AudioBuf, sin(2pi * t * freq)) +test_vect = convert(AudioBuf, sin(2pi * t * freq)) osc = SinOsc(freq) -render_output, active = AudioIO.render(osc, dev_input, test_info) +render_output = render(osc, dev_input, test_info) @test_approx_eq(render_output, test_vect) stop(osc) -render_output, active = AudioIO.render(osc, dev_input, test_info) -@test !active +render_output = render(osc, dev_input, test_info) +@test render_output == AudioSample[] info("Testing ArrayPlayer...") -v = rand(AudioIO.AudioSample, 44100) +v = rand(AudioSample, 44100) player = ArrayPlayer(v) -player.active = true -render_output, active = AudioIO.render(player, dev_input, test_info) +render_output = render(player, dev_input, test_info) @test render_output == v[1:test_info.buf_size] -@test active -render_output, active = AudioIO.render(player, dev_input, test_info) +render_output = render(player, dev_input, test_info) @test render_output == v[(test_info.buf_size + 1) : (2*test_info.buf_size)] -@test active stop(player) -render_output, active = AudioIO.render(player, dev_input, test_info) -@test !active +render_output = render(player, dev_input, test_info) +@test render_output == AudioSample[] # give a vector just a bit larger than 1 buffer size -v = rand(AudioIO.AudioSample, test_info.buf_size + 1) +v = rand(AudioSample, test_info.buf_size + 1) player = ArrayPlayer(v) -player.active = true -_, active = AudioIO.render(player, dev_input, test_info) -@test active -_, active = AudioIO.render(player, dev_input, test_info) -@test !active +render(player, dev_input, test_info) +render_output = render(player, dev_input, test_info) +@test render_output == v[test_info.buf_size+1:end] + +info("Testing Gain...") + +gained = TestNode() * 0.75 +render_output = render(gained, dev_input, test_info) +@test render_output == 0.75 * AudioSample[1:test_info.buf_size]