str = "(foo (bar baz)) (qux quux)"

ary = [[]]
str.scan(/\G\s*(\(|\)|[^()\s]+)/) do |x,|
case x
when "(" then ary << []
when ")" then x = ary.pop; ary.last << x
else ary.last << x
end
end
raise unless ary.size == 1
ary = ary.first

p ary #=> [["foo", ["bar", "baz"]], ["qux", "quux"]]
(foo, (bar, baz)), (qux, quux) = ary
p [foo, bar, baz, qux, quux] #=> ["foo", "bar", "baz", "qux", "quux"]

要件がよくわからないが、これだけのことじゃないの？