Add StructArrays.append!! (JuliaArrays#97)

* Expose grow_to_structarray! as a public API

* Rename: collect_to_structarray! -> _collect_to_structarray!

* Add collect_to_structarray!(dest, itr)

* Rename: collect_to_structarray! -> append!!

* Mention append!! in README

Author: tkf

README.md

@@ -211,3 +211,40 @@ end
 s = [MyType(rand(), a=1, b=2) for i in 1:10]
 StructArray(s)
 ```
+
+## Advanced: mutate-or-widen style accumulation
+
+StructArrays provides a function `StructArrays.append!!(dest, src)` (unexported) for "mutate-or-widen" style accumulation.  This function can be used via [`BangBang.append!!`](https://tkf.github.io/BangBang.jl/dev/#BangBang.append!!-Tuple{Any,Any}) and [`BangBang.push!!`](https://tkf.github.io/BangBang.jl/dev/#BangBang.push!!-Tuple{Any,Any,Any,Vararg{Any,N}%20where%20N}) as well.
+
+`StructArrays.append!!` works like `append!(dest, src)` if `dest` can contain all element types in `src` iterator; i.e., it _mutates_ `dest` in-place:
+
+```julia
+julia> dest = StructVector((a=[1], b=[2]))
+1-element StructArray(::Array{Int64,1}, ::Array{Int64,1}) with eltype NamedTuple{(:a, :b),Tuple{Int64,Int64}}:
+ (a = 1, b = 2)
+
+julia> StructArrays.append!!(dest, [(a = 3, b = 4)])
+2-element StructArray(::Array{Int64,1}, ::Array{Int64,1}) with eltype NamedTuple{(:a, :b),Tuple{Int64,Int64}}:
+ (a = 1, b = 2)
+ (a = 3, b = 4)
+
+julia> ans === dest
+true
+```
+
+Unlike `append!`, `append!!` can also _widen_ element type of `dest` array element types:
+
+```julia
+julia> StructArrays.append!!(dest, [(a = missing, b = 6)])
+3-element StructArray(::Array{Union{Missing, Int64},1}, ::Array{Int64,1}) with eltype NamedTuple{(:a, :b),Tuple{Union{Missing, Int64},Int64}}:
+ NamedTuple{(:a, :b),Tuple{Union{Missing, Int64},Int64}}((1, 2))
+ NamedTuple{(:a, :b),Tuple{Union{Missing, Int64},Int64}}((3, 4))
+ NamedTuple{(:a, :b),Tuple{Union{Missing, Int64},Int64}}((missing, 6))
+
+julia> ans === dest
+false
+```
+
+Since the original array `dest` cannot hold the input, a new array is created (`ans !== dest`).
+
+Combined with [function barriers](https://docs.julialang.org/en/latest/manual/performance-tips/#kernel-functions-1), `append!!` is a useful building block for implementing `collect`-like functions.

src/collect.jl

@@ -128,3 +128,41 @@ function widenarray(dest::AbstractArray, i, ::Type{T}) where T
     copyto!(new, 1, dest, 1, i-1)
     new
 end
+
+"""
+`append!!(dest, itr) -> dest′`
+
+Try to append `itr` into a vector `dest`.  Widen element type of
+`dest` if it cannot hold the elements of `itr`.  That is to say,
+
+```julia
+vcat(dest, StructVector(itr)) == append!!(dest, itr)
+```
+
+holds.  Note that `dest′` may or may not be the same object as `dest`.
+The state of `dest` is unpredictable after `append!!`
+is called (e.g., it may contain just half of the elements from `itr`).
+"""
+append!!(dest::AbstractVector, itr) =
+    _append!!(dest, itr, Base.IteratorSize(itr))
+
+function _append!!(dest::AbstractVector, itr, ::Union{Base.HasShape, Base.HasLength})
+    n = length(itr)  # itr may be stateful so do this first
+    fr = iterate(itr)
+    fr === nothing && return dest
+    el, st = fr
+    i = lastindex(dest) + 1
+    if iscompatible(el, dest)
+        resize!(dest, length(dest) + n)
+        @inbounds dest[i] = el
+        return collect_to_structarray!(dest, itr, i + 1, st)
+    else
+        new = widenstructarray(dest, i, el)
+        resize!(new, length(dest) + n)
+        @inbounds new[i] = el
+        return collect_to_structarray!(new, itr, i + 1, st)
+    end
+end
+
+_append!!(dest::AbstractVector, itr, ::Base.SizeUnknown) =
+    grow_to_structarray!(dest, itr)

test/runtests.jl

@@ -1,5 +1,5 @@
 using StructArrays
-using StructArrays: staticschema, iscompatible, _promote_typejoin
+using StructArrays: staticschema, iscompatible, _promote_typejoin, append!!
 using OffsetArrays: OffsetArray
 import Tables, PooledArrays, WeakRefStrings
 using Test
@@ -646,3 +646,24 @@ end
     str = String(take!(io))
     @test str == "StructArray(::Array{Int64,1}, ::Array{Int64,1})"
 end
+
+@testset "append!!" begin
+    dest_examples = [
+        ("mutate", StructVector(a = [1], b = [2])),
+        ("widen", StructVector(a = [1], b = [nothing])),
+    ]
+    itr = [(a = 1, b = 2), (a = 1, b = 2), (a = 1, b = 12)]
+    itr_examples = [
+        ("HasLength", () -> itr),
+        ("SizeUnknown", () -> (x for x in itr if isodd(x.a))),
+        # Broken due to https://github.com/JuliaArrays/StructArrays.jl/issues/100:
+        # ("empty", (x for x in itr if false)),
+        # Broken due to https://github.com/JuliaArrays/StructArrays.jl/issues/99:
+        # ("stateful", () -> Iterators.Stateful(itr)),
+    ]
+    @testset "$destlabel $itrlabel" for (destlabel, dest) in dest_examples,
+                                        (itrlabel, makeitr) in itr_examples
+
+        @test vcat(dest, StructVector(makeitr())) == append!!(copy(dest), makeitr())
+    end
+end