Skip to content

Convert AbstractArrays with strides to NumPy arrays #876

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 29 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
f3e9f3c
Add pyslice and PySlice for Python slices
mkitti Jan 6, 2021
48cb90b
PermutedDimsArray and StridedSubArray PyObject OL
mkitti Jan 6, 2021
bfbb1e9
pyembed ReinterpretArray as parent
mkitti Jan 11, 2021
9db36cd
Widen NpyArray to AbstractArray from StridedArray
mkitti Jan 12, 2021
8909d2b
Consolidate PySlice
mkitti Jan 12, 2021
9fd89da
Fix scalar indices in SubArray
mkitti Jan 12, 2021
d4c1899
Enhance coverage with tests
mkitti Jan 12, 2021
39eb96b
Fix Core.TypeofVararg pyany_toany for Julia 1.7
mkitti Jan 12, 2021
457d1f3
Fix roundtripeq typo for ranges
mkitti Jan 12, 2021
38a8db3
Fix Julia 1.0 isnothing, and 1.7 Core.TypeofVararg
mkitti Jan 12, 2021
5520835
Lower version for Core.TypeofVararg
mkitti Jan 12, 2021
b2c3b41
Remove @static from TypeOfVararg
mkitti Jan 12, 2021
bb3f68e
Skip Core.TypeofVararg test for 1.7 prerelease
mkitti Jan 12, 2021
a9e562b
Check version during AOT test
mkitti Jan 12, 2021
3d4760e
Fix AOT workflow due to resolved issues
mkitti Jan 12, 2021
09ab06a
Add --project when running tests
mkitti Jan 12, 2021
42eda17
Fix runtests.sh arg order
mkitti Jan 12, 2021
f05803c
Fix runtests.sh
mkitti Jan 12, 2021
57fd8fd
Cleanup tests
mkitti Jan 12, 2021
151bffe
Fix whitespace
mkitti Jan 12, 2021
60979d7
Annotate Core.TypeofVararg code
mkitti Jan 12, 2021
51dfc02
Remove PyObjectWithParent code
mkitti Jan 13, 2021
cd36aa2
Remove automatic conversion of slices to PySlice
mkitti Jan 13, 2021
fa4ed28
Use applicable rather than hasmethod
mkitti Jan 13, 2021
0c8db20
Remove PyObjectWithParent tests
mkitti Jan 13, 2021
777db62
Replace Core.TypeofVararg with typeof(Vararg)
mkitti Jan 15, 2021
d04dc61
Merge remote-tracking branch 'origin/master' into transpose_and_slice
mkitti Jan 17, 2021
97bb48b
Remove PySlice, pyslice, and other slice related code for a separate PR
mkitti Jan 28, 2021
d7a1030
Remove unused reference to StridedSubArray
mkitti Jan 28, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/PyCall.jl
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import Base: size, ndims, similar, copy, getindex, setindex!, stride,
filter!, hash, splice!, pop!, ==, isequal, push!,
append!, insert!, prepend!, unsafe_convert,
pushfirst!, popfirst!, firstindex, lastindex,
getproperty, setproperty!, propertynames
getproperty, setproperty!, propertynames,
ReinterpretArray, ReshapedArray

if isdefined(Base, :hasproperty) # Julia 1.2
import Base: hasproperty
Expand Down
15 changes: 14 additions & 1 deletion src/gc.jl
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,13 @@ const weakref_callback_meth = Ref{PyMethodDef}()
function pyembed(po::PyObject, jo::Any)
# If there's a need to support immutable embedding,
# the API needs to be changed to return the pointer.
isimmutable(jo) && throw(ArgumentError("pyembed: immutable argument not allowed"))
if isimmutable(jo)
if applicable(parent, jo)
return pyembed(po, parent(jo) )
else
throw(ArgumentError("pyembed: immutable argument not allowed"))
end
end
if ispynull(weakref_callback_obj)
cf = @cfunction(weakref_callback, PyPtr, (PyPtr,PyPtr))
weakref_callback_meth[] = PyMethodDef("weakref_callback", cf, METH_O)
Expand All @@ -43,3 +49,10 @@ function pyembed(po::PyObject, jo::Any)
pycall_gc[wo] = jo
return po
end

# Embed the mutable type underlying the immutable view of the array
# See Base.unsafe_convert(::Type{Ptr{T}}, jo::ArrayType) for specific array types
pyembed(po::PyObject, jo::SubArray ) = pyembed(po, jo.parent)
pyembed(po::PyObject, jo::ReshapedArray ) = pyembed(po, jo.parent)
pyembed(po::PyObject, jo::ReinterpretArray ) = pyembed(po, jo.parent)
pyembed(po::PyObject, jo::PermutedDimsArray) = pyembed(po, jo.parent)
6 changes: 3 additions & 3 deletions src/numpy.jl
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ const NPY_ARRAY_WRITEABLE = Int32(0x0400)
# dimensions. For example, although NumPy works with both row-major and
# column-major data, some Python libraries like OpenCV seem to require
# row-major data (the default in NumPy). In such cases, use PyReverseDims(array)
function NpyArray(a::StridedArray{T}, revdims::Bool) where T<:PYARR_TYPES
function NpyArray(a::AbstractArray{T}, revdims::Bool) where T<:PYARR_TYPES
@npyinitialize
size_a = revdims ? reverse(size(a)) : size(a)
strides_a = revdims ? reverse(strides(a)) : strides(a)
Expand All @@ -186,15 +186,15 @@ function NpyArray(a::StridedArray{T}, revdims::Bool) where T<:PYARR_TYPES
return PyObject(p, a)
end

function PyObject(a::StridedArray{T}) where T<:PYARR_TYPES
function PyObject(a::AbstractArray{T}) where T<:PYARR_TYPES
try
return NpyArray(a, false)
catch
return array2py(a) # fallback to non-NumPy version
end
end

function PyReverseDims(a::StridedArray{T,N}) where {T<:PYARR_TYPES,N}
function PyReverseDims(a::AbstractArray{T,N}) where {T<:PYARR_TYPES,N}
try
return NpyArray(a, true)
catch
Expand Down
38 changes: 38 additions & 0 deletions test/runtests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ const PyInt = pyversion < v"3" ? Int : Clonglong
@test roundtripeq(Int32)
@test roundtripeq(Dict(1 => "hello", 2 => "goodbye")) && roundtripeq(Dict())
@test roundtripeq(UInt8[1,3,4,5])
@test roundtripeq(1:5)
@test roundtripeq(5:2:10)
@test roundtrip(3 => 4) == (3,4)
@test roundtrip(Pair{Int,Int}, 3 => 4) == Pair(3,4)
@test eltype(roundtrip([Ref(1), Ref(2)])) == typeof(Ref(1))
Expand Down Expand Up @@ -107,6 +109,42 @@ const PyInt = pyversion < v"3" ? Int : Clonglong
@test GC.@preserve(o, pyincref.(PyArray(o))) == a
end
end
let A = Float64[1 2; 3 4]
# Normal array
B = copy(A)
C = PyArray( PyObject(B) )
@test C == B
B[1] = 3
@test C == B && C[1] == B[1]

# SubArray
B = view(A, 1:2, 2:2)
C = PyArray( PyObject(B) )
@test C == B
A[3] = 5
@test C == B && C[1] == A[3]

# ReshapedArray
B = Base.ReshapedArray( A, (1,4), () )
C = PyArray( PyObject(B) )
@test C == B
A[2] = 6
@test C == B && C[2] == A[2]

# PermutedDimsArray
B = PermutedDimsArray(A, (2,1) )
C = PyArray( PyObject(B) )
@test C == B
A[1] == 7
@test C == B && C[1] == A[1]

# ReinterpretArray
B = reinterpret(UInt64, A)
C = PyArray( PyObject(B) )
@test C == B
A[1] = 12
@test C == B && C[1] == reinterpret(UInt64, A[1])
end
end
@test PyVector(PyObject([1,3.2,"hello",true])) == [1,3.2,"hello",true]
@test PyDict(PyObject(Dict(1 => "hello", 2 => "goodbye"))) == Dict(1 => "hello", 2 => "goodbye")
Expand Down