diff --git a/src/MArray.jl b/src/MArray.jl
index a9f452e8..c2b99ae4 100644
--- a/src/MArray.jl
+++ b/src/MArray.jl
@@ -94,9 +94,12 @@ end
     if isbitstype(T)
         GC.@preserve v unsafe_store!(Base.unsafe_convert(Ptr{T}, pointer_from_objref(v)), convert(T, val), i)
     else
-        # This one is unsafe (#27)
-        # unsafe_store!(Base.unsafe_convert(Ptr{Ptr{Nothing}}, pointer_from_objref(v.data)), pointer_from_objref(val), i)
-        error("setindex!() with non-isbitstype eltype is not supported by StaticArrays. Consider using SizedArray.")
+        # This is inherently slow because we've got to rebuild a new Tuple to
+        # reassign to data.v.
+        #
+        # However it seems not possible to make fast, at least in julia 1.4
+        # TODO: Revisit if https://github.com/JuliaLang/julia/pull/34126 is merged
+        @inbounds v.data = setindex(v.data, val, i)
     end
 
     return val
diff --git a/test/MArray.jl b/test/MArray.jl
index 3a15eda8..63c81865 100644
--- a/test/MArray.jl
+++ b/test/MArray.jl
@@ -137,14 +137,29 @@
         m[4] = 14
         @test m.data === (11, 12, 13, 14)
 
+        m = @MMatrix [0 0; 0 0]
+        m[1] = Int8(11)
+        m[2] = Int8(12)
+        m[3] = Int8(13)
+        m[4] = Int8(14)
+        @test m.data === (11, 12, 13, 14)
+
+        v = @MVector [1.,2.,3.]
+        v[1] = Float16(11)
+        @test v.data === (11., 2., 3.)
+
+        v = @MArray [1,2,3]
         @test_throws BoundsError setindex!(v, 4, -1)
         mm = @MArray zeros(3,3,3,3)
         @test_throws BoundsError setindex!(mm, 4, -1)
         @test_throws BoundsError setindex!(mm, 4, 82)
 
-        # setindex with non-elbits type
-        m = MArray{Tuple{2,2,2}, String}(undef)
-        @test_throws ErrorException setindex!(m, "a", 1, 1, 1)
+        # setindex with non-bits eltype
+        m = fill("a", MMatrix{2,2, String})
+        m[1,1] = "b"
+        m[1,2] = "c"
+        @test m == ["b" "c";
+                    "a" "a"]
     end
 
     @testset "promotion" begin
diff --git a/test/MMatrix.jl b/test/MMatrix.jl
index aa85e765..e5a2ce0f 100644
--- a/test/MMatrix.jl
+++ b/test/MMatrix.jl
@@ -89,24 +89,4 @@
 
         @test length(m) === 4
     end
-
-    @testset "setindex!" begin
-        m = @MMatrix [0 0; 0 0]
-        m[1] = 11
-        m[2] = 12
-        m[3] = 13
-        m[4] = 14
-        @test m.data === (11, 12, 13, 14)
-
-        m = @MMatrix [0 0; 0 0]
-        m[1] = Int8(11)
-        m[2] = Int8(12)
-        m[3] = Int8(13)
-        m[4] = Int8(14)
-        @test m.data === (11, 12, 13, 14)
-
-        # setindex with non-elbits type
-        m = MMatrix{2,2,String}(undef)
-        @test_throws ErrorException setindex!(m, "a", 1, 1)
-    end
 end
diff --git a/test/MVector.jl b/test/MVector.jl
index bdc430a7..3dc1638a 100644
--- a/test/MVector.jl
+++ b/test/MVector.jl
@@ -69,23 +69,4 @@
 
         @test length(v) === 3
     end
-
-    @testset "setindex!" begin
-        v = @MVector [1,2,3]
-        v[1] = 11
-        v[2] = 12
-        v[3] = 13
-        @test v.data === (11, 12, 13)
-
-        v = @MVector [1.,2.,3.]
-        v[1] = Float16(11)
-        @test v.data === (11., 2., 3.)
-
-        @test_throws BoundsError setindex!(v, 4., -1)
-        @test_throws BoundsError setindex!(v, 4., 4)
-
-        # setindex with non-elbits type
-        v = MVector{2,String}(undef)
-        @test_throws ErrorException setindex!(v, "a", 1)
-    end
 end