support data in more byte-array/string types (#54)

Author: stevengj

Project.toml

@@ -1,6 +1,6 @@
 name = "StringEncodings"
 uuid = "69024149-9ee7-55f6-a4c4-859efe599b68"
-version = "0.3.6"
+version = "0.3.7"
 
 [deps]
 Libiconv_jll = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531"

src/StringEncodings.jl

@@ -20,6 +20,12 @@ export encoding, encodings_list, Encoding, @enc_str
 
 abstract type StringEncodingError end
 
+# contiguous 1d byte arrays compatible with C `unsigned char *` API
+const ByteVector= Union{Vector{UInt8},
+                        Base.FastContiguousSubArray{UInt8,1,<:Array{UInt8,1}},
+                        Base.CodeUnits{UInt8, String}, Base.CodeUnits{UInt8, SubString{String}}}
+const ByteString = Union{String,SubString{String}}
+
 # Specified encodings or the combination are not supported by iconv
 struct InvalidEncodingError <: StringEncodingError
     args::Tuple{String, String}
@@ -31,7 +37,7 @@ message(::Type{InvalidEncodingError}) = "Conversion from <<1>> to <<2>> not supp
 struct InvalidSequenceError <: StringEncodingError
     args::Tuple{String}
 end
-InvalidSequenceError(seq::Vector{UInt8}) = InvalidSequenceError((bytes2hex(seq),))
+InvalidSequenceError(seq::AbstractVector{UInt8}) = InvalidSequenceError((bytes2hex(seq),))
 message(::Type{InvalidSequenceError}) = "Byte sequence 0x<<1>> is invalid in source encoding or cannot be represented in target encoding"
 
 struct IConvError <: StringEncodingError
@@ -123,7 +129,7 @@ function finalize(s::Union{StringEncoder, StringDecoder})
     nothing
 end
 
-function iconv!(cd::Ptr{Nothing}, inbuf::Vector{UInt8}, outbuf::Vector{UInt8},
+function iconv!(cd::Ptr{Nothing}, inbuf::ByteVector, outbuf::ByteVector,
                 inbufptr::Ref{Ptr{UInt8}}, outbufptr::Ref{Ptr{UInt8}},
                 inbytesleft::Ref{Csize_t}, outbytesleft::Ref{Csize_t})
     inbufptr[] = pointer(inbuf)
@@ -499,14 +505,20 @@ end
 ## Functions to encode/decode strings
 
 """
-    decode([T,] a::Vector{UInt8}, enc)
+    decode([T,] a::AbstractVector{UInt8}, enc)
 
 Convert an array of bytes `a` representing text in encoding `enc` to a string of type `T`.
 By default, a `String` is returned.
 
+To `decode` an `s::String` of data in non-UTF-8 encoding, use
+`decode(codeunits(s), enc)` to act on the underlying byte array.
+
 `enc` can be specified either as a string or as an `Encoding` object.
+The input data `a` can be a `Vector{UInt8}` of bytes, a contiguous
+subarray thereof, or the `codeunits` of a `String` (or substring
+thereof).
 """
-function decode(::Type{T}, a::Vector{UInt8}, enc::Encoding) where {T<:AbstractString}
+function decode(::Type{T}, a::ByteVector, enc::Encoding) where {T<:AbstractString}
     b = IOBuffer(a)
     try
         T(read(StringDecoder(b, enc, encoding(T))))
@@ -515,19 +527,19 @@ function decode(::Type{T}, a::Vector{UInt8}, enc::Encoding) where {T<:AbstractSt
     end
 end
 
-decode(::Type{T}, a::Vector{UInt8}, enc::AbstractString) where {T<:AbstractString} =
+decode(::Type{T}, a::ByteVector, enc::AbstractString) where {T<:AbstractString} =
     decode(T, a, Encoding(enc))
 
-decode(a::Vector{UInt8}, enc::AbstractString) = decode(String, a, Encoding(enc))
-decode(a::Vector{UInt8}, enc::Union{AbstractString, Encoding}) = decode(String, a, enc)
+decode(a::ByteVector, enc::Union{AbstractString, Encoding}) = decode(String, a, enc)
 
 """
     encode(s::AbstractString, enc)
 
 Convert string `s` to an array of bytes representing text in encoding `enc`.
 `enc` can be specified either as a string or as an `Encoding` object.
 """
-function encode(s::AbstractString, enc::Encoding)
+encode(s::AbstractString, enc::Encoding) = encode(String(s), enc)
+function encode(s::ByteString, enc::Encoding)
     b = IOBuffer()
     p = StringEncoder(b, enc, encoding(typeof(s)))
     write(p, s)

src/encodings.jl

@@ -23,6 +23,7 @@ print(io::IO, ::Encoding{enc}) where {enc} = print(io, enc)
 
 ## Get the encoding used by a string type
 encoding(::Type{String})  = enc"UTF-8"
+encoding(::Type{SubString{String}})  = enc"UTF-8"
 
 encodings_list = ["1026", "1046", "1047", "10646-1:1993", "10646-1:1993/UCS4",
                   "437", "500", "500V1", "850", "851", "852", "855", "856", "857",

test/runtests.jl

@@ -270,6 +270,16 @@ end
     @test_throws ArgumentError readavailable(p)
 end
 
+# make sure encode/decode support various string/array types
+@testset "Array/String types" begin
+    s = "Bendaña"
+    enc = "Windows-1252"
+    se = "Benda\xf1a"
+    @test encode(Test.GenericString(s), enc) == codeunits(se)
+    @test encode(SubString(s, 1:6), enc) == encode(s[1:6], enc) == codeunits(se)[1:6]
+    @test s == decode(codeunits(se), enc) == decode(collect(codeunits(se)), enc)
+    @test s[1:6] == decode(@view(collect(codeunits(se))[1:6]), enc)
+end
 
 ## Test encodings support
 b = IOBuffer()