From ffefb9ea11b9e5c130fa2e5d5fd92ef6f96e9519 Mon Sep 17 00:00:00 2001 From: Sergey Prokhorov Date: Fri, 5 Oct 2012 19:31:32 +0400 Subject: [PATCH 1/4] Now can connect through ssl/not ssl proxy to *not ssl* host --- src/lhttpc_client.erl | 85 ++++++++++++++++++++++++++++++++++--------- src/lhttpc_lib.erl | 46 +++++++++++++++++++++++ 2 files changed, 113 insertions(+), 18 deletions(-) diff --git a/src/lhttpc_client.erl b/src/lhttpc_client.erl index 3a159120..fab91ca3 100644 --- a/src/lhttpc_client.erl +++ b/src/lhttpc_client.erl @@ -124,19 +124,7 @@ execute(From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> PartialDownload = proplists:is_defined(partial_download, Options), PartialDownloadOptions = proplists:get_value(partial_download, Options, []), NormalizedMethod = lhttpc_lib:normalize_method(Method), - Proxy = case proplists:get_value(proxy, Options) of - undefined -> - undefined; - ProxyUrl when is_list(ProxyUrl), not Ssl -> - % The point of HTTP CONNECT proxying is to use TLS tunneled in - % a plain HTTP/1.1 connection to the proxy (RFC2817). - throw(origin_server_not_https); - ProxyUrl when is_list(ProxyUrl) -> - lhttpc_lib:parse_url(ProxyUrl) - end, - {ChunkedUpload, Request} = lhttpc_lib:format_request(Path, NormalizedMethod, - Hdrs, Host, Port, Body, PartialUpload), - %SocketRequest = {socket, self(), Host, Port, Ssl}, + Pool = proplists:get_value(pool, Options, whereis(lhttpc_manager)), %% Get a socket for the pool or exit %Socket = lhttpc_manager:ensure_call(Pool, SocketRequest, Options), @@ -146,7 +134,6 @@ execute(From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> port = Port, ssl = Ssl, method = NormalizedMethod, - request = Request, requester = From, request_headers = Hdrs, socket = Socket, @@ -156,17 +143,18 @@ execute(From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> attempts = 1 + proplists:get_value(send_retry, Options, 1), partial_upload = PartialUpload, upload_window = UploadWindowSize, - chunked_upload = ChunkedUpload, partial_download = PartialDownload, download_window = proplists:get_value(window_size, PartialDownloadOptions, infinity), part_size = proplists:get_value(part_size, PartialDownloadOptions, infinity), - proxy = Proxy, - proxy_setup = (Socket =/= undefined), proxy_ssl_options = proplists:get_value(proxy_ssl_options, Options, []) }, - Response = case send_request(State) of + + State2 = configure_proxy(State, Body, Path, Ssl, + proplists:get_value(proxy, Options)), + + Response = case send_request(State2) of {R, undefined} -> {ok, R}; {R, NewSocket} -> @@ -183,6 +171,67 @@ execute(From, Host, Port, Ssl, Path, Method, Hdrs, Body, Options) -> end, {response, self(), Response}. + +configure_proxy(State, Body, Path, _Ssl, undefined) -> + %% no proxy at all + set_request(State#client_state{proxy=undefined}, Body, Path); +configure_proxy(State, Body, Path, true, ProxyUrl) when is_list(ProxyUrl) -> + %% connect through ssl/not ssl proxy to ssl host using CONNECT + Proxy = lhttpc_lib:parse_url(ProxyUrl), + set_request(State#client_state{ + proxy = Proxy, + proxy_setup = (State#client_state.socket =/= undefined)}, + Body, Path); +configure_proxy(State, Body, Path, false, ProxyUrl) -> + %% connect through ssl/not ssl proxy to not ssl host + %% just replace Path with full host URL + #client_state{request_headers=Hdrs, + host=DestHost, + port=DestPort} = State, + Proxy = lhttpc_lib:parse_url(ProxyUrl), + #lhttpc_url{ + %% host = Host, + %% port = Port, + user = User, + password = Passwd + %% is_ssl = SslProxy + } = Proxy, + NewPath = lhttpc_lib:format_url( + #lhttpc_url{host=DestHost, + port=DestPort, + path=Path, + is_ssl=false}), + Hdrs2 = case User of + "" -> + Hdrs; + User -> + AuthHdr = {"Proxy-Authorization", + "Basic " ++ base64:encode_to_string(User ++ ":" ++ Passwd)}, + [AuthHdr | Hdrs] + end, + set_request(State#client_state{ + %% ssl = SslProxy, % see request_first_destination + %% host = Host, + %% port = Port, + request_headers = Hdrs2, + proxy = Proxy, + proxy_setup = true}, + NewPath, Body). + + +set_request(State, Body, Path) -> + #client_state{partial_upload=PartialUpload, + method=NormalizedMethod, + request_headers=Hdrs, + host=Host, + port=Port} = State, + {ChunkedUpload, Request} = lhttpc_lib:format_request( + Path, NormalizedMethod, Hdrs, Host, Port, Body, PartialUpload), + State#client_state{request=Request, + chunked_upload = ChunkedUpload}. + + + %%------------------------------------------------------------------------------ %% @private %% @doc This function creates a new socket connection if needed, and it also diff --git a/src/lhttpc_lib.erl b/src/lhttpc_lib.erl index a83123b1..ea7084d1 100644 --- a/src/lhttpc_lib.erl +++ b/src/lhttpc_lib.erl @@ -33,6 +33,7 @@ -module(lhttpc_lib). -export([parse_url/1, + format_url/1, format_request/7, header_value/2, header_value/3, normalize_method/1, @@ -113,6 +114,51 @@ maybe_atom_to_list(List) -> List. %%------------------------------------------------------------------------------ +%% @spec (#lhttpc_url{}) -> string() +%% @doc +%% Format parsed URL to string representation. +%% @end +%%------------------------------------------------------------------------------ +format_url(URL) -> + #lhttpc_url{ + host = Host, + port = Port, + path = Path, + user = User, + password = Passwd, + is_ssl = IsSSL + } = URL, + U1 = add_scheme("", IsSSL), + U2 = add_credentials(U1, User, Passwd), + U3 = add_host(U2, Host), + U4 = add_port(U3, Port, IsSSL), + add_path(U4, Path). + +add_scheme(_, true) -> + "https://"; +add_scheme(_, false) -> + "http://". + +add_credentials(Scheme, "", "") -> + Scheme; +add_credentials(Scheme, User, Passwd) -> + Scheme ++ User ++ ":" ++ Passwd ++ "@". + +add_host(SUP, Host) -> + SUP ++ Host. + +add_port(SUPH, 80, false) -> + SUPH; +add_port(SUPH, 443, true) -> + SUPH; +add_port(SUPH, Port, _IsSSL) -> + SUPH ++ ":" ++ integer_to_list(Port). + +add_path(SUPHP, Path) -> + SUPHP ++ Path. + + + %% @spec (URL) -> #lhttpc_url{} %% URL = string() %% @doc From 2e23103e10438e715b18d1491b3f2c745c73747e Mon Sep 17 00:00:00 2001 From: Sergey Prokhorov Date: Fri, 5 Oct 2012 19:47:49 +0400 Subject: [PATCH 2/4] Another typo =) --- src/lhttpc_client.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lhttpc_client.erl b/src/lhttpc_client.erl index fab91ca3..501f50ff 100644 --- a/src/lhttpc_client.erl +++ b/src/lhttpc_client.erl @@ -216,7 +216,7 @@ configure_proxy(State, Body, Path, false, ProxyUrl) -> request_headers = Hdrs2, proxy = Proxy, proxy_setup = true}, - NewPath, Body). + Body, NewPath). set_request(State, Body, Path) -> From 68a1855172265c788caa345f301ebbe60dd73706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B5=D1=80=D0=B3=D0=B5=D0=B9=20=D0=9F=D1=80=D0=BE?= =?UTF-8?q?=D1=85=D0=BE=D1=80=D0=BE=D0=B2?= Date: Sat, 6 Oct 2012 04:07:53 +0400 Subject: [PATCH 3/4] format_url/1 improevements * ipv6 support * use iolist() + lists:flatten/1 for url construction --- src/lhttpc_lib.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lhttpc_lib.erl b/src/lhttpc_lib.erl index ea7084d1..fb34689b 100644 --- a/src/lhttpc_lib.erl +++ b/src/lhttpc_lib.erl @@ -132,7 +132,7 @@ format_url(URL) -> U2 = add_credentials(U1, User, Passwd), U3 = add_host(U2, Host), U4 = add_port(U3, Port, IsSSL), - add_path(U4, Path). + lists:flatten(add_path(U4, Path)). add_scheme(_, true) -> "https://"; @@ -142,20 +142,21 @@ add_scheme(_, false) -> add_credentials(Scheme, "", "") -> Scheme; add_credentials(Scheme, User, Passwd) -> - Scheme ++ User ++ ":" ++ Passwd ++ "@". + [Scheme, User, ":", Passwd, "@"]. add_host(SUP, Host) -> - SUP ++ Host. + Host2 = maybe_ipv6_enclose(Host), + [SUP, Host2]. add_port(SUPH, 80, false) -> SUPH; add_port(SUPH, 443, true) -> SUPH; add_port(SUPH, Port, _IsSSL) -> - SUPH ++ ":" ++ integer_to_list(Port). + [SUPH, ":", integer_to_list(Port)]. add_path(SUPHP, Path) -> - SUPHP ++ Path. + [SUPHP, Path]. From 8967faf7fb8e9c60a90519482d24757f8b2ee59d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B5=D1=80=D0=B3=D0=B5=D0=B9=20=D0=9F=D1=80=D0=BE?= =?UTF-8?q?=D1=85=D0=BE=D1=80=D0=BE=D0=B2?= Date: Sat, 6 Oct 2012 04:36:39 +0400 Subject: [PATCH 4/4] Tests for format_url + fix scheme://user@host case (no password) --- src/lhttpc_lib.erl | 2 + test/lhttpc_lib_tests.erl | 92 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/src/lhttpc_lib.erl b/src/lhttpc_lib.erl index fb34689b..50880186 100644 --- a/src/lhttpc_lib.erl +++ b/src/lhttpc_lib.erl @@ -141,6 +141,8 @@ add_scheme(_, false) -> add_credentials(Scheme, "", "") -> Scheme; +add_credentials(Scheme, User, "") -> + [Scheme, User, "@"]; add_credentials(Scheme, User, Passwd) -> [Scheme, User, ":", Passwd, "@"]. diff --git a/test/lhttpc_lib_tests.erl b/test/lhttpc_lib_tests.erl index c8e135a3..249376cc 100644 --- a/test/lhttpc_lib_tests.erl +++ b/test/lhttpc_lib_tests.erl @@ -222,5 +222,95 @@ parse_url_test_() -> user = "", password = "" }, - lhttpc_lib:parse_url("http://www.example.com?a=b")) + lhttpc_lib:parse_url("http://www.example.com?a=b")), + + %% Serialisation + ?_assertEqual("http://host/", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 80, + path = "/", + is_ssl = false, + user = "", + password = "" + })), + + ?_assertEqual("https://host/", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 443, + path = "/", + is_ssl = true, + user = "", + password = "" + })), + + ?_assertEqual("http://host:180/", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 180, + path = "/", + is_ssl = false, + user = "", + password = "" + })), + + ?_assertEqual("https://host:180/", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 180, + path = "/", + is_ssl = true, + user = "", + password = "" + })), + + ?_assertEqual("http://host:180/path", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 180, + path = "/path", + is_ssl = false, + user = "", + password = "" + })), + + ?_assertEqual("http://user@host:180/path", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 180, + path = "/path", + is_ssl = false, + user = "user", + password = "" + })), + + ?_assertEqual("http://user:pass@host:180/path", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 180, + path = "/path", + is_ssl = false, + user = "user", + password = "pass" + })), + ?_assertEqual("http://:pass@host:180/path", + lhttpc_lib:format_url(#lhttpc_url{ + host = "host", + port = 180, + path = "/path", + is_ssl = false, + user = "", + password = "pass" + })), + + ?_assertEqual("http://user:pass@[::1]:180/path", + lhttpc_lib:format_url(#lhttpc_url{ + host = "::1", + port = 180, + path = "/path", + is_ssl = false, + user = "user", + password = "pass" + })) ].