From fc85d7fd62ddb75902a5c8d1830d0477d7d1c550 Mon Sep 17 00:00:00 2001 From: rick olson Date: Thu, 13 Jun 2019 10:09:44 -0600 Subject: [PATCH 1/9] add build args for faraday gem versions --- docker-compose.yml | 6 +++++- tests/Dockerfile | 5 ++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a225749..cfad640 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,11 @@ services: - ./server/run.sh:/app/run.sh tests: - build: ./tests + build: + context: ./tests + args: + faraday_gem_ref: ${FARADAY_GEM_REF:-master} + faraday_http_gem_ref: ${FARADAY_HTTP_GEM_REF:-master} depends_on: - server - proxy diff --git a/tests/Dockerfile b/tests/Dockerfile index c4be577..1f94265 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -9,13 +9,12 @@ COPY Gemfile.lock . RUN gem install bundler -v '~> 2.0.1' --no-document RUN bundle install --without development test -j4 --retry 3 -ARG faraday_gem_ref=master -ARG faraday_http_gem_ref=master - # install just the non-git gems. These will change more frequently. RUN echo "gem 'faraday', git: 'https://github.com/lostisland/faraday'," >> Gemfile +ARG faraday_gem_ref=master RUN echo " ref: '$faraday_gem_ref', require: 'faraday'" >> Gemfile RUN echo "gem 'faraday-http', git: 'https://github.com/lostisland/faraday-http'," >> Gemfile +ARG faraday_http_gem_ref=master RUN echo " ref: '$faraday_http_gem_ref', require: 'faraday/http'" >> Gemfile RUN bundle install \ From 983be44843fa57d592ff0fe17df4facf71a14f3a Mon Sep 17 00:00:00 2001 From: rick olson Date: Thu, 13 Jun 2019 10:32:23 -0600 Subject: [PATCH 2/9] test socks proxy support in supported faraday adapters --- proxy/Dockerfile | 7 +++++++ proxy/main.go | 22 +++++++++++++++++++++- tests/Gemfile | 1 + tests/spec/insecure_spec.rb | 21 ++++++++++++++++++--- tests/spec/secure_spec.rb | 25 ++++++++++++++++++++----- tests/spec/support/adapters.rb | 3 ++- tests/spec/support/examples/proxy.rb | 7 ++++++- tests/spec/support/protocols.rb | 8 ++++++-- tests/spec/support/urls.rb | 10 ++++++++++ 9 files changed, 91 insertions(+), 13 deletions(-) diff --git a/proxy/Dockerfile b/proxy/Dockerfile index 9900d25..231d35f 100644 --- a/proxy/Dockerfile +++ b/proxy/Dockerfile @@ -1,5 +1,6 @@ FROM golang:1.12.3 RUN go get -u github.com/elazarl/goproxy +RUN go get -u github.com/armon/go-socks5 # no auth, http EXPOSE 8080 @@ -13,6 +14,12 @@ EXPOSE 9080 # auth:pass, https EXPOSE 9080 +# no auth, socks +EXPOSE 6000 + +# auth:pass, socks +EXPOSE 6001 + WORKDIR /app COPY . ./ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o proxy . diff --git a/proxy/main.go b/proxy/main.go index 5db204c..26def33 100644 --- a/proxy/main.go +++ b/proxy/main.go @@ -5,6 +5,8 @@ import ( "flag" "log" "net/http" + + socks5 "github.com/armon/go-socks5" ) var ( @@ -15,6 +17,24 @@ var ( func main() { flag.Parse() + unauthSocks, err := socks5.New(&socks5.Config{}) + if err != nil { + panic(err) + } + + authSocks, err := socks5.New(&socks5.Config{ + Credentials: socks5.StaticCredentials(map[string]string{ + "faraday": "live", + }), + }) + if err != nil { + panic(err) + } + + log.Println("Starting Socks Proxy servers on :6000, :6001...") + go unauthSocks.ListenAndServe("tcp", ":6000") + go authSocks.ListenAndServe("tcp", ":6001") + servers := &ServerList{} servers.Add(":8080", newProxy("http_proxy")) servers.Add(":9080", newProxy("http_auth_proxy")) @@ -39,7 +59,7 @@ func (l *ServerList) Listen() { } func (l *ServerList) listen(srv *http.Server) { - log.Printf("Starting Proxy server on %s...", srv.Addr) + log.Printf("Starting HTTP Proxy server on %s...", srv.Addr) if srv.TLSConfig == nil { srv.ListenAndServe() return diff --git a/tests/Gemfile b/tests/Gemfile index 40fcf8b..c88a10b 100644 --- a/tests/Gemfile +++ b/tests/Gemfile @@ -8,4 +8,5 @@ gem 'net-http-persistent' gem 'patron', '>= 0.4.2', platforms: :ruby gem 'rake' gem 'rspec' +gem 'socksify', '~> 1.7.1' gem 'typhoeus', '~> 1.3' diff --git a/tests/spec/insecure_spec.rb b/tests/spec/insecure_spec.rb index 21385d2..679f860 100644 --- a/tests/spec/insecure_spec.rb +++ b/tests/spec/insecure_spec.rb @@ -10,15 +10,30 @@ proxy: :http_proxy, server: :http, } - end if ServerProtocols.proxy? + end if ServerProtocols.http_proxy? + + describe "#{adapter} using Socks proxy with HTTP server" do + include_examples 'a proxied connection', adapter, { + proxy: :socks_proxy, + server: :http, + } + end if ServerProtocols.socks_proxy? && adapter.socks_proxy? describe "#{adapter} using authenticated HTTP proxy with HTTP server" do include_examples 'a proxied connection', adapter, { proxy: :http_auth_proxy, server: :http, - auth: "faraday:live", + auth: "faraday:live", + } + end if ServerProtocols.http_proxy? + + describe "#{adapter} using authenticated Socks proxy with HTTP server" do + include_examples 'a proxied connection', adapter, { + proxy: :socks_auth_proxy, + server: :http, + auth: "faraday:live", } - end if ServerProtocols.proxy? + end if ServerProtocols.socks_proxy? && adapter.socks_proxy? describe "#{adapter} with unverified HTTPS server" do let(:url_kind) { :https } diff --git a/tests/spec/secure_spec.rb b/tests/spec/secure_spec.rb index fe53cac..feb6d14 100644 --- a/tests/spec/secure_spec.rb +++ b/tests/spec/secure_spec.rb @@ -10,15 +10,30 @@ proxy: :http_proxy, server: :https, } - end if ServerProtocols.proxy? + end if ServerProtocols.http_proxy? - describe "#{adapter} using HTTP proxy with HTTPS server" do + describe "#{adapter} using authenticated HTTP proxy with HTTPS server" do include_examples 'a proxied connection', adapter, { proxy: :http_auth_proxy, server: :https, auth: "faraday:live", } - end if ServerProtocols.proxy? + end if ServerProtocols.http_proxy? + + describe "#{adapter} using Socks proxy with HTTPS server" do + include_examples 'a proxied connection', adapter, { + proxy: :socks_proxy, + server: :https, + } + end if ServerProtocols.socks_proxy? && adapter.socks_proxy? + + describe "#{adapter} using authenticated Socks proxy with HTTPS server" do + include_examples 'a proxied connection', adapter, { + proxy: :socks_auth_proxy, + server: :https, + auth: "faraday:live", + } + end if ServerProtocols.socks_proxy? && adapter.socks_proxy? describe "#{adapter} using HTTPS proxy with HTTPS server" do it "fails to connect" do @@ -29,7 +44,7 @@ end expect { conn.get 'wat' }.to raise_error(Faraday::ConnectionFailed) end - end if ServerProtocols.proxy? && !adapter.https_proxy_bug? + end if ServerProtocols.http_proxy? && !adapter.https_proxy_bug? describe "#{adapter} using authenticated HTTPS proxy with HTTPS server" do it "fails to connect" do @@ -40,5 +55,5 @@ end expect { conn.get 'wat' }.to raise_error(Faraday::ConnectionFailed) end - end if ServerProtocols.proxy? && !adapter.https_proxy_bug? + end if ServerProtocols.http_proxy? && !adapter.https_proxy_bug? end diff --git a/tests/spec/support/adapters.rb b/tests/spec/support/adapters.rb index 07ff6de..a0b8cc6 100644 --- a/tests/spec/support/adapters.rb +++ b/tests/spec/support/adapters.rb @@ -52,6 +52,7 @@ def to_s :connect_with_response_body, # enables CONNECT tests WITH response body :unverified_https_bug, # https://github.com/technoweenie/faraday-live/issues/4 :https_proxy_bug, # https://github.com/technoweenie/faraday-live/issues/6 + :socks_proxy, ].each do |feature| define_method("#{feature}?") { @features.include?(feature) } end @@ -81,7 +82,7 @@ def connect_method? :trace_method, :connect_with_response_body), :net_http => Adapter.new(:net_http, - :trace_method, :connect_with_response_body), + :socks_proxy, :trace_method, :connect_with_response_body), :patron => Adapter.new(:patron, :unverified_https_bug), diff --git a/tests/spec/support/examples/proxy.rb b/tests/spec/support/examples/proxy.rb index 3cd722b..ca4564e 100644 --- a/tests/spec/support/examples/proxy.rb +++ b/tests/spec/support/examples/proxy.rb @@ -25,7 +25,12 @@ end # proxy does not modify https requests - expected = server_url_kind == :https ? '' : "goproxy (#{proxy_url_kind})" + expected = case proxy_url_kind + when :socks_proxy, :socks_auth_proxy + "" + else + server_url_kind == :https ? "" : "goproxy (#{proxy_url_kind})" + end include_examples 'common request tests', server_url_kind, adapter, response_header: { 'Via' => expected, diff --git a/tests/spec/support/protocols.rb b/tests/spec/support/protocols.rb index b14728f..c1ed8d3 100644 --- a/tests/spec/support/protocols.rb +++ b/tests/spec/support/protocols.rb @@ -13,8 +13,12 @@ def self.unverified_https? protocols.include?(:unverified) end - def self.proxy? - protocols.include?(:proxy) + def self.http_proxy? + protocols.include?(:proxy) || protocols.include?(:http_proxy) + end + + def self.socks_proxy? + protocols.include?(:proxy) || protocols.include?(:socks_proxy) end def self.test?(*protos) diff --git a/tests/spec/support/urls.rb b/tests/spec/support/urls.rb index 4cd6ffd..86aa2aa 100644 --- a/tests/spec/support/urls.rb +++ b/tests/spec/support/urls.rb @@ -33,4 +33,14 @@ def self.https_auth_proxy_server(auth = nil) auth += "@" if auth "http://#{auth}#{ENV['PROXY_HOST']}:9443" end + + def self.socks_proxy_server(auth = nil) + auth += "@" if auth + "socks://#{auth}#{ENV['PROXY_HOST']}:6000" + end + + def self.socks_auth_proxy_server(auth = nil) + auth += "@" if auth + "socks://#{auth}#{ENV['PROXY_HOST']}:6001" + end end From 8fc620ec0c4f2c9a66e037cc2d069e399d7c555c Mon Sep 17 00:00:00 2001 From: rick olson Date: Thu, 13 Jun 2019 11:08:44 -0600 Subject: [PATCH 3/9] lint --- tests/Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Gemfile b/tests/Gemfile index c88a10b..eff4ab6 100644 --- a/tests/Gemfile +++ b/tests/Gemfile @@ -8,5 +8,5 @@ gem 'net-http-persistent' gem 'patron', '>= 0.4.2', platforms: :ruby gem 'rake' gem 'rspec' -gem 'socksify', '~> 1.7.1' +gem 'socksify', git: 'https://github.com/astro/socksify-ruby', ref: 'dae610084b4cebd0af92731dbf29f3dd9b9fd9d6' gem 'typhoeus', '~> 1.3' From e8ba3cd776cc214b4c1ae85fd39e51d6a9964ec8 Mon Sep 17 00:00:00 2001 From: rick olson Date: Thu, 13 Jun 2019 11:30:12 -0600 Subject: [PATCH 4/9] re-arrange commands so docker doesn't re-install bundler every time the gemfile changes --- tests/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/Dockerfile b/tests/Dockerfile index 1f94265..87bf934 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -2,11 +2,13 @@ FROM ruby:2.6.2 as build ENV LANG C.UTF-8 WORKDIR /app -COPY Gemfile . -COPY Gemfile.lock . # install all non-git gems RUN gem install bundler -v '~> 2.0.1' --no-document + +COPY Gemfile . +COPY Gemfile.lock . + RUN bundle install --without development test -j4 --retry 3 # install just the non-git gems. These will change more frequently. From c4a731c2e3304466b708cbef57dcec7c710ef413 Mon Sep 17 00:00:00 2001 From: rick olson Date: Thu, 13 Jun 2019 11:30:25 -0600 Subject: [PATCH 5/9] use https://github.com/astro/socksify-ruby/pull/33 --- tests/Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Gemfile b/tests/Gemfile index c88a10b..eff4ab6 100644 --- a/tests/Gemfile +++ b/tests/Gemfile @@ -8,5 +8,5 @@ gem 'net-http-persistent' gem 'patron', '>= 0.4.2', platforms: :ruby gem 'rake' gem 'rspec' -gem 'socksify', '~> 1.7.1' +gem 'socksify', git: 'https://github.com/astro/socksify-ruby', ref: 'dae610084b4cebd0af92731dbf29f3dd9b9fd9d6' gem 'typhoeus', '~> 1.3' From 4cd7cb26ee1b144c4c98e9e6b8dcbfeb2e6ae0c9 Mon Sep 17 00:00:00 2001 From: rick olson Date: Thu, 13 Jun 2019 11:30:39 -0600 Subject: [PATCH 6/9] document new TEST_PROTO options --- docs/dev.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/dev.md b/docs/dev.md index f7b9675..5a93c31 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -38,7 +38,9 @@ This runs tests against these webservers and proxies: 1. HTTP server on port 80 (`TEST_PROTO=http`) 2. Self-signed HTTPS server on port 443 (`TEST_PROTO=unverified`) 3. Verified and valid HTTPS server on port 443 (`TEST_PROTO=https`) -4. Proxy server (`TEST_PROTO=proxy`) +4. HTTP proxy server (`TEST_PROTO=http_proxy`) +5. SOCKS proxy server (`TEST_PROTO=socks_proxy`) +6. Both HTTP and SOCKS proxy servers (`TEST_PROTO=proxy`) You can choose to run one or more explicitly: From 190e2eaec24a3c9254df6c442cd3b02d79941df6 Mon Sep 17 00:00:00 2001 From: risk danger olson Date: Thu, 19 Sep 2019 17:44:25 -0600 Subject: [PATCH 7/9] socks adapter is not checked by default --- docs/dev.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev.md b/docs/dev.md index 5a93c31..186dff0 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -40,7 +40,7 @@ This runs tests against these webservers and proxies: 3. Verified and valid HTTPS server on port 443 (`TEST_PROTO=https`) 4. HTTP proxy server (`TEST_PROTO=http_proxy`) 5. SOCKS proxy server (`TEST_PROTO=socks_proxy`) -6. Both HTTP and SOCKS proxy servers (`TEST_PROTO=proxy`) +6. LEGACY: Only HTTP proxy server (`TEST_PROTO=proxy`) You can choose to run one or more explicitly: From 553628717783acb95563e2e7368416146b983548 Mon Sep 17 00:00:00 2001 From: rick olson Date: Thu, 19 Sep 2019 18:09:23 -0600 Subject: [PATCH 8/9] allow specified socksify gem version for future tests --- docker-compose.yml | 1 + tests/Dockerfile | 5 +++++ tests/Gemfile | 1 - tests/spec/support/protocols.rb | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index cfad640..7183345 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,7 @@ services: args: faraday_gem_ref: ${FARADAY_GEM_REF:-master} faraday_http_gem_ref: ${FARADAY_HTTP_GEM_REF:-master} + socksify_gem_ref: ${FARADAY_SOCKSIFY_GEM_REF:-master} depends_on: - server - proxy diff --git a/tests/Dockerfile b/tests/Dockerfile index 87bf934..d8d1f55 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -15,10 +15,15 @@ RUN bundle install --without development test -j4 --retry 3 RUN echo "gem 'faraday', git: 'https://github.com/lostisland/faraday'," >> Gemfile ARG faraday_gem_ref=master RUN echo " ref: '$faraday_gem_ref', require: 'faraday'" >> Gemfile + RUN echo "gem 'faraday-http', git: 'https://github.com/lostisland/faraday-http'," >> Gemfile ARG faraday_http_gem_ref=master RUN echo " ref: '$faraday_http_gem_ref', require: 'faraday/http'" >> Gemfile +RUN echo "gem 'socksify', git: 'https://github.com/astro/socksify-ruby'," >> Gemfile +ARG socksify_gem_ref=master +RUN echo " ref: '$socksify_gem_ref'" >> Gemfile + RUN bundle install \ # Remove unneeded files (cached *.gem, *.o, *.c) && rm -rf /usr/local/bundle/cache/*.gem \ diff --git a/tests/Gemfile b/tests/Gemfile index eff4ab6..40fcf8b 100644 --- a/tests/Gemfile +++ b/tests/Gemfile @@ -8,5 +8,4 @@ gem 'net-http-persistent' gem 'patron', '>= 0.4.2', platforms: :ruby gem 'rake' gem 'rspec' -gem 'socksify', git: 'https://github.com/astro/socksify-ruby', ref: 'dae610084b4cebd0af92731dbf29f3dd9b9fd9d6' gem 'typhoeus', '~> 1.3' diff --git a/tests/spec/support/protocols.rb b/tests/spec/support/protocols.rb index c1ed8d3..19ea46c 100644 --- a/tests/spec/support/protocols.rb +++ b/tests/spec/support/protocols.rb @@ -18,7 +18,7 @@ def self.http_proxy? end def self.socks_proxy? - protocols.include?(:proxy) || protocols.include?(:socks_proxy) + protocols.include?(:socks_proxy) end def self.test?(*protos) From 41cf319e0c64a4be01261216ad572844c26563a0 Mon Sep 17 00:00:00 2001 From: rick olson Date: Thu, 19 Sep 2019 18:11:30 -0600 Subject: [PATCH 9/9] consistent env var names --- docker-compose.yml | 6 +++--- tests/spec/support/adapters.rb | 2 +- tests/spec/support/http_methods.rb | 2 +- tests/spec/support/protocols.rb | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7183345..e7bac22 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,9 +50,9 @@ services: environment: - HTTP_HOST=faraday-live.localhost - PROXY_HOST=live-proxy.localhost - - FARADAY_ADAPTER=${TEST_ADAPTER:-} - - FARADAY_METHOD=${TEST_METHOD:-} - - SERVER_PROTOCOL=${TEST_PROTO:-} + - TEST_ADAPTER=${TEST_ADAPTER:-} + - TEST_METHOD=${TEST_METHOD:-} + - TEST_PROTO=${TEST_PROTO:-} volumes: - certca:/root/.local/share/mkcert/:ro - certdata:/certs:ro diff --git a/tests/spec/support/adapters.rb b/tests/spec/support/adapters.rb index a0b8cc6..e76c4ee 100644 --- a/tests/spec/support/adapters.rb +++ b/tests/spec/support/adapters.rb @@ -28,7 +28,7 @@ def self.adapter_keys end def self.explicit_adapters - ENV['FARADAY_ADAPTER'].to_s.split(',').map! do |key| + ENV['TEST_ADAPTER'].to_s.split(',').map! do |key| key.strip! key.downcase! key.to_sym diff --git a/tests/spec/support/http_methods.rb b/tests/spec/support/http_methods.rb index 1d22018..c920b35 100644 --- a/tests/spec/support/http_methods.rb +++ b/tests/spec/support/http_methods.rb @@ -29,7 +29,7 @@ def self.http_methods! end def self.explicit_methods - ENV['FARADAY_METHOD'].to_s.split(',').map! do |key| + ENV['TEST_METHOD'].to_s.split(',').map! do |key| key.strip! key.downcase! key.to_sym diff --git a/tests/spec/support/protocols.rb b/tests/spec/support/protocols.rb index 19ea46c..0433a35 100644 --- a/tests/spec/support/protocols.rb +++ b/tests/spec/support/protocols.rb @@ -38,7 +38,7 @@ def self.protocols! end def self.explicit_protocols - ENV['SERVER_PROTOCOL'].to_s.split(',').map! do |key| + ENV['TEST_PROTO'].to_s.split(',').map! do |key| key.strip! key.downcase! key.to_sym