Merge pull request #139 from antham/add-proxy-url

Add the ability to customize HTTP client
antham · Dec 27, 2023 · 7de8fdf · 7de8fdf
2 parents c5d6164 72352dd
commit 7de8fdf
Show file tree

Hide file tree

Showing 5 changed files with 130 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -36,6  36,17 @@ Use "yogo [command] --help" for more information about a command.
 
 ⚠️ Performing too much calls will trigger a CAPTCHA that you will need to solve through a browser. Add a delay to prevent this.
 
 ## Environment variable
 
 You can customize the behaviour of Yogo through several environment variables:
 
 | Name                   | Default value                                                                                                          | Usage                                             |
 |------------------------|------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------|
 | `HTTP_PROXY`           | Empty                                                                                                                  | Define an HTTP proxy for the requests             |
 | `HTTPS_PROXY`          | Empty                                                                                                                  | Define an HTTPs proxy for the requests            |
 | `YOGO_USER_AGENT`      | See the `defaultUserAgent` const in the [client](https://github.com/antham/yogo/blob/master/internal/client/client.go) | The user agent used to perfom the requests        |
 | `YOGO_REQUEST_TIMEOUT` | 10s                                                                                                                    | Duration of a request before reaching the timeout |
 
 ## Flag
 
 Use the `--json` output flag to get the output as JSON.

diff --git a/go.mod b/go.mod
@@ -8,6  8,7 @@ require (
 	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.8.4
 	golang.org/x/net v0.17.0
 	golang.org/x/term v0.15.0
 )
 
@@ -24,8  25,8 @@ require (
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
-	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

diff --git a/go.sum b/go.sum
@@ -87,6  87,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u 2 /6zg i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz aE=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b 2E5dAYhXwXZwtnZ6UAqBI28 e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs Hm0VOH/i9J2 nxYbc=

diff --git a/internal/client/client.go b/internal/client/client.go
@@ -8,19  8,22 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
 	"os"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/google/uuid"
 	"golang.org/x/net/http/httpproxy"
 )
 
 var ErrCaptcha = errors.New("failure when trying to access content: a CAPTCHA is probably activated, look to the web interface")
 
 const refURL = "https://yopmail.com"
-const defaultHttpTimeout = 10
 const defaultRequestTimeout = 10
 const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
 
 type mailKind string
 
@@ -179,12  182,17 @@ func (c Client[M]) decorateURL(URL string, apiVersion string, disableDefaultQuer
 }
 
 type browser struct {
-	cookies         map[string]string
-	enableDebugMode bool
 	cookies           map[string]string
 	enableDebugMode   bool
 	httpClientFactory httpClientFactory
 }
 
 func newBrowser(enableDebugMode bool) *browser {
-	return &browser{cookies: map[string]string{}, enableDebugMode: enableDebugMode}
 	return &browser{
 		cookies:           map[string]string{},
 		enableDebugMode:   enableDebugMode,
 		httpClientFactory: httpClientFactory{},
 	}
 }
 
 func (b *browser) setCookie(key string, value string) {
@@ -219,7  227,11 @@ func (b *browser) fetch(method string, URL string, headers map[string]string, bo
 	if len(b.cookies) > 0 {
 		r.Header.Add("Cookie", b.buildCookie())
 	}
-	r.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36")
 	userAgent := os.Getenv("YOGO_USER_AGENT")
 	if userAgent == "" {
 		userAgent = defaultUserAgent
 	}
 	r.Header.Add("User-Agent", userAgent)
 	if b.enableDebugMode {
 		buff, err := httputil.DumpRequest(r, true)
 		if err != nil {
@@ -230,7  242,10 @@ func (b *browser) fetch(method string, URL string, headers map[string]string, bo
 		fmt.Println("------------------------------------------------------")
 	}
 
-	c := http.Client{Timeout: defaultHttpTimeout * time.Second}
 	c, err := b.httpClientFactory.create()
 	if err != nil {
 		return nil, wrapError(errMsg, err)
 	}
 	res, err := c.Do(r)
 	if err != nil {
 		return nil, wrapError(errMsg, err)
@@ -297,3  312,35 @@ func checkMailCAPTCHA(content string) error {
 	}
 	return nil
 }
 
 type httpClientFactory struct{}
 
 func (h httpClientFactory) create() (*http.Client, error) {
 	timeout := defaultRequestTimeout * time.Second
 	if os.Getenv("YOGO_REQUEST_TIMEOUT") != "" {
 		t, err := strconv.Atoi(os.Getenv("YOGO_REQUEST_TIMEOUT"))
 		if err != nil {
 			return nil, err
 		}
 		timeout = time.Duration(t) * time.Second
 	}
 	client := &http.Client{Timeout: timeout}
 	config := httpproxy.FromEnvironment()
 	URL := ""
 	switch {
 	case config.HTTPProxy != "":
 		URL = config.HTTPProxy
 	case config.HTTPSProxy != "":
 		URL = config.HTTPSProxy
 	}
 	if URL != "" {
 		u, err := url.Parse(URL)
 		if err != nil {
 			return nil, err
 		}
 		transport := &http.Transport{}
 		transport.Proxy = http.ProxyURL(u)
 		client.Transport = transport
 	}
 	return client, nil
 }
diff --git a/internal/client/client_test.go b/internal/client/client_test.go
@@ -4,6  4,7 @@ import (
 	"errors"
 	"io"
 	"net/http"
 	"os"
 	"testing"
 	"time"
 
@@ -513,6  514,67 @@ func TestFlushMail(t *testing.T) {
 	}
 }
 
 func TestHTTPClientFactoryCreate(t *testing.T) {
 	type scenario struct {
 		name  string
 		setup func()
 		test  func(*http.Client, error)
 	}
 
 	for _, s := range []scenario{{
 		"0 values",
 		func() {
 		},
 		func(c *http.Client, err error) {
 			assert.NoError(t, err)
 		},
 	}, {
 		"Define a timeout alone",
 		func() {
 			os.Setenv("YOGO_REQUEST_TIMEOUT", "10")
 		},
 		func(c *http.Client, err error) {
 			assert.NoError(t, err)
 			assert.Equal(t, time.Second*10, c.Timeout)
 		},
 	}, {
 		"Define an HTTP proxy URL",
 		func() {
 			os.Setenv("HTTP_PROXY", "http://localhost:8000")
 		},
 		func(c *http.Client, err error) {
 			assert.NoError(t, err)
 		},
 	}, {
 		"Define an HTTPs proxy URL",
 		func() {
 			os.Setenv("HTTPS_PROXY", "http://localhost:8000")
 		},
 		func(c *http.Client, err error) {
 			assert.NoError(t, err)
 		},
 	}, {
 		"Define a wrong proxy URL",
 		func() {
 			os.Setenv("HTTP_PROXY", "l\n")
 		},
 		func(c *http.Client, err error) {
 			assert.Error(t, err)
 			assert.EqualError(t, err, `parse "l\n": net/url: invalid control character in URL`)
 		},
 	}} {
 		t.Run(s.name, func(t *testing.T) {
 			os.Setenv("HTTP_PROXY", "")
 			os.Setenv("HTTPS_PROXY", "")
 			os.Setenv("YOGO_REQUEST_TIMEOUT", "")
 			s.setup()
 			h := httpClientFactory{}
 			c, err := h.create()
 			s.test(c, err)
 		})
 	}
 }
 
 func mockYopmailSetup() {
 	httpmock.RegisterResponder("GET", refURL "/ver/3.1/webmail.js",
 		httpmock.NewStringResponder(200, "xxx http://whatever.com?q=s&yj=ytest&t=a xxxxx"))