From e1358e498ce17fa0513bf0b8585107f8abe9f084 Mon Sep 17 00:00:00 2001 From: Dmitry Verkhoturov Date: Sun, 23 Mar 2025 00:14:48 +0100 Subject: [PATCH] Update timeout configuration to use time.Duration for consistency --- backend/extractor/pics.go | 2 +- backend/extractor/pics_test.go | 5 +- backend/extractor/readability.go | 2 +- backend/extractor/readability_test.go | 85 ++++++++++++++++++--------- backend/main.go | 2 +- backend/rest/server_test.go | 4 +- 6 files changed, 66 insertions(+), 34 deletions(-) diff --git a/backend/extractor/pics.go b/backend/extractor/pics.go index 7117dc7f..1b51b3f3 100644 --- a/backend/extractor/pics.go +++ b/backend/extractor/pics.go @@ -59,7 +59,7 @@ func (f *UReadability) extractPics(iselect *goquery.Selection, url string) (main // getImageSize loads image to get size func (f *UReadability) getImageSize(url string) (size int) { - httpClient := &http.Client{Timeout: time.Second * 30} + httpClient := &http.Client{Timeout: 30 * time.Second} req, err := http.NewRequest("GET", url, nil) if err != nil { log.Printf("[WARN] can't create request to get pic from %s", url) diff --git a/backend/extractor/pics_test.go b/backend/extractor/pics_test.go index 282be116..294646b0 100644 --- a/backend/extractor/pics_test.go +++ b/backend/extractor/pics_test.go @@ -9,6 +9,7 @@ import ( "os" "strings" "testing" + "time" "github.com/PuerkitoBio/goquery" "github.com/stretchr/testify/assert" @@ -27,7 +28,7 @@ func TestExtractPics(t *testing.T) { defer ts.Close() t.Log("test main pic") - lr := UReadability{TimeOut: 30, SnippetSize: 200} + lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200} a, err := lr.Extract(context.Background(), ts.URL+"/2015/09/25/poiezdka-s-apple-maps/") require.NoError(t, err) allImages := []string{ @@ -41,7 +42,7 @@ func TestExtractPics(t *testing.T) { func TestExtractPicsDirectly(t *testing.T) { t.Log("test pic directly") - lr := UReadability{TimeOut: 30, SnippetSize: 200} + lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200} t.Run("normal image retrieval", func(t *testing.T) { data := ` View Page Source` diff --git a/backend/extractor/readability.go b/backend/extractor/readability.go index d837f484..daf567d5 100644 --- a/backend/extractor/readability.go +++ b/backend/extractor/readability.go @@ -73,7 +73,7 @@ func (f *UReadability) extractWithRules(ctx context.Context, reqURL string, rule log.Printf("[INFO] extract %s", reqURL) rb := &Response{} - httpClient := &http.Client{Timeout: time.Second * f.TimeOut} + httpClient := &http.Client{Timeout: f.TimeOut} req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) if err != nil { log.Printf("[WARN] failed to create request for %s, error=%v", reqURL, err) diff --git a/backend/extractor/readability_test.go b/backend/extractor/readability_test.go index c4a79c05..e9f57150 100644 --- a/backend/extractor/readability_test.go +++ b/backend/extractor/readability_test.go @@ -40,29 +40,60 @@ func TestExtractURL(t *testing.T) { })) defer ts.Close() - lr := UReadability{TimeOut: 30, SnippetSize: 200} - t.Log("full url") - rb, err := lr.Extract(context.Background(), ts.URL+"/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/") - assert.NoError(t, err) - assert.Equal(t, ts.URL+"/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/", rb.URL, "not changed") - assert.Equal(t, "Всем миром для общей пользы • Umputun тут был", rb.Title) - assert.Equal(t, 9665, len(rb.Content)) + lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200} + + tests := []struct { + name string + url string + wantURL string + wantTitle string + wantContentLen int + wantErr bool + }{ + { + name: "full url", + url: ts.URL + "/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/", + wantURL: ts.URL + "/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/", + wantTitle: "Всем миром для общей пользы • Umputun тут был", + wantContentLen: 9665, + wantErr: false, + }, + { + name: "short url", + url: ts.URL + "/IAvTHr", + wantURL: ts.URL + "/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/", + wantTitle: "Всем миром для общей пользы • Umputun тут был", + wantContentLen: 9665, + wantErr: false, + }, + { + name: "bad body", + url: ts.URL + "/bad_body", + wantErr: true, + }, + { + name: "bad url", + url: "http://bad_url", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rb, err := lr.Extract(context.Background(), tt.url) + + if tt.wantErr { + assert.Error(t, err) + assert.Nil(t, rb) + return + } - t.Log("short url") - rb, err = lr.Extract(context.Background(), ts.URL+"/IAvTHr") - assert.NoError(t, err) - assert.Equal(t, ts.URL+"/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/", rb.URL, "full url") - assert.Equal(t, 9665, len(rb.Content)) - - t.Log("bad body") - rb, err = lr.Extract(context.Background(), ts.URL+"/bad_body") - //assert.Error(t, err) // TODO: uncomment, wtf?! this should return error! - assert.Nil(t, rb) - - t.Log("bad url") - rb, err = lr.Extract(context.Background(), "http://bad_url") - assert.Error(t, err) - assert.Nil(t, rb) + assert.NoError(t, err) + assert.Equal(t, tt.wantURL, rb.URL) + assert.Equal(t, tt.wantTitle, rb.Title) + assert.Equal(t, tt.wantContentLen, len(rb.Content)) + }) + } } func TestExtractGeneral(t *testing.T) { @@ -92,7 +123,7 @@ func TestExtractGeneral(t *testing.T) { })) defer ts.Close() - lr := UReadability{TimeOut: 30, SnippetSize: 200} + lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200} a, err := lr.Extract(context.Background(), ts.URL+"/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/") assert.NoError(t, err) assert.Equal(t, "Всем миром для общей пользы • Umputun тут был", a.Title) @@ -114,7 +145,7 @@ func TestExtractGeneral(t *testing.T) { } func TestNormalizeLinks(t *testing.T) { - lr := UReadability{TimeOut: 30, SnippetSize: 200} + lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200} inp := `blah sdfasd something blah33 xx` u, _ := url.Parse("http://ukeeper.com/blah") out, links := lr.normalizeLinks(inp, &http.Request{URL: u}) @@ -130,7 +161,7 @@ func TestNormalizeLinks(t *testing.T) { } func TestNormalizeLinksIssue(t *testing.T) { - lr := UReadability{TimeOut: 30, SnippetSize: 200} + lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200} _, err := lr.Extract(context.Background(), "https://git-scm.com/book/en/v2/Git-Tools-Submodules") assert.NoError(t, err) } @@ -150,8 +181,8 @@ func (m RulesMock) Disable(_ context.Context, _ primitive.ObjectID) error { retu func (m RulesMock) All(_ context.Context) []datastore.Rule { return make([]datastore.Rule, 0) } func TestGetContentCustom(t *testing.T) { - lr := UReadability{TimeOut: 30, SnippetSize: 200, Rules: RulesMock{}} - httpClient := &http.Client{Timeout: time.Second * 30} + lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200, Rules: RulesMock{}} + httpClient := &http.Client{Timeout: 30 * time.Second} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.String() == "/2015/09/25/poiezdka-s-apple-maps/" { fh, err := os.Open("testdata/poiezdka-s-apple-maps.html") diff --git a/backend/main.go b/backend/main.go index 68bc3bf3..e8a6ef03 100644 --- a/backend/main.go +++ b/backend/main.go @@ -41,7 +41,7 @@ func main() { log.Fatalf("[ERROR] can't connect to mongo %v", err) } srv := rest.Server{ - Readability: extractor.UReadability{TimeOut: 30, SnippetSize: 300, Rules: db.GetStores()}, + Readability: extractor.UReadability{TimeOut: 30 * time.Second, SnippetSize: 300, Rules: db.GetStores()}, Token: opts.Token, Credentials: opts.Credentials, Version: revision, diff --git a/backend/rest/server_test.go b/backend/rest/server_test.go index 4ed659f1..bb6aa8f7 100644 --- a/backend/rest/server_test.go +++ b/backend/rest/server_test.go @@ -36,7 +36,7 @@ func TestServer_FileServer(t *testing.T) { require.NoError(t, err) srv := Server{ - Readability: extractor.UReadability{TimeOut: 30, SnippetSize: 300}, + Readability: extractor.UReadability{TimeOut: 30 * time.Second, SnippetSize: 300}, Credentials: map[string]string{"admin": "password"}, } ts := httptest.NewServer(srv.routes(dir)) @@ -613,7 +613,7 @@ func startupT(t *testing.T) (*httptest.Server, *Server) { db, err := datastore.New("mongodb://localhost:27017/", "test_ureadability", 0) assert.NoError(t, err) srv := Server{ - Readability: extractor.UReadability{TimeOut: 30, SnippetSize: 300, Rules: db.GetStores()}, + Readability: extractor.UReadability{TimeOut: 30 * time.Second, SnippetSize: 300, Rules: db.GetStores()}, Credentials: map[string]string{"admin": "password"}, Version: "dev-test", }