diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fa1c497..642b494f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: ENABLE_MONGO_TESTS: "true" - name: golangci-lint - uses: golangci/golangci-lint-action@v6 + uses: golangci/golangci-lint-action@v7 with: version: latest working-directory: backend diff --git a/backend/.golangci.yml b/backend/.golangci.yml index ac6c4e9f..0264a282 100644 --- a/backend/.golangci.yml +++ b/backend/.golangci.yml @@ -1,7 +1,8 @@ +version: "2" run: tests: false - timeout: 5m linters: + default: none enable: - bodyclose - copyloopvar @@ -9,29 +10,74 @@ linters: - dupl - errcheck - gochecknoinits + - gocognit - goconst - gocritic - gocyclo - - gofmt - - goimports - goprintffuncname - gosec - - gosimple - govet - ineffassign - misspell - nakedret + - nolintlint + - prealloc - revive - rowserrcheck - staticcheck - - stylecheck - - typecheck + - testifylint - unconvert - unparam - unused - whitespace - disable-all: true - -issues: - exclude-dirs: - - vendor + settings: + goconst: + min-len: 2 + min-occurrences: 2 + revive: + enable-all-rules: true + rules: + - name: unused-receiver + disabled: true + - name: line-length-limit + disabled: true + - name: add-constant + disabled: true + - name: cognitive-complexity + disabled: true + - name: function-length + disabled: true + - name: cyclomatic + disabled: true + - name: nested-structs + disabled: true + gocritic: + disabled-checks: + - hugeParam + enabled-tags: + - performance + - style + - experimental + govet: + enable: + - shadow + lll: + line-length: 140 + misspell: + locale: US + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ \ No newline at end of file diff --git a/backend/datastore/mongo_test.go b/backend/datastore/mongo_test.go index 977ba3cd..ada5b178 100644 --- a/backend/datastore/mongo_test.go +++ b/backend/datastore/mongo_test.go @@ -24,9 +24,9 @@ func TestMongoCreation(t *testing.T) { func TestWrongConnectionString(t *testing.T) { server, err := New("wrong", "test_ureadability", time.Millisecond*100) - assert.Error(t, err) + require.Error(t, err) assert.Nil(t, server) server, err = New("", "", time.Millisecond*100) - assert.Error(t, err) + require.Error(t, err) assert.Nil(t, server) } diff --git a/backend/datastore/rules.go b/backend/datastore/rules.go index eb56ce22..c6bc5721 100644 --- a/backend/datastore/rules.go +++ b/backend/datastore/rules.go @@ -42,7 +42,7 @@ func (r RulesDAO) Get(ctx context.Context, rURL string) (Rule, bool) { var rules []Rule q := bson.M{"domain": u.Host, "enabled": true} log.Printf("[DEBUG] query %v", q) - cursor, err := r.Collection.Find(ctx, q) + cursor, err := r.Find(ctx, q) if err != nil { log.Printf("[DEBUG] error looking for rules for %s", rURL) return Rule{}, false @@ -65,13 +65,15 @@ func (r RulesDAO) GetByID(ctx context.Context, id primitive.ObjectID) (Rule, boo // Save upsert rule func (r RulesDAO) Save(ctx context.Context, rule Rule) (Rule, error) { - ch, err := r.Collection.UpdateOne(ctx, bson.M{"domain": rule.Domain}, bson.M{"$set": rule}, options.Update().SetUpsert(true)) + ch, err := r.UpdateOne(ctx, bson.M{"domain": rule.Domain}, bson.M{"$set": rule}, options.Update().SetUpsert(true)) if err != nil { log.Printf("[WARN] failed to save, error=%v, article=%v", err, rule) return rule, err } if ch.UpsertedID != nil { - rule.ID = ch.UpsertedID.(primitive.ObjectID) + if oid, ok := ch.UpsertedID.(primitive.ObjectID); ok { + rule.ID = oid + } } // if rule was updated, we have no id, so try to find it by domain if rule.ID == primitive.NilObjectID { @@ -86,13 +88,13 @@ func (r RulesDAO) Save(ctx context.Context, rule Rule) (Rule, error) { // Disable marks enabled=false, by id func (r RulesDAO) Disable(ctx context.Context, id primitive.ObjectID) error { - _, err := r.Collection.UpdateOne(ctx, bson.M{"_id": id}, bson.M{"$set": bson.M{"enabled": false}}) + _, err := r.UpdateOne(ctx, bson.M{"_id": id}, bson.M{"$set": bson.M{"enabled": false}}) return err } // All returns list of all rules, both enabled and disabled func (r RulesDAO) All(ctx context.Context) []Rule { - cursor, err := r.Collection.Find(ctx, bson.M{}) + cursor, err := r.Find(ctx, bson.M{}) if err != nil { log.Printf("[WARN] failed to retrieve all rules, error=%v", err) return []Rule{} diff --git a/backend/datastore/rules_test.go b/backend/datastore/rules_test.go index 8f65558b..b2b444b8 100644 --- a/backend/datastore/rules_test.go +++ b/backend/datastore/rules_test.go @@ -28,7 +28,7 @@ func TestRules(t *testing.T) { // save a rule srule, err := rules.Save(context.Background(), rule) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, rule.Domain, srule.Domain) ruleID := srule.ID @@ -46,7 +46,7 @@ func TestRules(t *testing.T) { // disable the rule err = rules.Disable(context.Background(), grule.ID) - assert.NoError(t, err) + require.NoError(t, err) assert.NotContains(t, rules.All(context.Background()), grule) // get the rule by ID, should be marked as disabled @@ -64,7 +64,7 @@ func TestRules(t *testing.T) { // save a rule once more, should result in the same ID updatedRule, err := rules.Save(context.Background(), rule) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, rule.Domain, updatedRule.Domain) assert.Equal(t, ruleID, updatedRule.ID) } @@ -84,7 +84,7 @@ func TestRulesCanceledContext(t *testing.T) { rule := Rule{Domain: "example.com", Enabled: true} srule, err := rules.Save(ctx, rule) assert.Equal(t, rule, srule) - assert.Error(t, err) + require.Error(t, err) // retrieve a rule, wrong rule grule, found := rules.Get(context.Background(), "http://user^:passwo^rd@foo.com/") @@ -95,7 +95,7 @@ func TestRulesCanceledContext(t *testing.T) { assert.Empty(t, grule, "canceled context") assert.False(t, found, "canceled context") assert.Empty(t, rules.All(ctx)) - assert.Error(t, rules.Disable(ctx, rule.ID)) + require.Error(t, rules.Disable(ctx, rule.ID)) // get a rule by ID with canceled context grule, found = rules.GetByID(ctx, rule.ID) assert.Empty(t, grule) diff --git a/backend/extractor/pics.go b/backend/extractor/pics.go index 1b51b3f3..f066442e 100644 --- a/backend/extractor/pics.go +++ b/backend/extractor/pics.go @@ -60,7 +60,7 @@ func (f *UReadability) extractPics(iselect *goquery.Selection, url string) (main // getImageSize loads image to get size func (f *UReadability) getImageSize(url string) (size int) { httpClient := &http.Client{Timeout: 30 * time.Second} - req, err := http.NewRequest("GET", url, nil) + req, err := http.NewRequest("GET", url, http.NoBody) if err != nil { log.Printf("[WARN] can't create request to get pic from %s", url) return 0 diff --git a/backend/extractor/pics_test.go b/backend/extractor/pics_test.go index 294646b0..3a427ef1 100644 --- a/backend/extractor/pics_test.go +++ b/backend/extractor/pics_test.go @@ -20,10 +20,10 @@ func TestExtractPics(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fh, err := os.Open("testdata/poiezdka-s-apple-maps.html") testHTML, err := io.ReadAll(fh) - require.NoError(t, err) - require.NoError(t, fh.Close()) + assert.NoError(t, err) + assert.NoError(t, fh.Close()) _, err = w.Write(testHTML) - require.NoError(t, err) + assert.NoError(t, err) })) defer ts.Close() @@ -51,7 +51,7 @@ func TestExtractPicsDirectly(t *testing.T) { sel := d.Find("img") im, allImages, ok := lr.extractPics(sel, "url") assert.True(t, ok) - assert.Equal(t, 1, len(allImages)) + assert.Len(t, allImages, 1) assert.Equal(t, "https://cdn1.tnwcdn.com/wp-content/blogs.dir/1/files/2016/01/page-source.jpg", im) }) @@ -73,7 +73,7 @@ func TestExtractPicsDirectly(t *testing.T) { sel := d.Find("img") im, allImages, ok := lr.extractPics(sel, "url") assert.True(t, ok) - assert.Equal(t, 1, len(allImages)) + assert.Len(t, allImages, 1) assert.Equal(t, "http://bad_url", im) }) @@ -82,13 +82,13 @@ func TestExtractPicsDirectly(t *testing.T) { w.Header().Set("Content-Length", "1") // error on reading body })) defer ts.Close() - data := fmt.Sprintf(``, ts.URL) + data := fmt.Sprintf(``, ts.URL) d, err := goquery.NewDocumentFromReader(strings.NewReader(data)) require.NoError(t, err) sel := d.Find("img") im, allImages, ok := lr.extractPics(sel, "url") assert.True(t, ok) - assert.Equal(t, 1, len(allImages)) + assert.Len(t, allImages, 1) assert.Equal(t, ts.URL, im) }) } diff --git a/backend/extractor/readability.go b/backend/extractor/readability.go index daf567d5..b8cf954a 100644 --- a/backend/extractor/readability.go +++ b/backend/extractor/readability.go @@ -74,7 +74,7 @@ func (f *UReadability) extractWithRules(ctx context.Context, reqURL string, rule rb := &Response{} httpClient := &http.Client{Timeout: f.TimeOut} - req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) + req, err := http.NewRequestWithContext(ctx, "GET", reqURL, http.NoBody) if err != nil { log.Printf("[WARN] failed to create request for %s, error=%v", reqURL, err) return nil, err @@ -180,7 +180,7 @@ func (f *UReadability) getContent(ctx context.Context, body, reqURL string, rule r := f.Rules if rule, found := r.Get(ctx, reqURL); found { if content, rich, err = customParser(body, reqURL, rule); err == nil { - return content, rich, err + return content, rich, nil } log.Printf("[WARN] custom extractor failed for %s, error=%v", reqURL, err) // back to general parser } @@ -208,8 +208,8 @@ func (f *UReadability) normalizeLinks(data string, reqContext *http.Request) (re dstLink := srcLink if absLink, changed := absoluteLink(srcLink); changed { dstLink = absLink - srcLink = fmt.Sprintf(`"%s"`, srcLink) - absLink = fmt.Sprintf(`"%s"`, absLink) + srcLink = fmt.Sprintf("%q", srcLink) + absLink = fmt.Sprintf("%q", absLink) result = strings.ReplaceAll(result, srcLink, absLink) log.Printf("[DEBUG] normalized %s -> %s", srcLink, dstLink) normalizedCount++ diff --git a/backend/extractor/readability_test.go b/backend/extractor/readability_test.go index e9f57150..8771c54d 100644 --- a/backend/extractor/readability_test.go +++ b/backend/extractor/readability_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.mongodb.org/mongo-driver/bson/primitive" "github.com/ukeeper/ukeeper-redabilty/backend/datastore" @@ -83,15 +84,15 @@ func TestExtractURL(t *testing.T) { rb, err := lr.Extract(context.Background(), tt.url) if tt.wantErr { - assert.Error(t, err) + require.Error(t, err) assert.Nil(t, rb) return } - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, tt.wantURL, rb.URL) assert.Equal(t, tt.wantTitle, rb.Title) - assert.Equal(t, tt.wantContentLen, len(rb.Content)) + assert.Len(t, rb.Content, tt.wantContentLen) }) } } @@ -125,20 +126,20 @@ func TestExtractGeneral(t *testing.T) { lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200} a, err := lr.Extract(context.Background(), ts.URL+"/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/") - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "Всем миром для общей пользы • Umputun тут был", a.Title) assert.Equal(t, ts.URL+"/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/", a.URL) assert.Equal(t, "Не первый раз я практикую идею “а давайте, ребята, сделаем для общего блага …”, и вот опять. В нашем подкасте радио-т есть незаменимый инструмент, позволяющий собирать новости, готовить их к выпуску, ...", a.Excerpt) assert.Contains(t, ts.URL, a.Domain) a, err = lr.Extract(context.Background(), ts.URL+"/v48b6Q") - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "UWP - Выпуск 369", a.Title) assert.Equal(t, ts.URL+"/p/2015/11/22/podcast-369/", a.URL) assert.Equal(t, "2015-11-22 Нагло ходил в гости. Табличка на двери сработала на 50%Никогда нас школа не хвалила. Девочка осваивает новый прибор. Мое неприятие их логики. И разошлись по будкам …Отбиваюсь от опасных ...", a.Excerpt) assert.Equal(t, "https://podcast.umputun.com/images/uwp/uwp369.jpg", a.Image) assert.Contains(t, ts.URL, a.Domain) - assert.Equal(t, 13, len(a.AllLinks)) + assert.Len(t, a.AllLinks, 13) assert.Contains(t, a.AllLinks, "https://podcast.umputun.com/media/ump_podcast369.mp3") assert.Contains(t, a.AllLinks, "https://podcast.umputun.com/images/uwp/uwp369.jpg") log.Printf("links=%v", a.AllLinks) @@ -150,20 +151,19 @@ func TestNormalizeLinks(t *testing.T) { u, _ := url.Parse("http://ukeeper.com/blah") out, links := lr.normalizeLinks(inp, &http.Request{URL: u}) assert.Equal(t, `blah sdfasd something blah33 xx`, out) - assert.Equal(t, 3, len(links)) + assert.Len(t, links, 3) inp = ` View Page Source` _, links = lr.normalizeLinks(inp, &http.Request{URL: u}) - assert.Equal(t, 1, len(links)) + assert.Len(t, links, 1) assert.Equal(t, "http://cdn1.tnwcdn.com/wp-content/blogs.dir/1/files/2016/01/page-source.jpg", links[0]) - } func TestNormalizeLinksIssue(t *testing.T) { lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200} _, err := lr.Extract(context.Background(), "https://git-scm.com/book/en/v2/Git-Tools-Submodules") - assert.NoError(t, err) + require.NoError(t, err) } type RulesMock struct{} @@ -196,14 +196,14 @@ func TestGetContentCustom(t *testing.T) { })) defer ts.Close() resp, err := httpClient.Get(ts.URL + "/2015/09/25/poiezdka-s-apple-maps/") - assert.NoError(t, err) + require.NoError(t, err) defer resp.Body.Close() dataBytes, err := io.ReadAll(resp.Body) - assert.NoError(t, err) + require.NoError(t, err) body := string(dataBytes) content, rich, err := lr.getContent(context.Background(), body, ts.URL+"/2015/09/25/poiezdka-s-apple-maps/", nil) - assert.NoError(t, err) - assert.Equal(t, 6988, len(content)) - assert.Equal(t, 7169, len(rich)) + require.NoError(t, err) + assert.Len(t, content, 6988) + assert.Len(t, rich, 7169) } diff --git a/backend/extractor/text.go b/backend/extractor/text.go index ba21cefe..cf3c288c 100644 --- a/backend/extractor/text.go +++ b/backend/extractor/text.go @@ -11,6 +11,13 @@ import ( "golang.org/x/net/html/charset" ) +const ( + // DefaultContentType is the default content type for HTML documents + DefaultContentType = "text/html" + // DefaultEncoding is the default character encoding for HTML documents + DefaultEncoding = "utf-8" +) + // get clean text from html content func (f *UReadability) getText(content, title string) string { cleanText := sanitize.HTML(content) @@ -63,8 +70,8 @@ func (f *UReadability) toUtf8(content []byte, header http.Header) (contentType, body := string(content) result = body - contentType = "text/html" - origEncoding = "utf-8" + contentType = DefaultContentType + origEncoding = DefaultEncoding if h := header.Get("Content-Type"); h != "" { contentType, origEncoding = getContentTypeAndEncoding(h) diff --git a/backend/main.go b/backend/main.go index e8a6ef03..b4a38109 100644 --- a/backend/main.go +++ b/backend/main.go @@ -1,3 +1,4 @@ +// Package main is the entry point for the ukeeper-readability service package main import ( @@ -33,7 +34,13 @@ func main() { if _, err := flags.Parse(&opts); err != nil { os.Exit(1) } - setupLog(opts.Debug) + // Setup logging + var options []log.Option + if opts.Debug { + options = []log.Option{log.Debug, log.CallerFile} + } + options = append(options, log.Msec, log.LevelBraces) + log.Setup(options...) log.Printf("[INFO] started ukeeper-readability service %s", revision) db, err := datastore.New(opts.MongoURI, opts.MongoDB, opts.MongoDelay) @@ -58,11 +65,3 @@ func main() { srv.Run(ctx, opts.Address, opts.Port, opts.FrontendDir) } - -func setupLog(dbg bool) { - if dbg { - log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces) - return - } - log.Setup(log.Msec, log.LevelBraces) -} diff --git a/backend/main_test.go b/backend/main_test.go index 33bc6dcd..6697d8ae 100644 --- a/backend/main_test.go +++ b/backend/main_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_Main(t *testing.T) { @@ -48,11 +49,11 @@ func Test_Main(t *testing.T) { waitForHTTPServerStart(port) resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/ping", port)) - assert.NoError(t, err) + require.NoError(t, err) defer resp.Body.Close() assert.Equal(t, 200, resp.StatusCode) body, err := io.ReadAll(resp.Body) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "pong", string(body)) } diff --git a/backend/rest/server.go b/backend/rest/server.go index 72c39610..9d64aeca 100644 --- a/backend/rest/server.go +++ b/backend/rest/server.go @@ -18,7 +18,7 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/render" log "github.com/go-pkgz/lgr" - UM "github.com/go-pkgz/rest" + um "github.com/go-pkgz/rest" "github.com/go-pkgz/rest/logger" "go.mongodb.org/mongo-driver/bson/primitive" @@ -38,7 +38,7 @@ type Server struct { } // JSON is a map alias, just for convenience -type JSON map[string]interface{} +type JSON map[string]any // Run the listen and request's router, activate rest server func (s *Server) Run(ctx context.Context, address string, port int, frontendDir string) { @@ -70,9 +70,9 @@ func (s *Server) Run(ctx context.Context, address string, port int, frontendDir func (s *Server) routes(frontendDir string) chi.Router { router := chi.NewRouter() - router.Use(middleware.RequestID, middleware.RealIP, UM.Recoverer(log.Default())) + router.Use(middleware.RequestID, middleware.RealIP, um.Recoverer(log.Default())) router.Use(middleware.Throttle(1000), middleware.Timeout(60*time.Second)) - router.Use(UM.AppInfo("ureadability", "Umputun", s.Version), UM.Ping) + router.Use(um.AppInfo("ureadability", "Umputun", s.Version), um.Ping) router.Use(tollbooth_chi.LimitHandler(tollbooth.NewLimiter(50, nil))) router.Use(logger.New(logger.Log(log.Default()), logger.WithBody, logger.Prefix("[INFO]")).Handler) @@ -95,9 +95,10 @@ func (s *Server) routes(frontendDir string) chi.Router { router.Get("/edit/{id}", s.handleEdit) _ = os.Mkdir(filepath.Join(frontendDir, "static"), 0o700) - fs, err := UM.NewFileServer("/", filepath.Join(frontendDir, "static"), UM.FsOptSPA) + fs, err := um.NewFileServer("/", filepath.Join(frontendDir, "static"), um.FsOptSPA) if err != nil { - log.Fatalf("unable to create file server, %v", err) + log.Printf("[ERROR] unable to create file server, %v", err) + return nil } router.Get("/*", func(w http.ResponseWriter, r *http.Request) { fs.ServeHTTP(w, r) @@ -118,6 +119,7 @@ func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) { if err != nil { log.Printf("[WARN] failed to render index template, %v", err) http.Error(w, err.Error(), http.StatusInternalServerError) + return } } @@ -133,6 +135,7 @@ func (s *Server) handleAdd(w http.ResponseWriter, _ *http.Request) { if err != nil { log.Printf("[WARN] failed to render add template, %v", err) http.Error(w, err.Error(), http.StatusInternalServerError) + return } } @@ -154,6 +157,7 @@ func (s *Server) handleEdit(w http.ResponseWriter, r *http.Request) { if err != nil { log.Printf("[WARN] failed to render edit template, %v", err) http.Error(w, err.Error(), http.StatusInternalServerError) + return } } @@ -236,7 +240,7 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) { } } - var responses []extractor.Response + responses := make([]extractor.Response, 0, len(testURLs)) for _, url := range testURLs { url = strings.TrimSpace(url) if url == "" { @@ -262,12 +266,13 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) { Content string } - var results []result - for _, r := range responses { + results := make([]result, 0, len(responses)) + for i := range responses { + r := &responses[i] results = append(results, result{ Title: r.Title, Excerpt: r.Excerpt, - //nolint: gosec // this content is escaped by Extractor, so it's safe to use it as is + //nolint:gosec // this content is escaped by Extractor, so it's safe to use it as is Rich: template.HTML(r.Rich), Content: r.Content, }) @@ -282,6 +287,7 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) { err = s.rulePage.ExecuteTemplate(w, "preview.gohtml", data) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + return } } @@ -345,6 +351,7 @@ func (s *Server) toggleRule(w http.ResponseWriter, r *http.Request) { err = s.indexPage.ExecuteTemplate(w, "rule-row", rule) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) + return } } @@ -368,7 +375,7 @@ func getBid(id string) primitive.ObjectID { // source: https://github.com/99designs/basicauth-go/blob/master/basicauth.go func basicAuth(realm string, credentials map[string]string) func(http.Handler) http.Handler { unauthorized := func(w http.ResponseWriter, realm string) { - w.Header().Add("WWW-Authenticate", fmt.Sprintf(`Basic realm="%s"`, realm)) + w.Header().Add("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", realm)) w.WriteHeader(http.StatusUnauthorized) } diff --git a/backend/rest/server_test.go b/backend/rest/server_test.go index 8723132c..3bbc353d 100644 --- a/backend/rest/server_test.go +++ b/backend/rest/server_test.go @@ -32,7 +32,7 @@ func TestServer_FileServer(t *testing.T) { testHTMLName := "test-ureadability.html" dir := os.TempDir() testHTMLFile := filepath.Join(dir, testHTMLName) - err := os.WriteFile(testHTMLFile, []byte("some html"), 0o700) + err := os.WriteFile(testHTMLFile, []byte("some html"), 0o600) require.NoError(t, err) srv := Server{ @@ -74,7 +74,7 @@ func TestServer_Shutdown(t *testing.T) { st := time.Now() srv.Run(ctx, "127.0.0.1", 0, "../web") - assert.True(t, time.Since(st).Seconds() < 1, "should take about 100ms") + assert.Less(t, time.Since(st), time.Second, "should take about 100ms") <-done } @@ -85,24 +85,24 @@ func TestServer_WrongAuth(t *testing.T) { // no credentials client := &http.Client{Timeout: 5 * time.Second} req, err := http.NewRequest("POST", ts.URL+"/api/rule", strings.NewReader("{}")) - assert.NoError(t, err) + require.NoError(t, err) r, err := client.Do(req) require.NoError(t, err) - assert.NoError(t, r.Body.Close()) + require.NoError(t, r.Body.Close()) assert.Equal(t, http.StatusUnauthorized, r.StatusCode) // wrong user req.SetBasicAuth("wrong_user", "password") r, err = client.Do(req) require.NoError(t, err) - assert.NoError(t, r.Body.Close()) + require.NoError(t, r.Body.Close()) assert.Equal(t, http.StatusUnauthorized, r.StatusCode) // wrong password req.SetBasicAuth("admin", "wrong_password") r, err = client.Do(req) require.NoError(t, err) - assert.NoError(t, r.Body.Close()) + require.NoError(t, r.Body.Close()) assert.Equal(t, http.StatusUnauthorized, r.StatusCode) } @@ -114,10 +114,10 @@ func TestServer_Extract(t *testing.T) { if r.URL.String() == "/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/" { fh, err := os.Open("../extractor/testdata/vsiem-mirom-dlia-obshchiei-polzy.html") testHTML, err := io.ReadAll(fh) - require.NoError(t, err) - require.NoError(t, fh.Close()) + assert.NoError(t, err) + assert.NoError(t, fh.Close()) _, err = w.Write(testHTML) - require.NoError(t, err) + assert.NoError(t, err) return } })) @@ -126,14 +126,14 @@ func TestServer_Extract(t *testing.T) { // happy path resp, err := post(t, ts.URL+"/api/extract", fmt.Sprintf(`{"url": "%s/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/"}`, tss.URL)) - assert.NoError(t, err) + require.NoError(t, err) b, err := io.ReadAll(resp.Body) - assert.NoError(t, err) + require.NoError(t, err) require.Equal(t, http.StatusOK, resp.StatusCode, string(b)) - assert.NoError(t, resp.Body.Close()) + require.NoError(t, resp.Body.Close()) response := extractor.Response{} err = json.Unmarshal(b, &response) - assert.NoError(t, err) + require.NoError(t, err) // legacy endpoint, same response is expected legacyBody, code := get(t, ts.URL+"/api/content/v1/parser"+ @@ -141,32 +141,32 @@ func TestServer_Extract(t *testing.T) { require.Equal(t, http.StatusOK, code) legacyResponse := extractor.Response{} err = json.Unmarshal([]byte(legacyBody), &legacyResponse) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, response.Content, legacyResponse.Content) // wrong body resp, err = post(t, ts.URL+"/api/extract", "wrong_body") - assert.NoError(t, err) + require.NoError(t, err) b, err = io.ReadAll(resp.Body) - assert.NoError(t, err) + require.NoError(t, err) require.Equal(t, http.StatusInternalServerError, resp.StatusCode, string(b)) - assert.NoError(t, resp.Body.Close()) + require.NoError(t, resp.Body.Close()) // no URL resp, err = post(t, ts.URL+"/api/extract", "{}") - assert.NoError(t, err) + require.NoError(t, err) b, err = io.ReadAll(resp.Body) - assert.NoError(t, err) + require.NoError(t, err) require.Equal(t, http.StatusBadRequest, resp.StatusCode, string(b)) - assert.NoError(t, resp.Body.Close()) + require.NoError(t, resp.Body.Close()) // bad URL resp, err = post(t, ts.URL+"/api/extract", `{"url": "http://bad_url"}`) - assert.NoError(t, err) + require.NoError(t, err) b, err = io.ReadAll(resp.Body) - assert.NoError(t, err) + require.NoError(t, err) require.Equal(t, http.StatusBadRequest, resp.StatusCode, string(b)) - assert.NoError(t, resp.Body.Close()) + require.NoError(t, resp.Body.Close()) } func TestServer_LegacyExtract(t *testing.T) { @@ -177,10 +177,10 @@ func TestServer_LegacyExtract(t *testing.T) { if r.URL.String() == "/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/" { fh, err := os.Open("../extractor/testdata/vsiem-mirom-dlia-obshchiei-polzy.html") testHTML, err := io.ReadAll(fh) - require.NoError(t, err) - require.NoError(t, fh.Close()) + assert.NoError(t, err) + assert.NoError(t, fh.Close()) _, err = w.Write(testHTML) - require.NoError(t, err) + assert.NoError(t, err) return } })) @@ -192,7 +192,7 @@ func TestServer_LegacyExtract(t *testing.T) { require.Equal(t, http.StatusOK, code) resp := extractor.Response{} err := json.Unmarshal([]byte(b), &resp) - assert.NoError(t, err) + require.NoError(t, err) assert.NotEmpty(t, resp.Content) // no url @@ -200,7 +200,7 @@ func TestServer_LegacyExtract(t *testing.T) { require.Equal(t, http.StatusExpectationFailed, code) errResponse := rest.JSON{} err = json.Unmarshal([]byte(b), &errResponse) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "no url passed", errResponse["error"]) // wrong url @@ -215,7 +215,7 @@ func TestServer_LegacyExtract(t *testing.T) { assert.Equal(t, http.StatusExpectationFailed, code) errResponse = rest.JSON{} err = json.Unmarshal([]byte(b), &errResponse) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "no token passed", errResponse["error"]) // wrong token @@ -224,7 +224,7 @@ func TestServer_LegacyExtract(t *testing.T) { assert.Equal(t, http.StatusUnauthorized, code) errResponse = rest.JSON{} err = json.Unmarshal([]byte(b), &errResponse) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, "wrong token passed", errResponse["error"]) // right token @@ -246,7 +246,7 @@ func TestServer_RuleHappyFlow(t *testing.T) { err = json.NewDecoder(r.Body).Decode(&rule) require.NoError(t, err) require.Equal(t, http.StatusOK, r.StatusCode) - assert.NoError(t, r.Body.Close()) + require.NoError(t, r.Body.Close()) assert.Equal(t, randomDomainName, rule.Domain) assert.Equal(t, "test content", rule.Content) ruleID := rule.ID.Hex() @@ -265,12 +265,12 @@ func TestServer_RuleHappyFlow(t *testing.T) { // disable the rule r, err = post(t, ts.URL+"/api/toggle-rule/"+rule.ID.Hex(), "") - assert.NoError(t, err) + require.NoError(t, err) // read body for error message body, err := io.ReadAll(r.Body) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, http.StatusOK, r.StatusCode, string(body)) - assert.NoError(t, r.Body.Close()) + require.NoError(t, r.Body.Close()) // get the rule by ID, should look the same as "Enabled" status is only visible on the main page b, code = get(t, ts.URL+"/edit/"+ruleID) @@ -291,7 +291,7 @@ func TestServer_RuleHappyFlow(t *testing.T) { err = json.NewDecoder(r.Body).Decode(&rule) require.NoError(t, err) require.Equal(t, http.StatusOK, r.StatusCode) - assert.NoError(t, r.Body.Close()) + require.NoError(t, r.Body.Close()) assert.Equal(t, randomDomainName, rule.Domain) assert.Equal(t, "new content", rule.Content) assert.Equal(t, ruleID, rule.ID.Hex()) @@ -305,8 +305,8 @@ func TestServer_RuleUnhappyFlow(t *testing.T) { r, err := postFormUrlencoded(t, ts.URL+"/api/rule", "") require.NoError(t, err) body, err := io.ReadAll(r.Body) - assert.NoError(t, err) - assert.NoError(t, r.Body.Close()) + require.NoError(t, err) + require.NoError(t, r.Body.Close()) require.Equal(t, http.StatusBadRequest, r.StatusCode) assert.Equal(t, "Domain is required\n", string(body)) @@ -326,8 +326,8 @@ func TestServer_FakeAuth(t *testing.T) { r, err := post(t, ts.URL+"/api/auth", `""`) require.NoError(t, err) body, err := io.ReadAll(r.Body) - assert.NoError(t, err) - assert.NoError(t, r.Body.Close()) + require.NoError(t, err) + require.NoError(t, r.Body.Close()) assert.Equal(t, http.StatusOK, r.StatusCode) assert.Contains(t, string(body), `"pong":`) } @@ -338,8 +338,9 @@ func TestServer_HandleIndex(t *testing.T) { randomDomainName := randStringBytesRmndr(42) + ".com" // add a test rule - _, err := postFormUrlencoded(t, ts.URL+"/api/rule", fmt.Sprintf(`domain=%s&content=test+content`, randomDomainName)) + r, err := postFormUrlencoded(t, ts.URL+"/api/rule", fmt.Sprintf(`domain=%s&content=test+content`, randomDomainName)) require.NoError(t, err) + require.NoError(t, r.Body.Close()) // test index page resp, err := http.Get(ts.URL + "/") @@ -381,6 +382,7 @@ func TestServer_HandleEdit(t *testing.T) { // add a test rule r, err := postFormUrlencoded(t, ts.URL+"/api/rule", fmt.Sprintf(`domain=%s&content=test+content`, randomDomainName)) require.NoError(t, err) + defer r.Body.Close() var rule datastore.Rule err = json.NewDecoder(r.Body).Decode(&rule) require.NoError(t, err) @@ -418,6 +420,7 @@ func TestServer_ToggleRule(t *testing.T) { // add a test rule r, err := postFormUrlencoded(t, ts.URL+"/api/rule", fmt.Sprintf(`domain=%s&content=test+content`, randomDomainName)) require.NoError(t, err) + defer r.Body.Close() var rule datastore.Rule err = json.NewDecoder(r.Body).Decode(&rule) require.NoError(t, err) @@ -432,7 +435,7 @@ func TestServer_ToggleRule(t *testing.T) { body, err := io.ReadAll(r.Body) require.NoError(t, err) - assert.NoError(t, r.Body.Close()) + require.NoError(t, r.Body.Close()) assert.Contains(t, string(body), `class="rules__row rules__row_disabled"`, string(body)) // toggle rule again (enable) @@ -443,7 +446,7 @@ func TestServer_ToggleRule(t *testing.T) { body, err = io.ReadAll(r.Body) require.NoError(t, err) - assert.NoError(t, r.Body.Close()) + require.NoError(t, r.Body.Close()) assert.NotContains(t, string(body), `class="rules__row rules__row_disabled"`) } @@ -458,7 +461,7 @@ func TestServer_ToggleRuleNotFound(t *testing.T) { body, err := io.ReadAll(r.Body) require.NoError(t, err) - assert.NoError(t, r.Body.Close()) + require.NoError(t, r.Body.Close()) assert.Contains(t, string(body), `Rule not found`, string(body)) } @@ -530,10 +533,10 @@ func TestServer_Preview(t *testing.T) { if r.URL.String() == "/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/" { fh, err := os.Open("../extractor/testdata/vsiem-mirom-dlia-obshchiei-polzy.html") testHTML, err := io.ReadAll(fh) - require.NoError(t, err) - require.NoError(t, fh.Close()) + assert.NoError(t, err) + assert.NoError(t, fh.Close()) _, err = w.Write(testHTML) - require.NoError(t, err) + assert.NoError(t, err) return } })) @@ -542,30 +545,30 @@ func TestServer_Preview(t *testing.T) { // happy path with no rule resp, err := postFormUrlencoded(t, ts.URL+"/api/preview", fmt.Sprintf(`test_urls=%s/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/&content=`, tss.URL)) - assert.NoError(t, err) + require.NoError(t, err) b, err := io.ReadAll(resp.Body) - assert.NoError(t, err) + require.NoError(t, err) require.Equal(t, http.StatusOK, resp.StatusCode, string(b)) - assert.NoError(t, resp.Body.Close()) + require.NoError(t, resp.Body.Close()) assert.Contains(t, string(b), "Всем миром для общей пользы • Umputun тут был") // happy path with custom rule resp, err = postFormUrlencoded(t, ts.URL+"/api/preview", fmt.Sprintf(`test_urls=%s/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/&content=article`, tss.URL)) - assert.NoError(t, err) + require.NoError(t, err) b, err = io.ReadAll(resp.Body) - assert.NoError(t, err) + require.NoError(t, err) require.Equal(t, http.StatusOK, resp.StatusCode, string(b)) - assert.NoError(t, resp.Body.Close()) + require.NoError(t, resp.Body.Close()) assert.Contains(t, string(b), "Всем миром для общей пользы") // no URL resp, err = post(t, ts.URL+"/api/preview", "") - assert.NoError(t, err) + require.NoError(t, err) b, err = io.ReadAll(resp.Body) - assert.NoError(t, err) + require.NoError(t, err) require.Equal(t, http.StatusOK, resp.StatusCode, string(b)) - assert.NoError(t, resp.Body.Close()) + require.NoError(t, resp.Body.Close()) assert.Contains(t, string(b), "No preview results available.") // 10Mb body supposed to hit the form parsing limit @@ -580,17 +583,17 @@ func TestServer_Preview(t *testing.T) { func get(t *testing.T, url string) (response string, statusCode int) { r, err := http.Get(url) - assert.NoError(t, err) + require.NoError(t, err) body, err := io.ReadAll(r.Body) - assert.NoError(t, err) - assert.NoError(t, r.Body.Close()) + require.NoError(t, err) + require.NoError(t, r.Body.Close()) return string(body), r.StatusCode } func post(t *testing.T, url, body string) (*http.Response, error) { client := &http.Client{Timeout: 5 * time.Second} req, err := http.NewRequest("POST", url, strings.NewReader(body)) - assert.NoError(t, err) + require.NoError(t, err) req.SetBasicAuth("admin", "password") return client.Do(req) } @@ -598,7 +601,7 @@ func post(t *testing.T, url, body string) (*http.Response, error) { func postFormUrlencoded(t *testing.T, url, body string) (*http.Response, error) { client := &http.Client{Timeout: 5 * time.Second} req, err := http.NewRequest("POST", url, strings.NewReader(body)) - assert.NoError(t, err) + require.NoError(t, err) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") req.SetBasicAuth("admin", "password") return client.Do(req) @@ -611,7 +614,7 @@ func startupT(t *testing.T) (*httptest.Server, *Server) { } db, err := datastore.New("mongodb://localhost:27017/", "test_ureadability", 0) - assert.NoError(t, err) + require.NoError(t, err) srv := Server{ Readability: extractor.UReadability{TimeOut: 30 * time.Second, SnippetSize: 300, Rules: db.GetStores()}, Credentials: map[string]string{"admin": "password"},