From 5695610af9988fa3da444f04d8071683e95d8831 Mon Sep 17 00:00:00 2001 From: Dmitry Verkhoturov Date: Mon, 21 Apr 2025 23:46:14 +0100 Subject: [PATCH] Fix repository name and imports, improve code structure - Fix repository name from ukeeper-redabilty to ukeeper-readability in all imports and go.mod - Introduce Stores struct to better organize DAOs - Update CLI flags format to use hyphen-separated-words instead of underscores - Update README to fix incorrect badge links - Change config parameters to use consistent naming conventions - Fix frontend-dir flag name in code to match README --- README.md | 6 +++--- backend/datastore/mongo.go | 13 ++++++++++--- backend/datastore/rules_test.go | 8 ++++++-- backend/extractor/readability.go | 6 ++++-- backend/extractor/readability_test.go | 2 +- backend/go.mod | 2 +- backend/main.go | 17 +++++++++++------ backend/main_test.go | 4 ++-- backend/rest/server.go | 5 +++-- backend/rest/server_test.go | 12 +++++++++--- 10 files changed, 50 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 6bef4391..9a9639d2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## ukeeper-readability [![build](https://github.com/ukeeper/ukeeper-redabilty/actions/workflows/ci.yml/badge.svg)](https://github.com/ukeeper/ukeeper-redabilty/actions/workflows/ci.yml) [![Coverage Status](https://coveralls.io/repos/github/ukeeper/ukeeper-readability/badge.svg?branch=master)](https://coveralls.io/github/ukeeper/ukeeper-readability?branch=master) +## ukeeper-readability [![build](https://github.com/ukeeper/ukeeper-readability/actions/workflows/ci.yml/badge.svg)](https://github.com/ukeeper/ukeeper-readability/actions/workflows/ci.yml) [![Coverage Status](https://coveralls.io/repos/github/ukeeper/ukeeper-readability/badge.svg?branch=master)](https://coveralls.io/github/ukeeper/ukeeper-readability?branch=master) ### Running instructions @@ -10,8 +10,8 @@ |--------------|-----------------|----------------|-------------------------------------------------------| | address | UKEEPER_ADDRESS | all interfaces | web server listening address | | port | UKEEPER_PORT | `8080` | web server port | -| mongo_uri | MONGO_URI | none | MongoDB connection string, _required_ | -| frontend_dir | FRONTEND_DIR | `/srv/web` | directory with frontend files | +| mongo-uri | MONGO_URI | none | MongoDB connection string, _required_ | +| frontend-dir | FRONTEND_DIR | `/srv/web` | directory with frontend files | | token | TOKEN | none | token for /content/v1/parser endpoint auth | | mongo-delay | MONGO_DELAY | `0` | mongo initial delay | | mongo-db | MONGO_DB | `ureadability` | mongo database name | diff --git a/backend/datastore/mongo.go b/backend/datastore/mongo.go index c7c0ff49..1d3ef5fc 100644 --- a/backend/datastore/mongo.go +++ b/backend/datastore/mongo.go @@ -37,15 +37,22 @@ func New(connectionURI, dbName string, delay time.Duration) (*MongoServer, error return &MongoServer{client: client, dbName: dbName}, nil } +// Stores contains all DAO instances +type Stores struct { + Rules RulesDAO +} + // GetStores initialize collections and make indexes -func (m *MongoServer) GetStores() (rules RulesDAO) { +func (m *MongoServer) GetStores() Stores { rIndexes := []mongo.IndexModel{ {Keys: bson.D{{Key: "enabled", Value: 1}, {Key: "domain", Value: 1}}}, {Keys: bson.D{{Key: "user", Value: 1}, {Key: "domain", Value: 1}, {Key: "enabled", Value: 1}}}, {Keys: bson.D{{Key: "domain", Value: 1}, {Key: "match_urls", Value: 1}}}, } - rules = RulesDAO{Collection: m.collection("rules", rIndexes)} - return rules + + return Stores{ + Rules: RulesDAO{Collection: m.collection("rules", rIndexes)}, + } } // collection makes collection with indexes diff --git a/backend/datastore/rules_test.go b/backend/datastore/rules_test.go index b2b444b8..46a0298a 100644 --- a/backend/datastore/rules_test.go +++ b/backend/datastore/rules_test.go @@ -19,7 +19,9 @@ func TestRules(t *testing.T) { server, err := New("mongodb://localhost:27017/", "test_ureadability", 0) require.NoError(t, err) assert.NotNil(t, server.client) - rules := server.GetStores() + stores := server.GetStores() + assert.NotNil(t, stores) + rules := stores.Rules assert.NotNil(t, rules) rule := Rule{ Domain: randStringBytesRmndr(42) + ".com", @@ -74,7 +76,9 @@ func TestRulesCanceledContext(t *testing.T) { server, err := New("mongodb://wrong", "", 0) require.NoError(t, err) assert.NotNil(t, server.client) - rules := server.GetStores() + stores := server.GetStores() + assert.NotNil(t, stores) + rules := stores.Rules assert.NotNil(t, rules) ctx, cancel := context.WithCancel(context.Background()) diff --git a/backend/extractor/readability.go b/backend/extractor/readability.go index b8cf954a..63e1d016 100644 --- a/backend/extractor/readability.go +++ b/backend/extractor/readability.go @@ -16,7 +16,7 @@ import ( "github.com/mauidude/go-readability" "go.mongodb.org/mongo-driver/bson/primitive" - "github.com/ukeeper/ukeeper-redabilty/backend/datastore" + "github.com/ukeeper/ukeeper-readability/backend/datastore" ) // Rules interface with all methods to access datastore @@ -56,7 +56,9 @@ var ( reDot = regexp.MustCompile(`\D(\.)\S`) ) -const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15" +const ( + userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15" +) // Extract fetches page and retrieves article func (f *UReadability) Extract(ctx context.Context, reqURL string) (*Response, error) { diff --git a/backend/extractor/readability_test.go b/backend/extractor/readability_test.go index 8771c54d..1c2ff3e9 100644 --- a/backend/extractor/readability_test.go +++ b/backend/extractor/readability_test.go @@ -15,7 +15,7 @@ import ( "github.com/stretchr/testify/require" "go.mongodb.org/mongo-driver/bson/primitive" - "github.com/ukeeper/ukeeper-redabilty/backend/datastore" + "github.com/ukeeper/ukeeper-readability/backend/datastore" ) func TestExtractURL(t *testing.T) { diff --git a/backend/go.mod b/backend/go.mod index c166c080..dd0dd6bf 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,4 +1,4 @@ -module github.com/ukeeper/ukeeper-redabilty/backend +module github.com/ukeeper/ukeeper-readability/backend go 1.23.0 diff --git a/backend/main.go b/backend/main.go index c4195d8a..7fe18985 100644 --- a/backend/main.go +++ b/backend/main.go @@ -11,9 +11,9 @@ import ( log "github.com/go-pkgz/lgr" "github.com/jessevdk/go-flags" - "github.com/ukeeper/ukeeper-redabilty/backend/datastore" - "github.com/ukeeper/ukeeper-redabilty/backend/extractor" - "github.com/ukeeper/ukeeper-redabilty/backend/rest" + "github.com/ukeeper/ukeeper-readability/backend/datastore" + "github.com/ukeeper/ukeeper-readability/backend/extractor" + "github.com/ukeeper/ukeeper-readability/backend/rest" ) var revision string @@ -21,10 +21,10 @@ var revision string var opts struct { Address string `long:"address" env:"UKEEPER_ADDRESS" default:"" description:"listening address"` Port int `long:"port" env:"UKEEPER_PORT" default:"8080" description:"port"` - FrontendDir string `long:"frontend_dir" env:"FRONTEND_DIR" default:"/srv/web" description:"directory with frontend templates and static/ directory for static assets"` + FrontendDir string `long:"frontend-dir" env:"FRONTEND_DIR" default:"/srv/web" description:"directory with frontend templates and static/ directory for static assets"` Credentials map[string]string `long:"creds" env:"CREDS" description:"credentials for protected calls (POST, DELETE /rules)"` Token string `long:"token" env:"UKEEPER_TOKEN" description:"token for /content/v1/parser endpoint auth"` - MongoURI string `short:"m" long:"mongo_uri" env:"MONGO_URI" required:"true" description:"MongoDB connection string"` + MongoURI string `short:"m" long:"mongo-uri" env:"MONGO_URI" required:"true" description:"MongoDB connection string"` MongoDelay time.Duration `long:"mongo-delay" env:"MONGO_DELAY" default:"0" description:"mongo initial delay"` MongoDB string `long:"mongo-db" env:"MONGO_DB" default:"ureadability" description:"mongo database name"` Debug bool `long:"dbg" env:"DEBUG" description:"debug mode"` @@ -46,8 +46,13 @@ func main() { if err != nil { log.Fatalf("[ERROR] can't connect to mongo %v", err) } + stores := db.GetStores() srv := rest.Server{ - Readability: extractor.UReadability{TimeOut: 30 * time.Second, SnippetSize: 300, Rules: db.GetStores()}, + Readability: extractor.UReadability{ + TimeOut: 30 * time.Second, + SnippetSize: 300, + Rules: stores.Rules, + }, Token: opts.Token, Credentials: opts.Credentials, Version: revision, diff --git a/backend/main_test.go b/backend/main_test.go index 6697d8ae..05b642e3 100644 --- a/backend/main_test.go +++ b/backend/main_test.go @@ -23,9 +23,9 @@ func Test_Main(t *testing.T) { port := chooseRandomUnusedPort() os.Args = []string{"test", "--port=" + strconv.Itoa(port), "--dbg", - "--mongo_uri=mongodb://localhost:27017/", + "--mongo-uri=mongodb://localhost:27017/", "--mongo-db=test_ureadability", - "--frontend_dir=web", + "--frontend-dir=web", } done := make(chan struct{}) diff --git a/backend/rest/server.go b/backend/rest/server.go index b3b19748..153de0e3 100644 --- a/backend/rest/server.go +++ b/backend/rest/server.go @@ -18,8 +18,8 @@ import ( "github.com/go-pkgz/routegroup" "go.mongodb.org/mongo-driver/bson/primitive" - "github.com/ukeeper/ukeeper-redabilty/backend/datastore" - "github.com/ukeeper/ukeeper-redabilty/backend/extractor" + "github.com/ukeeper/ukeeper-readability/backend/datastore" + "github.com/ukeeper/ukeeper-readability/backend/extractor" ) // Server is a basic rest server providing access to store and invoking parser @@ -176,6 +176,7 @@ func (s *Server) extractArticle(w http.ResponseWriter, r *http.Request) { // if token is not set for application, it won't be checked func (s *Server) extractArticleEmulateReadability(w http.ResponseWriter, r *http.Request) { token := r.URL.Query().Get("token") + if s.Token != "" && token == "" { rest.SendErrorJSON(w, r, log.Default(), http.StatusExpectationFailed, nil, "no token passed") return diff --git a/backend/rest/server_test.go b/backend/rest/server_test.go index f98b069d..35b88d15 100644 --- a/backend/rest/server_test.go +++ b/backend/rest/server_test.go @@ -19,8 +19,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/ukeeper/ukeeper-redabilty/backend/datastore" - "github.com/ukeeper/ukeeper-redabilty/backend/extractor" + "github.com/ukeeper/ukeeper-readability/backend/datastore" + "github.com/ukeeper/ukeeper-readability/backend/extractor" ) const letterBytes = "abcdefghijklmnopqrstuvwxyz" @@ -615,8 +615,14 @@ func startupT(t *testing.T) (*httptest.Server, *Server) { db, err := datastore.New("mongodb://localhost:27017/", "test_ureadability", 0) require.NoError(t, err) + + stores := db.GetStores() srv := Server{ - Readability: extractor.UReadability{TimeOut: 30 * time.Second, SnippetSize: 300, Rules: db.GetStores()}, + Readability: extractor.UReadability{ + TimeOut: 30 * time.Second, + SnippetSize: 300, + Rules: stores.Rules, + }, Credentials: map[string]string{"admin": "password"}, Version: "dev-test", }