diff --git a/.github/workflows/staticcheck.yaml b/.github/workflows/staticcheck.yaml index 8c50d50..4de5bb9 100644 --- a/.github/workflows/staticcheck.yaml +++ b/.github/workflows/staticcheck.yaml @@ -1,4 +1,4 @@ -on: [push, pull_request] +on: [pull_request] jobs: verify: runs-on: ubuntu-latest diff --git a/api/docs/api.yaml b/api/docs/api.yaml index e090100..e6d9f5f 100644 --- a/api/docs/api.yaml +++ b/api/docs/api.yaml @@ -3,26 +3,24 @@ info: title: GitHub Banners API description: API for generating GitHub user statistics banners as SVG images version: 1.0.0 - servers: - url: http://localhost:8080 description: Local development server - + - url: https://api.bnrs.dev + description: Public demo paths: - /banners/preview/: + /banners/preview: get: summary: Get banner preview for a GitHub user description: | Generates and returns an SVG banner with GitHub user statistics. - + The banner includes: - Total repositories count - Original vs forked repositories breakdown - Total stars received - Total forks - Top programming languages used - - Data is cached for performance (see caching strategy in docs). operationId: getBannerPreview parameters: - name: username @@ -38,8 +36,8 @@ paths: description: Banner type schema: type: string - enum: [wide] - example: wide + enum: [dark, default] + example: dark responses: '200': description: Successfully generated banner @@ -48,7 +46,7 @@ paths: schema: type: string format: binary - example: ... + example: '...' '400': description: Invalid request parameters content: @@ -68,27 +66,28 @@ paths: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - example: - error: user doesn't exist + examples: + user_doesnt_exist: + value: + error: user doesn't exist '500': description: Internal server error or service unavailable content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - example: - error: can't get preview - - /banners/: + examples: + cant_get_preview: + value: + error: can't get preview + /banners: post: summary: Create a persistent banner description: | Creates a long-term stored banner for a GitHub user. - - **Note**: This endpoint is not yet implemented (returns 501). - Future implementation will: + - Generate and store banner in storage service - - Return a persistent URL for embedding + - Return a relative URL for embedding - Support automatic refresh of stored banners operationId: createBanner requestBody: @@ -98,13 +97,39 @@ paths: schema: $ref: '#/components/schemas/CreateBannerRequest' responses: - '501': - description: Not implemented + '400': + description: Invalid request content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' - + examples: + invalid_json: + value: + error: invalid json + invalid_banner_type: + value: + error: invalid banner type + '404': + description: Requested user doesn't exist + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + user_doesnt_exist: + value: + error: user doesn't exist + '500': + description: Server can't create banner due to internal issues + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + examples: + cant_create_banner: + value: + error: can't create banner components: schemas: ErrorResponse: @@ -115,194 +140,18 @@ components: error: type: string description: Error message describing what went wrong - CreateBannerRequest: type: object required: - username - - banner_type + - type properties: username: type: string description: GitHub username example: torvalds - banner_type: + type: type: string - enum: [wide] + enum: [dark, default] description: Type of banner to create - example: wide - - BannerPreviewRequest: - type: object - description: Request sent to the renderer service - properties: - username: - type: string - example: torvalds - banner_type: - type: string - example: wide - total_repos: - type: integer - description: Total number of repositories - example: 42 - original_repos: - type: integer - description: Number of original (non-forked) repositories - example: 35 - forked_repos: - type: integer - description: Number of forked repositories - example: 7 - total_stars: - type: integer - description: Total stars across all original repositories - example: 150000 - total_forks: - type: integer - description: Total forks across all original repositories - example: 45000 - languages: - type: object - description: Language name to repository count mapping - additionalProperties: - type: integer - example: - Go: 15 - Python: 10 - TypeScript: 8 - - GithubUserStats: - type: object - description: Calculated statistics for a GitHub user - properties: - totalRepos: - type: integer - description: Total number of repositories - example: 42 - originalRepos: - type: integer - description: Number of original (non-forked) repositories - example: 35 - forkedRepos: - type: integer - description: Number of forked repositories - example: 7 - totalStars: - type: integer - description: Total stars received on original repositories - example: 150000 - totalForks: - type: integer - description: Total forks of original repositories - example: 45000 - languages: - type: object - description: Programming language to repository count - additionalProperties: - type: integer - example: - Go: 15 - Python: 10 - - GithubBannerInfoEvent: - type: object - description: Kafka event for banner info (future use) - properties: - event_type: - type: string - example: github_banner_info_ready - event_version: - type: integer - example: 1 - produced_at: - type: string - format: date-time - payload: - $ref: '#/components/schemas/BannerEventPayload' - - BannerEventPayload: - type: object - description: Payload for banner Kafka events - properties: - username: - type: string - example: torvalds - banner_type: - type: string - example: wide - storage_path: - type: string - description: Path where banner is stored - example: /banners/torvalds/wide.svg - stats: - $ref: '#/components/schemas/GithubUserStats' - fetched_at: - type: string - format: date-time - description: When the data was fetched from GitHub - - GithubRepository: - type: object - description: Repository data stored in database - properties: - id: - type: integer - format: int64 - description: GitHub repository ID - owner_username: - type: string - description: Repository owner's GitHub username - pushed_at: - type: string - format: date-time - nullable: true - updated_at: - type: string - format: date-time - nullable: true - language: - type: string - nullable: true - description: Primary programming language - stars_count: - type: integer - description: Number of stars - forks_count: - type: integer - description: Number of forks - is_fork: - type: boolean - description: Whether this is a forked repository - - GithubUserData: - type: object - description: Complete GitHub user data stored in database - properties: - username: - type: string - name: - type: string - nullable: true - company: - type: string - nullable: true - location: - type: string - nullable: true - bio: - type: string - nullable: true - public_repos: - type: integer - followers: - type: integer - following: - type: integer - repositories: - type: array - items: - $ref: '#/components/schemas/GithubRepository' - fetched_at: - type: string - format: date-time \ No newline at end of file + example: dark diff --git a/api/docs/architecture.md b/api/docs/architecture.md index fe15357..ef6d7b5 100644 --- a/api/docs/architecture.md +++ b/api/docs/architecture.md @@ -4,212 +4,23 @@ ![alt text](image.png) -## Directory Structure +### 1. Clean Architecture -``` -internal/ -├── app/ -│ └── user_stats/ -│ └── worker.go # Background worker for scheduled stats updates -├── cache/ -│ ├── stats.go # In-memory cache for user stats -│ ├── preview.go # In-memory cache for rendered banners -│ └── banner_info_hash.go # Hash utility for banner cache keys -├── config/ -│ ├── config.go # Main application config (env variables) -│ ├── kafka.go # Kafka configuration (future use) -│ └── psgr.go # PostgreSQL configuration -├── domain/ -│ ├── banner.go # Banner types, BannerInfo, LTBannerInfo, Banner structs -│ ├── types.go # GithubRepository, GithubUserData, GithubUserStats models -│ ├── errors.go # Domain errors (ErrNotFound, ErrUnavailable, ConflictError) -│ ├── preview/ -│ │ ├── usecase.go # PreviewUsecase - orchestrates stats + rendering -│ │ ├── service.go # PreviewService - caches renderer results with singleflight -│ │ └── errors.go # Preview-specific errors -│ └── user_stats/ -│ ├── service.go # UserStatsService - core business logic with cache strategy -│ ├── calculator.go # CalculateStats - aggregates repository statistics -│ ├── models.go # CachedStats, WorkerConfig structs -│ ├── cache.go # Cache interface definition -│ └── interface.go # Repository and fetcher interfaces -├── handlers/ -│ ├── banners.go # HTTP handlers for /banners/* endpoints -│ ├── dto.go # Future: DTOs for Create endpoint -│ └── error_response.go # Error response helper -├── infrastructure/ -│ ├── db/ -│ │ └── connection.go # PostgreSQL connection setup -│ ├── github/ -│ │ ├── fetcher.go # GitHub API client with rate limit handling -│ │ └── clients_pool.go # Multi-token client pool for rate limit distribution -│ ├── kafka/ -│ │ ├── producer.go # Kafka event producer (future use) -│ │ └── dto.go # Kafka event DTOs -│ ├── renderer/ -│ │ ├── renderer.go # Renderer HTTP client for banner rendering -│ │ ├── dto.go # Renderer request/response DTOs -│ │ └── http/ -│ │ └── client.go # HTTP client factory for renderer -│ ├── httpauth/ -│ │ ├── hmac_signer.go # HMAC request signing for inter-service auth -│ │ └── round_tripper.go # Auth HTTP round tripper -│ ├── server/ -│ │ └── server.go # HTTP server setup with CORS -│ └── storage/ -│ ├── client.go # Storage service HTTP client -│ └── dto.go # Storage request/response DTOs -├── logger/ -│ └── logger.go # Structured logging (slog-based) -├── migrations/ -│ ├── migrations.go # Goose migration runner (embedded SQL files) -│ ├── 001_create_users_table.sql -│ ├── 002_create_repositories_table.sql -│ └── 003_create_banners_table.sql -└── repo/ - ├── banners/ # Banner repository (future use) - │ ├── interface.go - │ ├── postgres.go - │ ├── postgres_mapper.go - │ └── postgres_queries.go - └── github_user_data/ - ├── storage.go # GithubDataPsgrRepo struct - ├── get.go # GetUserData - fetch user from DB - ├── save.go # SaveUserData - persist user to DB - ├── repos_upsert.go # Batch upsert repositories - ├── usernames.go # GetAllUsernames - for worker refresh - ├── error_mapping.go # PostgreSQL error mapping - └── storage_test.go # Repository tests -``` - -## Data Flow - -### Banner Preview Request - -``` -1. Client ──▶ GET /banners/preview/?username=X&type=wide - -2. Handler (BannersHandler.Preview) ──▶ PreviewUsecase.GetPreview(username, type) - -3. PreviewUsecase flow: - ├─▶ Validate banner type - ├─▶ StatsService.GetStats(username) - │ └─▶ See StatsService flow below - └─▶ PreviewProvider.GetPreview(bannerInfo) - └─▶ See PreviewService flow below - -4. StatsService.GetStats flow: - ├─▶ Check in-memory cache - │ ├─▶ If fresh (<10min): return cached stats - │ └─▶ If stale (>10min but <24h): return cached, trigger async refresh - ├─▶ If cache miss: Check database (repo.GetUserData) - │ └─▶ If found: cache it, return stats - └─▶ If db miss: Fetch from GitHub API (fetcher.FetchUserData) - ├─▶ Save to database (repo.SaveUserData) - ├─▶ Calculate stats (CalculateStats) - ├─▶ Cache results - └─▶ Return stats - -5. PreviewService.GetPreview flow: - ├─▶ Generate hash key from BannerInfo (excludes FetchedAt) - ├─▶ Check in-memory cache by hash - │ └─▶ If found: return cached banner - └─▶ If miss: Render via singleflight (dedupe concurrent requests) - ├─▶ Call renderer.RenderPreview(bannerInfo) - │ └─▶ HTTP POST to renderer service (with HMAC auth) - ├─▶ Cache result by hash - └─▶ Return banner (SVG bytes) - -6. Handler ──▶ Response (image/svg+xml) -``` - -### Background Worker Flow - -``` -StatsWorker.Start (runs every hour by default) - │ - ▼ -RefreshAll(ctx, config) - │ - ├─▶ Get all usernames from database (repo.GetAllUsernames) - │ - └─▶ Worker pool (configurable concurrency): - └─▶ RecalculateAndSync(username) - ├─▶ Fetch fresh data from GitHub API - ├─▶ Save to database - ├─▶ Calculate stats - └─▶ Update cache -``` - -## Database Schema - -### Users Table - -```sql -CREATE TABLE IF NOT EXISTS users ( - username TEXT PRIMARY KEY, - name TEXT, - company TEXT, - location TEXT, - bio TEXT, - public_repos_count INT NOT NULL, - followers_count INT, - following_count INT, - fetched_at TIMESTAMP NOT NULL -); - -CREATE INDEX idx_users_username ON users(username); -``` - -### Repositories Table - -```sql -CREATE TABLE IF NOT EXISTS repositories ( - github_id BIGINT PRIMARY KEY, - owner_username TEXT NOT NULL, - pushed_at TIMESTAMP, - updated_at TIMESTAMP, - language TEXT, - stars_count INT NOT NULL, - forks_count INT NOT NULL, - is_fork BOOLEAN NOT NULL, - CONSTRAINT fk_repository_owner - FOREIGN KEY (owner_username) - REFERENCES users(username) ON DELETE CASCADE -); - -CREATE INDEX idx_repositories_owner_username ON repositories(owner_username); -``` - -### Banners Table - -```sql -CREATE TABLE IF NOT EXISTS banners ( - github_username TEXT PRIMARY KEY, - banner_type TEXT NOT NULL, - storage_path TEXT NOT NULL, - is_active BOOLEAN NOT NULL DEFAULT true, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -); - -CREATE INDEX idx_banners_github_username ON banners(github_username); -``` - -## Key Design Patterns - -### 1. Clean Architecture / Hexagonal Architecture - -- **Domain Layer**: Pure business logic (`domain/`, `domain/user_stats/`, `domain/preview/`) -- **Application Layer**: Use cases and app services (`app/`) -- **Infrastructure Layer**: External services (`infrastructure/`) -- **Interface Layer**: HTTP handlers (`handlers/`) +- **Domain Layer**: Pure business logic usecases and services (`domain/`, `domain/user_stats/`, `domain/preview/`) + Define logic +- **Transport Layer**: HTTP handlers (`handlers/`) + Define API +- **Application Layer** Workers/shedulers (`app/`) + Define sheduling +- **Infrastructure Layer**: External services domain uses through interfaces (`infrastructure/`) + "Helpers" for domain logic, doesn't contain any business logic of the service ### 2. Repository Pattern -- `GithubUserDataRepository` interface for data persistence -- PostgreSQL implementation in `repo/github_user_data/` -- Abstracts database operations from domain logic +- github user data repository ( PostgreSQL ) +- banners repository ( PostgreSQL ) + +These repositories use unified `internal/repo/errors.go` to let domain understand errors ### 3. Cache-Aside Pattern with TTL Strategy @@ -219,10 +30,10 @@ CREATE INDEX idx_banners_github_username ON banners(github_username); - Hard TTL (24 hours): Data considered stale after soft TTL, returned but async refreshed - **Preview cache**: Uses hash of BannerInfo (excluding FetchedAt) as key -### 4. Worker Pattern +### 4. Workers -- Background scheduled tasks via `StatsWorker` -- Concurrent processing with configurable batch size and worker count +- Background scheduled tasks via `StatsWorker` and `BannersWorker` +- Concurrent processing with configurable concurrency rate ( gorutines count for every update ) - Results/errors collected via channels ### 5. Singleflight Pattern @@ -236,24 +47,41 @@ CREATE INDEX idx_banners_github_username ON banners(github_username); - Automatic token rotation based on rate limit status - Prevents single-token rate limit exhaustion -## External Dependencies +### 7. Errors flow + +- Errors come from domain wrapped using errors from `errors.go` in usecase's package you are using +- For example long-term usecase return errors wrapped with `internal/domain/long-term/errors.go` so handler can understand, what happened +- Important! errors can come with a lot of context, and if it's negative error, it's better to log it, cause it contains a lot of information about the source of error +- But handlers shouldn't just call in as HTTP response err.Error() cause it can give out private service issues + +### 8. Username normalization + +- When our service calls github api, it will work with usernames in all cases: "hurtki"/"HURTKI" +- So our github data repository is ready for this, with `username_normalized` column, that allows to update table more efficiently and allows constraints to work +- But it still contains `username` filed which contains username with actual case +- Also banners table contains normalized username to restrict creating of two banners with same username +- Also cache for stats uses + +## Main Dependencies -| Service | Purpose | Library | -| ---------- | ------------------------ | ---------------------- | -| PostgreSQL | Persistent storage | `jackc/pgx/v5` | -| GitHub API | User data source | `google/go-github/v81` | -| Kafka | Event streaming (future) | `IBM/sarama` | -| Renderer | Banner image generation | HTTP client | -| Storage | Banner file storage | HTTP client | -| Goose | Database migrations | `pressly/goose/v3` | -| Chi | HTTP routing | `go-chi/chi/v5` | -| go-cache | In-memory caching | `patrickmn/go-cache` | -| xxhash | Fast hashing | `cespare/xxhash/v2` | -| singleflight | Request deduplication | `golang.org/x/sync/singleflight` | +| Service | Purpose | Library | +| ------------ | ------------------------ | -------------------------------- | +| PostgreSQL | Persistent storage | `jackc/pgx/v5` | +| GitHub API | User data source | `google/go-github/v81` | +| Kafka | Event streaming (future) | `IBM/sarama` | +| Renderer | Banner image generation | HTTP client | +| Storage | Banner file storage | HTTP client | +| Goose | Database migrations | `pressly/goose/v3` | +| Chi | HTTP routing | `go-chi/chi/v5` | +| go-cache | In-memory caching | `patrickmn/go-cache` | +| xxhash | Fast hashing | `cespare/xxhash/v2` | +| singleflight | Request deduplication | `golang.org/x/sync/singleflight` | -## Inter-Service Communication +## Inter-Service Communication ( not implemented on handlers side ) -Services communicate via HTTP with HMAC-based authentication: +> Now everything is started in docker compose network, that allows not to secure inter-services communucations + +Services will communicate via HTTP with HMAC-based authentication: ``` API Service ──▶ HMAC Signer (signs request with timestamp + secret) @@ -262,6 +90,8 @@ API Service ──▶ HMAC Signer (signs request with timestamp + secret) ``` Headers added: -- `X-Service`: Service identifier (e.g., "api") + +- `X-Signature`: HMAC-SHA256 of "method[\n]url_path[/n]timestamp[/n]service_name" - `X-Timestamp`: Unix timestamp -- `X-Signature`: HMAC-SHA256 of "service:timestamp" \ No newline at end of file +- `X-Service`: Service identifier (e.g., "api") + diff --git a/api/docs/image.png b/api/docs/image.png deleted file mode 100644 index 9aada94..0000000 Binary files a/api/docs/image.png and /dev/null differ diff --git a/api/docs/manual.md b/api/docs/manual.md deleted file mode 100644 index 25867f2..0000000 --- a/api/docs/manual.md +++ /dev/null @@ -1,385 +0,0 @@ -# GitHub Banners API - Manual - -## Prerequisites - -- Go 1.22+ -- PostgreSQL 14+ -- GitHub Personal Access Token(s) -- Renderer service (for banner rendering) -- Storage service (for persistent banner storage, optional) - -## Environment Variables - -Create a `.env` file based on `.env.example`: - -```bash -# Server port (default: 8080) -PORT=8080 - -# CORS allowed origins (comma-separated) -CORS_ORIGINS=example.com,www.example.com - -# GitHub tokens for API access (comma-separated, supports multiple for rate limit distribution) -GITHUB_TOKENS=ghp_token1,ghp_token2 - -# Rate limiting (requests per second, currently not enforced) -RATE_LIMIT_RPS=10 - -# Cache configuration (valid units: ms, s, m, h) -CACHE_TTL=5m - -# Request timeout for external APIs -REQUEST_TIMEOUT=10s - -# Logging (levels: DEBUG, INFO, WARN, ERROR) -LOG_LEVEL=DEBUG -LOG_FORMAT=json - -# Secret key for inter-service communication (HMAC signing) -SERVICES_SECRET_KEY=your-secret-key - -# Renderer service URL -RENDERER_BASE_URL=https://renderer/ - -# Storage service URL -STORAGE_BASE_URL=http://storage/ - -# PostgreSQL configuration -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres -POSTGRES_DB=banners -DB_HOST=localhost -PGPORT=5432 -``` - -## Running the Service - -### Local Development - -```bash -# Install dependencies -go mod download - -# Set environment variables (or use .env file with your preferred loader) -export POSTGRES_USER=postgres -export POSTGRES_PASSWORD=postgres -export POSTGRES_DB=banners -export DB_HOST=localhost -export PGPORT=5432 -export GITHUB_TOKENS=your_github_token -export RENDERER_BASE_URL=http://localhost:3000/ -export STORAGE_BASE_URL=http://localhost:3001/ - -# Run the service -go run main.go -``` - -### Using Docker - -```bash -# Build the image -docker build -t github-banners-api . - -# Run the container -docker run -p 8080:8080 \ - -e POSTGRES_USER=postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=banners \ - -e DB_HOST=host.docker.internal \ - -e PGPORT=5432 \ - -e GITHUB_TOKENS=your_github_token \ - -e RENDERER_BASE_URL=http://renderer:3000/ \ - -e STORAGE_BASE_URL=http://storage:3001/ \ - -e SERVICES_SECRET_KEY=your-secret-key \ - github-banners-api -``` - -### With Docker Compose - -```yaml -version: '3.8' -services: - postgres: - image: postgres:14 - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: banners - ports: - - "5432:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - - api: - build: . - ports: - - "8080:8080" - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: banners - DB_HOST: postgres - PGPORT: "5432" - GITHUB_TOKENS: your_token_here - LOG_LEVEL: DEBUG - RENDERER_BASE_URL: http://renderer:3000/ - STORAGE_BASE_URL: http://storage:3001/ - SERVICES_SECRET_KEY: your-secret-key - depends_on: - - postgres - - renderer: - image: your-renderer-image - ports: - - "3000:3000" - - storage: - image: your-storage-image - ports: - - "3001:3001" - -volumes: - postgres_data: -``` - -## Database Migrations - -Migrations run automatically on startup using Goose (embedded SQL files). Manual commands: - -```bash -# Install Goose CLI -go install github.com/pressly/goose/v3/cmd/goose@latest - -# Apply migrations -goose postgres "postgres://user:pass@host:port/db" up - -# Rollback last migration -goose postgres "postgres://user:pass@host:port/db" down - -# Create new migration -goose -dir internal/migrations create migration_name sql -``` - -### Migration Files - -| File | Description | -|------|-------------| -| `001_create_users_table.sql` | Stores GitHub user profile data | -| `002_create_repositories_table.sql` | Stores user repositories data | -| `003_create_banners_table.sql` | Stores banner metadata (for future use) | - -## Testing - -### Run All Tests - -```bash -go test ./... -``` - -### Run Tests with Coverage - -```bash -go test -cover ./... -go test -coverprofile=coverage.out ./... -go tool cover -html=coverage.out -``` - -### Run Specific Package Tests - -```bash -# Test user stats service -go test ./internal/domain/user_stats/... - -# Test preview usecase -go test ./internal/domain/preview/... - -# Test repository -go test ./internal/repo/github_user_data/... - -# Test renderer infrastructure -go test ./internal/infrastructure/renderer/... -``` - -### Run with Verbose Output - -```bash -go test -v ./... -``` - -## API Usage - -### Get Banner Preview - -```bash -# Basic request -curl "http://localhost:8080/banners/preview/?username=torvalds&type=wide" - -# Response: SVG image (Content-Type: image/svg+xml) -``` - -### Query Parameters - -| Parameter | Required | Description | -|-----------|----------|-------------| -| `username` | Yes | GitHub username | -| `type` | Yes | Banner type (currently only "wide" supported) | - -### Error Responses - -```bash -# Invalid banner type -curl "http://localhost:8080/banners/preview/?username=torvalds&type=invalid" -# Response: {"error": "invalid banner type"} - -# User not found -curl "http://localhost:8080/banners/preview/?username=nonexistentuser12345&type=wide" -# Response: {"error": "user doesn't exist"} - -# Missing parameters -curl "http://localhost:8080/banners/preview/?username=torvalds" -# Response: {"error": "invalid inputs"} -``` - -### HTTP Status Codes - -| Code | Description | -|------|-------------| -| 200 | Success - returns SVG banner | -| 400 | Bad request - invalid parameters or banner type | -| 404 | Not found - user doesn't exist | -| 500 | Internal error - can't get preview (service unavailable) | -| 501 | Not implemented - POST /banners/ endpoint | - -## Caching Strategy - -### Stats Cache (User Statistics) - -- **Storage**: In-memory (`patrickmn/go-cache`) -- **Soft TTL**: 10 minutes - data considered fresh -- **Hard TTL**: 24 hours - maximum cache lifetime -- **Behavior**: - - Fresh data: returned immediately - - Stale data: returned immediately, async refresh triggered - - Cache miss: check DB, then GitHub API - -### Preview Cache (Rendered Banners) - -- **Storage**: In-memory with hash-based keys -- **Key**: Hash of BannerInfo (username + type + stats, excludes FetchedAt) -- **Purpose**: Deduplicate identical banner requests -- **Singleflight**: Prevents thundering herd on cache miss - -## Background Worker - -The stats worker runs periodically and refreshes cached user data: - -- **Default interval**: 1 hour -- **Batch size**: 1 (configurable) -- **Concurrency**: 1 worker (configurable) -- **Process**: Fetches all usernames from DB, refreshes each from GitHub API - -Configuration in `main.go`: -```go -statsWorker := user_stats_worker.NewStatsWorker( - statsService.RefreshAll, - time.Hour, // interval - logger, - userstats.WorkerConfig{ - BatchSize: 1, - Concurrency: 1, - CacheTTL: time.Hour, - }, -) -``` - -## Known Limitations - -1. **POST /banners/** - Not implemented (returns 501) -2. **Kafka integration** - Code exists but not wired up in main.go -3. **Rate limiting** - Not currently enforced per-request -4. **Health endpoint** - Not implemented -5. **Storage client** - Initialized but not used (for future banner persistence) - -## Troubleshooting - -### Database Connection Issues - -```bash -# Check PostgreSQL is running -pg_isready -h localhost -p 5432 - -# Test connection -psql -h localhost -U postgres -d banners -``` - -### GitHub API Rate Limit - -- Multiple tokens can be configured for rate limit distribution -- Tokens rotate automatically based on remaining quota -- Check remaining quota via GitHub API: `curl -H "Authorization: token YOUR_TOKEN" https://api.github.com/rate_limit` -- Fetcher logs token status on startup - -### Renderer Service Issues - -- Verify `RENDERER_BASE_URL` is correct and accessible -- Check `SERVICES_SECRET_KEY` matches between services -- Renderer must accept HMAC-signed requests - -### Cache Issues - -- Cache is in-memory only (lost on restart) -- TTL configurable via `CACHE_TTL` -- Clear by restarting the service - -## Development - -### Project Structure Convention - -This project follows standard Go project layout: -- `main.go` - Application entry point -- `internal/` - Private application code - - `domain/` - Business logic (entities, services) - - `handlers/` - HTTP handlers - - `infrastructure/` - External integrations - - `repo/` - Data repositories - - `cache/` - Caching layer - - `config/` - Configuration - -### Adding New Banner Types - -1. Add type to `internal/domain/banner.go`: - ```go - const ( - TypeWide BannerType = iota - TypeNarrow // Add new type - ) - ``` - -2. Update mappings: - ```go - var BannerTypes = map[string]BannerType{ - "wide": TypeWide, - "narrow": TypeNarrow, - } - ``` - -3. Update renderer service to handle new type - -### Adding New Endpoints - -1. Create handler in `internal/handlers/` -2. Register route in `main.go` -3. Add to `docs/api.yaml` - -### Error Handling - -Domain errors are defined in `internal/domain/errors.go`: -- `ErrNotFound` - Resource not found -- `ErrUnavailable` - Service unavailable -- `ConflictError` - Conflict with current state - -Preview errors in `internal/domain/preview/errors.go`: -- `ErrInvalidBannerType` -- `ErrUserDoesntExist` -- `ErrInvalidInputs` -- `ErrCantGetPreview` \ No newline at end of file diff --git a/api/go.mod b/api/go.mod index 616e589..5b9bd7b 100644 --- a/api/go.mod +++ b/api/go.mod @@ -16,16 +16,35 @@ require ( github.com/stretchr/testify v1.11.1 go.uber.org/mock v0.6.0 golang.org/x/oauth2 v0.34.0 - golang.org/x/sync v0.17.0 + golang.org/x/sync v0.19.0 ) require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.5.2+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/ebitengine/purego v0.10.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect @@ -35,16 +54,43 @@ require ( github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.18.1 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/kr/text v0.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.10 // indirect github.com/mfridman/interpolate v0.0.2 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect + github.com/shirou/gopsutil/v4 v4.26.2 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/testcontainers/testcontainers-go v0.41.0 // indirect + github.com/testcontainers/testcontainers-go/modules/postgres v0.41.0 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.43.0 // indirect - golang.org/x/net v0.46.0 // indirect - golang.org/x/text v0.30.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/api/go.sum b/api/go.sum index 8578888..5b5c013 100644 --- a/api/go.sum +++ b/api/go.sum @@ -1,13 +1,39 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/IBM/sarama v1.46.3 h1:njRsX6jNlnR+ClJ8XmkO+CM4unbrNr/2vB5KK6UA+IE= github.com/IBM/sarama v1.46.3/go.mod h1:GTUYiF9DMOZVe3FwyGT+dtSPceGFIgA+sPc5u6CBwko= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= @@ -16,13 +42,25 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-chi/chi/v5 v5.2.4 h1:WtFKPHwlywe8Srng8j2BhOD9312j9cGUxG1SP4V2cR4= github.com/go-chi/chi/v5 v5.2.4/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v81 v81.0.0 h1:hTLugQRxSLD1Yei18fk4A5eYjOGLUBKAl/VCqOfFkZc= @@ -61,24 +99,54 @@ github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJk github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/maxatome/go-testdeep v1.14.0 h1:rRlLv1+kI8eOI3OaBXZwb3O7xY3exRzdW5QyX48g9wI= github.com/maxatome/go-testdeep v1.14.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM= github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY= github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg= @@ -91,6 +159,10 @@ github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= +github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI= +github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -102,7 +174,27 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.41.0 h1:mfpsD0D36YgkxGj2LrIyxuwQ9i2wCKAD+ESsYM1wais= +github.com/testcontainers/testcontainers-go v0.41.0/go.mod h1:pdFrEIfaPl24zmBjerWTTYaY0M6UHsqA1YSvsoU40MI= +github.com/testcontainers/testcontainers-go/modules/postgres v0.41.0 h1:AOtFXssrDlLm84A2sTTR/AhvJiYbrIuCO59d+Ro9Tb0= +github.com/testcontainers/testcontainers-go/modules/postgres v0.41.0/go.mod h1:k2a09UKhgSp6vNpliIY0QSgm4Hi7GXVTzWvWgUemu/8= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -112,6 +204,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -123,20 +217,30 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -146,6 +250,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/api/internal/cache/preview.go b/api/internal/cache/preview.go index 682ce9f..c7cae59 100644 --- a/api/internal/cache/preview.go +++ b/api/internal/cache/preview.go @@ -26,8 +26,11 @@ func NewPreviewMemoryCache(ttl time.Duration) *PreviewMemoryCache { // Get gets rendered banner from cache and returns it // Second return is hash, that will be the same for same bannerInfo ( excluding FetchedAt field, for different FetchedAt fields it will be same, if other fields are the same) -// Third return is found, if found is false, then banner pointer is nil ( but hash is valid ) +// Third return is found: if found is false -> banner pointer is nil ( but hash is valid ) +// It uses lower case of username in cache key, so hurtki and HURTKI as usernames are same func (c *PreviewMemoryCache) Get(bf domain.BannerInfo) (*domain.Banner, string, bool) { + bf.Username = domain.NormalizeGithubUsername(bf.Username) + hashKey := c.hashCounter.Hash(bf) if item, found := c.cache.Get(hashKey); found { if banner, ok := item.(*domain.Banner); ok { diff --git a/api/internal/cache/stats.go b/api/internal/cache/stats.go index eb9b926..5262cec 100644 --- a/api/internal/cache/stats.go +++ b/api/internal/cache/stats.go @@ -3,10 +3,13 @@ package cache import ( "time" + "github.com/hurtki/github-banners/api/internal/domain" userstats "github.com/hurtki/github-banners/api/internal/domain/user_stats" "github.com/patrickmn/go-cache" ) +// StatsMemoryCache is in memory cache for storing statistics for username +// It uses lower case of username as cache key, so hurtki and HURTKI are same type StatsMemoryCache struct { cache *cache.Cache } @@ -18,7 +21,9 @@ func NewStatsMemoryCache(defaultTTL time.Duration) *StatsMemoryCache { } func (c *StatsMemoryCache) Get(username string) (*userstats.CachedStats, bool) { - if item, found := c.cache.Get(username); found { + normalizedUsername := domain.NormalizeGithubUsername(username) + + if item, found := c.cache.Get(normalizedUsername); found { if stats, ok := item.(*userstats.CachedStats); ok { return stats, true } @@ -27,7 +32,8 @@ func (c *StatsMemoryCache) Get(username string) (*userstats.CachedStats, bool) { } func (c *StatsMemoryCache) Set(username string, entry *userstats.CachedStats, ttl time.Duration) { - c.cache.Set(username, entry, ttl) + normalizedUsername := domain.NormalizeGithubUsername(username) + c.cache.Set(normalizedUsername, entry, ttl) } func (c *StatsMemoryCache) Delete(username string) { diff --git a/api/internal/domain/gh-username.go b/api/internal/domain/gh-username.go new file mode 100644 index 0000000..38d9c27 --- /dev/null +++ b/api/internal/domain/gh-username.go @@ -0,0 +1,10 @@ +package domain + +import "strings" + +// NormalizeGithubUsername is the only source of +// github username normalization +// used in repository level, migrations, cache level, domain surroundings +func NormalizeGithubUsername(username string) string { + return strings.ToLower(username) +} diff --git a/api/internal/domain/user_stats/service.go b/api/internal/domain/user_stats/service.go index 44f4160..11e93d1 100644 --- a/api/internal/domain/user_stats/service.go +++ b/api/internal/domain/user_stats/service.go @@ -26,23 +26,23 @@ func NewUserStatsService(repo GithubUserDataRepository, fetcher UserDataFetcher, func (s *UserStatsService) GetStats(ctx context.Context, username string) (domain.GithubUserStats, error) { cached, found := s.cache.Get(username) if found { - //fresh <10mins + // fresh <10mins age := time.Since(cached.UpdatedAt) if age <= SoftTTL { return cached.Stats, nil } - //stalte >10mins but <24 hours + // state >10mins but <24 hours go func() { - bgCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - _, _ = s.RecalculateAndSync(bgCtx, username) + _, _ = s.RecalculateAndSync(timeoutCtx, username) }() return cached.Stats, nil } - //checking database if cache missed - dbData, err := s.repo.GetUserData(context.TODO(), username) + // checking database if cache missed + dbData, err := s.repo.GetUserData(ctx, username) if err == nil { stats := CalculateStats(dbData.Repositories) stats.FetchedAt = dbData.FetchedAt @@ -58,20 +58,20 @@ func (s *UserStatsService) GetStats(ctx context.Context, username string) (domai // fetch api -> save db -> calc stats -> write cache func (s *UserStatsService) RecalculateAndSync(ctx context.Context, username string) (domain.GithubUserStats, error) { - //fetching raw data from github + // fetching raw data from github data, err := s.fetcher.FetchUserData(ctx, username) if err != nil { - return domain.GithubUserStats{}, err + return domain.GithubUserStats{}, fmt.Errorf("can't fetch data for user: %w", err) } stats := CalculateStats(data.Repositories) stats.FetchedAt = data.FetchedAt - //updating database with the new raw data - if err := s.repo.SaveUserData(context.TODO(), *data); err != nil { + // updating database with the new raw data + if err := s.repo.SaveUserData(ctx, *data); err != nil { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return domain.GithubUserStats{}, err } - // TODO, we really need to somehow log that we can't save to database or so something with it + // TODO: refactor stats service, to get away from this wrong apporoach } s.cache.Set(username, &CachedStats{ Stats: stats, diff --git a/api/internal/handlers/banners.go b/api/internal/handlers/banners.go index f6a967b..04165d8 100644 --- a/api/internal/handlers/banners.go +++ b/api/internal/handlers/banners.go @@ -39,13 +39,14 @@ func (h *BannersHandler) Preview(rw http.ResponseWriter, req *http.Request) { if err != nil { switch { case errors.Is(err, preview.ErrInvalidBannerType): - h.error(rw, http.StatusBadRequest, err.Error()) + h.error(rw, http.StatusBadRequest, "invalid banner type") case errors.Is(err, preview.ErrUserDoesntExist): - h.error(rw, http.StatusNotFound, err.Error()) + h.error(rw, http.StatusNotFound, "user not found on github") case errors.Is(err, preview.ErrInvalidInputs): - h.error(rw, http.StatusBadRequest, err.Error()) + h.error(rw, http.StatusBadRequest, "invalid inputs") case errors.Is(err, preview.ErrCantGetPreview): - h.error(rw, http.StatusInternalServerError, err.Error()) + h.logger.Error("failed to get preview", "err", err, "source", fn) + h.error(rw, http.StatusInternalServerError, "can't get preview") default: h.logger.Warn("unhandled error from usecase", "source", fn, "err", err) h.error(rw, http.StatusInternalServerError, "can't get preview") @@ -80,15 +81,15 @@ func (h *BannersHandler) Create(rw http.ResponseWriter, req *http.Request) { if err != nil { switch { case errors.Is(err, longterm.ErrUserDoesntExist): - h.error(rw, http.StatusNotFound, err.Error()) + h.error(rw, http.StatusNotFound, "user doesn't exist") case errors.Is(err, longterm.ErrInvalidBannerType): - h.error(rw, http.StatusBadRequest, err.Error()) + h.error(rw, http.StatusBadRequest, "invalid banner type") case errors.Is(err, longterm.ErrCantCreateBanner): h.logger.Error("failed to create long-term banner", "source", fn, "err", err) - h.error(rw, http.StatusInternalServerError, "Failed to process banner creation request") + h.error(rw, http.StatusInternalServerError, "can't create banner") default: h.logger.Warn("unhandled error from usecase", "source", fn, "err", err) - h.error(rw, http.StatusInternalServerError, "Failed to process banner creation request") + h.error(rw, http.StatusInternalServerError, "can't create banner") } return } diff --git a/api/internal/infrastructure/db/connection.go b/api/internal/infrastructure/db/connection.go index 77aa9bb..6317684 100644 --- a/api/internal/infrastructure/db/connection.go +++ b/api/internal/infrastructure/db/connection.go @@ -43,7 +43,8 @@ func NewDB(conf *config.PostgresConfig, logger logger.Logger) (*sql.DB, error) { continue } - if err := db.Ping(); err != nil { + err = db.Ping() + if err != nil { logger.Warn(fmt.Sprintf("Cannot ping database, try number: %d/%d", i+1, dataBaseConnectionTriesCount), "source", fn, "err", err) db.Close() time.Sleep(retryTimeBetweenTries) diff --git a/api/internal/migrations/004_username_normalized_github_data.go b/api/internal/migrations/004_username_normalized_github_data.go new file mode 100644 index 0000000..ba7b9d8 --- /dev/null +++ b/api/internal/migrations/004_username_normalized_github_data.go @@ -0,0 +1,170 @@ +package migrations + +import ( + "context" + "database/sql" + + "github.com/hurtki/github-banners/api/internal/domain" + "github.com/pressly/goose/v3" +) + +func init() { + goose.AddMigrationContext(upUsernameNormalizedGithubData, downUsernameNormalizedGithubData) +} + +func upUsernameNormalizedGithubData(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, ` +-- deleting foreign key constraint from repositories table +-- to escape conflicts, because now we will change users table +alter table repositories +drop constraint fk_repository_owner; + +alter table users +drop constraint users_pkey; + +drop index if exists idx_users_username; + +alter table users +add column username_normalized text; +`) + if err != nil { + return err + } + + err = normalizeStringRow(ctx, tx, "users", "username", "username_normalized", domain.NormalizeGithubUsername) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, ` +delete from users a +using users b +where a.username_normalized = b.username_normalized +and a.ctid > b.ctid; + +alter table users +alter column username_normalized set not null; + +alter table users +add constraint users_pkey primary key (username_normalized); + +alter table users +alter column username set not null; + +create index idx_users_username_normalized on users(username_normalized); + +/* +before: +username text primary key + +after: +username_normalized text primary key +username text not null +*/ + +-- normalize repositories table +alter table repositories +add column owner_username_normalized text; +`) + + err = normalizeStringRow(ctx, tx, "repositories", "owner_username", "owner_username_normalized", domain.NormalizeGithubUsername) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, ` +-- now delete ownwer_username column +-- real username should be stored in users table, repositories table can't contain this +drop index if exists idx_repositories_owner_username; +alter table repositories +drop column owner_username; + +alter table repositories +alter column owner_username_normalized set not null; + +alter table repositories +add constraint fk_repository_owner + foreign key (owner_username_normalized) + references users(username_normalized) on delete cascade; + +CREATE INDEX idx_repositories_owner_username ON repositories(owner_username_normalized); + +/* +before: +owner_username text not null +fk constraint + +after: +owner_username_normalized text not null +fk constraint +*/ + +-- create schema for better separation of gihub data and our service data +-- cause right now "users" +create schema github_data; + +alter table users +set schema github_data; + +alter table repositories +set schema github_data; + `) + return err +} + +func downUsernameNormalizedGithubData(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, ` +-- drop foreign key from repositories +alter table github_data.repositories +drop constraint fk_repository_owner; + +-- revert owner_username_normalized back to owner_username +alter table github_data.repositories +add column owner_username text; + +update github_data.repositories r +set owner_username = u.username +from github_data.users u +where r.owner_username_normalized = u.username_normalized; + +drop index if exists idx_repositories_owner_username; +create index idx_repositories_owner_username on github_data.repositories(owner_username); + +alter table github_data.repositories +drop column owner_username_normalized; + +-- revert users table primary key +alter table github_data.users +drop constraint users_pkey; + +alter table github_data.users +drop column username_normalized; + +-- recreate original primary key on username +alter table github_data.users +add constraint users_pkey primary key (username); + +alter table github_data.users +alter column username set not null; + +-- recreate index if needed +create index idx_users_username on github_data.users(username); + +-- restore foreign key on repositories.owner_username +alter table github_data.repositories +add constraint fk_repository_owner + foreign key (owner_username) + references github_data.users(username) on delete cascade; + +-- move tables back to original schema (public) +alter table github_data.users +set schema public; + +alter table github_data.repositories +set schema public; + +-- optional: drop schema github_data if empty +-- drop schema github_data; + `) + return err +} diff --git a/api/internal/migrations/005_username_normalized_banners.go b/api/internal/migrations/005_username_normalized_banners.go new file mode 100644 index 0000000..d3dd7a2 --- /dev/null +++ b/api/internal/migrations/005_username_normalized_banners.go @@ -0,0 +1,83 @@ +package migrations + +import ( + "context" + "database/sql" + + "github.com/hurtki/github-banners/api/internal/domain" + "github.com/pressly/goose/v3" +) + +func init() { + goose.AddMigrationContext(upUsernameNormalizedBanners, downUsernameNormalizedBanners) +} + +func upUsernameNormalizedBanners(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, ` +alter table banners +drop constraint banners_github_username_banner_type_key; + +alter table banners +add column github_username_normalized text; +`) + + if err != nil { + return err + } + + err = normalizeStringRow(ctx, tx, "banners", "github_username", "github_username_normalized", domain.NormalizeGithubUsername) + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, ` +alter table banners +alter column github_username_normalized set not null; + +alter table banners +drop column github_username; + +-- deduplicate +delete from banners a +using banners b +-- same lowered ( normalized username ) +where a.github_username_normalized = b.github_username_normalized +-- same banner type +and a.banner_type = b.banner_type; +-- their ctids are different ( different rows ) +and a.ctid > b.ctid + +alter table banners +add constraint banners_github_username_normalized_banner_type_key unique (github_username_normalized, banner_type); + +create index idx_banners_username_normalized +on banners (github_username_normalized); + `) + return err +} + +func downUsernameNormalizedBanners(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, ` +alter table banners +drop constraint banners_github_username_normalized_banner_type_key; + +drop index if exists idx_banners_username_normalized; + +alter table banners +add column github_username text; + +update banners +set github_username = github_username_normalized; + +alter table banners +alter column github_username set not null; + +alter table banners +drop column github_username_normalized; + +alter table banners +add constraint banners_github_username_banner_type_key +unique (github_username, banner_type); + `) + return err +} diff --git a/api/internal/migrations/gh_username_normalization_test.go b/api/internal/migrations/gh_username_normalization_test.go new file mode 100644 index 0000000..486e33e --- /dev/null +++ b/api/internal/migrations/gh_username_normalization_test.go @@ -0,0 +1,101 @@ +package migrations + +import ( + "log" + "testing" + + "github.com/hurtki/github-banners/api/internal/config" + "github.com/hurtki/github-banners/api/internal/infrastructure/db" + "github.com/hurtki/github-banners/api/internal/logger" + "github.com/pressly/goose/v3" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/postgres" +) + +func Test004Success(t *testing.T) { + + dbName := "users" + dbUser := "user" + dbPassword := "password" + + postgresContainer, err := postgres.Run(t.Context(), + "postgres:15", + postgres.WithDatabase(dbName), + postgres.WithUsername(dbUser), + postgres.WithPassword(dbPassword), + postgres.BasicWaitStrategies(), + ) + + defer func() { + if err := testcontainers.TerminateContainer(postgresContainer); err != nil { + log.Printf("failed to terminate container: %s", err) + } + }() + if err != nil { + log.Printf("failed to start container: %s", err) + return + } + host, err := postgresContainer.Host(t.Context()) + require.NoError(t, err) + + port, err := postgresContainer.MappedPort(t.Context(), "5432") + require.NoError(t, err) + + cfg := config.PostgresConfig{User: dbUser, DBName: dbName, Password: dbPassword, DBHost: host, DBPort: port.Port()} + + logger := logger.NewLogger("ERROR", "TEXT") + + db, err := db.NewDB(&cfg, logger) + require.NoError(t, err) + + err = goose.UpTo(db, ".", 3) + require.NoError(t, err) + + _, err = db.ExecContext(t.Context(), `INSERT INTO users ( + username, + name, + company, + location, + bio, + public_repos_count, + followers_count, + following_count, + fetched_at +) VALUES +('hurtki', 'Ivan Petrov', 'Example Corp', 'Haifa, Israel', 'Backend developer working with Go', 42, 120, 75, NOW()), +('HURTKI', 'Ivan Petrov', 'Example Corp', 'Haifa, Israel', 'Uppercase username test', 42, 120, 75, NOW()), +('HurtKi', 'Ivan Petrov', 'Example Corp', 'Haifa, Israel', 'Mixed case username test', 42, 120, 75, NOW()), +('johnDoe', 'John Doe', 'Acme Inc', 'New York, USA', 'Software engineer', 15, 80, 20, NOW()), +('JOHNDOE', 'John Doe', 'Acme Inc', 'New York, USA', 'Uppercase duplicate test', 15, 80, 20, NOW()), +('JaneSmith', 'Jane Smith', 'TechSoft', 'London, UK', 'Full-stack developer', 27, 150, 60, NOW()), +('janesmith', 'Jane Smith', 'TechSoft', 'London, UK', 'Lowercase duplicate test', 27, 150, 60, NOW()), +('DEV_GUY', 'Alex Brown', 'Startup Labs', 'Berlin, Germany', 'Open source contributor', 9, 40, 10, NOW()), +('dev_guy', 'Alex Brown', 'Startup Labs', 'Berlin, Germany', 'Case variant with underscore', 9, 40, 10, NOW()), +('SomeUser123', 'Chris White', 'DataWorks', 'Toronto, Canada', 'Data engineer', 33, 95, 44, NOW()), +('someuser123', 'Chris White', 'DataWorks', 'Toronto, Canada', 'Case duplicate with numbers', 33, 95, 44, NOW()), +('MiXeDCaSeUser', 'Taylor Green', 'CloudNine', 'Sydney, Australia', 'Cloud infrastructure engineer', 21, 60, 18, NOW());`) + require.NoError(t, err) + + err = goose.UpTo(db, ".", 4) + require.NoError(t, err) + + rows, err := db.QueryContext(t.Context(), ` + SELECT username_normalized + FROM github_data.users + WHERE username_normalized != lower(username); +`) + require.NoError(t, err) + defer rows.Close() + + var invalid []string + for rows.Next() { + var u string + err := rows.Scan(&u) + require.NoError(t, err) + invalid = append(invalid, u) + } + + require.NoError(t, rows.Err()) + require.Empty(t, invalid, "all usernames must be normalized to lowercase") +} diff --git a/api/internal/migrations/normalize_string_row.go b/api/internal/migrations/normalize_string_row.go new file mode 100644 index 0000000..323be5e --- /dev/null +++ b/api/internal/migrations/normalize_string_row.go @@ -0,0 +1,82 @@ +package migrations + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +type stringNormalizationEntry struct { + srcRow string + normalizedRow string +} + +func normalizeStringRow( + ctx context.Context, + tx *sql.Tx, + tableName string, + srcRowName string, + destRowName string, + normalizeFunc func(string) string, +) error { + rows, err := tx.QueryContext(ctx, fmt.Sprintf("select %s from %s", srcRowName, tableName)) + if err != nil { + return err + } + defer rows.Close() + + entries := make([]stringNormalizationEntry, 0) + + for rows.Next() { + var entry stringNormalizationEntry + if err := rows.Scan(&entry.srcRow); err != nil { + return err + } + entry.normalizedRow = normalizeFunc(entry.srcRow) + entries = append(entries, entry) + } + + if err := rows.Err(); err != nil { + return err + } + + batchLength := 32767 + // max postgres positiona args count: 65535 ( / 2 = 32767.5 ) + // we are having two pos args per entry => 32767 optimal + for i := 0; i < len(entries); i += batchLength { + end := min(i+batchLength, len(entries)) + + chunk := entries[i:end] + + // process chunk + + if len(chunk) == 0 { + return nil + } + var posArgs []string + var values []any + + for i, e := range chunk { + posArgs = append(posArgs, fmt.Sprintf("($%d, $%d)", (i*2)+1, (i*2)+2)) + values = append(values, e.srcRow, e.normalizedRow) + } + + query := fmt.Sprintf(` + update %s as t + set %s = v.normalized + from (values %s) as v(src, normalized) + where t.%s = v.src; + `, tableName, destRowName, strings.Join(posArgs, ","), srcRowName) + + _, err := tx.ExecContext(ctx, query, values...) + + // process end + + if err != nil { + return err + } + } + return nil + +} diff --git a/api/internal/repo/banners/postgres_queries.go b/api/internal/repo/banners/postgres_queries.go index 35134b8..37f8c57 100644 --- a/api/internal/repo/banners/postgres_queries.go +++ b/api/internal/repo/banners/postgres_queries.go @@ -12,7 +12,7 @@ import ( func (r *PostgresRepo) GetActiveBanners(ctx context.Context) ([]domain.LTBannerMetadata, error) { fn := "internal.repo.banners.PostgresRepo.GetActiveBanners" - const q = `select github_username, banner_type, storage_path from banners where is_active = true` + const q = `select github_username_normalized, banner_type, storage_path from banners where is_active = true` rows, err := r.db.QueryContext(ctx, q) if err != nil { r.logger.Error("unexpected error when querying banners", "source", fn, "err", err) @@ -67,14 +67,14 @@ func (r *PostgresRepo) SaveBanner(ctx context.Context, b domain.LTBannerMetadata } const q = ` - insert into banners (github_username, banner_type, storage_path, is_active) + insert into banners (github_username_normalized, banner_type, storage_path, is_active) values ($1, $2, $3, $4) - on conflict (github_username, banner_type) do update set + on conflict (github_username_normalized, banner_type) do update set is_active = EXCLUDED.is_active, storage_path = EXCLUDED.storage_path; ` - _, err = r.db.ExecContext(ctx, q, b.Username, btStr, b.UrlPath, b.Active) + _, err = r.db.ExecContext(ctx, q, domain.NormalizeGithubUsername(b.Username), btStr, b.UrlPath, b.Active) if err != nil { if strings.Contains(err.Error(), "duplicate key") || strings.Contains(err.Error(), "unique constraint") { @@ -95,9 +95,9 @@ func (r *PostgresRepo) DeactivateBanner(ctx context.Context, githubUsername stri const q = ` update banners set is_active = false - where github_username = $1 and banner_type = $2 and is_active = true` + where github_username_normalized = $1 and banner_type = $2 and is_active = true` - res, err := r.db.ExecContext(ctx, q, githubUsername, domain.BannerTypesBackward[bannerType]) + res, err := r.db.ExecContext(ctx, q, domain.NormalizeGithubUsername(githubUsername), domain.BannerTypesBackward[bannerType]) if err != nil { r.logger.Error("unexpected error when deactivating banner", "source", fn, "err", err) return repoerr.ErrRepoInternal{Note: err.Error()} @@ -119,10 +119,10 @@ func (r *PostgresRepo) GetBanner(ctx context.Context, githubUsername string, ban fn := "internal.repo.banners.PostgresRepo.GetBanner" const q = ` select storage_path, is_active from banners - where github_username = $1 and banner_type = $2;` + where github_username_normalized = $1 and banner_type = $2;` meta := domain.LTBannerMetadata{Username: githubUsername, BannerType: bannerType} - err := r.db.QueryRowContext(ctx, q, githubUsername, domain.BannerTypesBackward[bannerType]).Scan(&meta.UrlPath, &meta.Active) + err := r.db.QueryRowContext(ctx, q, domain.NormalizeGithubUsername(githubUsername), domain.BannerTypesBackward[bannerType]).Scan(&meta.UrlPath, &meta.Active) if err != nil { if errors.Is(err, sql.ErrNoRows) { return domain.LTBannerMetadata{}, repoerr.ErrNothingFound diff --git a/api/internal/repo/github_user_data/get.go b/api/internal/repo/github_user_data/get.go index 84254c6..e53e36b 100644 --- a/api/internal/repo/github_user_data/get.go +++ b/api/internal/repo/github_user_data/get.go @@ -33,9 +33,9 @@ func (r *GithubDataPsgrRepo) GetUserData(ctx context.Context, username string) ( }() row := tx.QueryRowContext(ctx, ` - select username, name, company, location, bio, public_repos_count, followers_count, following_count, fetched_at from users - where username = $1; - `, username) + select username, name, company, location, bio, public_repos_count, followers_count, following_count, fetched_at from github_data.users + where username_normalized = $1; + `, domain.NormalizeGithubUsername(username)) data := domain.GithubUserData{} @@ -46,9 +46,9 @@ func (r *GithubDataPsgrRepo) GetUserData(ctx context.Context, username string) ( } rows, err := tx.QueryContext(ctx, ` - select github_id, owner_username, pushed_at, updated_at, language, stars_count, is_fork, forks_count from repositories - where owner_username = $1; - `, username) + select github_id, pushed_at, updated_at, language, stars_count, is_fork, forks_count from github_data.repositories + where owner_username_normalized = $1; + `, domain.NormalizeGithubUsername(username)) if err != nil { return domain.GithubUserData{}, r.handleError(err, fn+".selectRepositoriesQuery") @@ -58,10 +58,11 @@ func (r *GithubDataPsgrRepo) GetUserData(ctx context.Context, username string) ( for rows.Next() { githubRepo := domain.GithubRepository{} - err = rows.Scan(&githubRepo.ID, &githubRepo.OwnerUsername, &githubRepo.PushedAt, &githubRepo.UpdatedAt, &githubRepo.Language, &githubRepo.StarsCount, &githubRepo.Fork, &githubRepo.ForksCount) + err = rows.Scan(&githubRepo.ID, &githubRepo.PushedAt, &githubRepo.UpdatedAt, &githubRepo.Language, &githubRepo.StarsCount, &githubRepo.Fork, &githubRepo.ForksCount) if err != nil { return domain.GithubUserData{}, r.handleError(err, fn+".scanRepositoryRow") } + githubRepo.OwnerUsername = data.Username githubRepos = append(githubRepos, githubRepo) } diff --git a/api/internal/repo/github_user_data/repos_upsert.go b/api/internal/repo/github_user_data/repos_upsert.go index aed497b..42ff15c 100644 --- a/api/internal/repo/github_user_data/repos_upsert.go +++ b/api/internal/repo/github_user_data/repos_upsert.go @@ -31,7 +31,7 @@ func (r *GithubDataPsgrRepo) upsertRepoBatch(ctx context.Context, tx *sql.Tx, ba posParams = append(posParams, fmt.Sprintf("(%s)", strings.Join(tempPosArgs, ", "))) args = append(args, repo.ID, - repo.OwnerUsername, + domain.NormalizeGithubUsername(repo.OwnerUsername), repo.PushedAt, repo.UpdatedAt, repo.Language, @@ -43,10 +43,10 @@ func (r *GithubDataPsgrRepo) upsertRepoBatch(ctx context.Context, tx *sql.Tx, ba } query := fmt.Sprintf(` - insert into repositories (github_id, owner_username, pushed_at, updated_at, language, stars_count, is_fork, forks_count) + insert into github_data.repositories (github_id, owner_username_normalized, pushed_at, updated_at, language, stars_count, is_fork, forks_count) values %s on conflict (github_id) do update set - owner_username = excluded.owner_username, + owner_username_normalized = excluded.owner_username_normalized, pushed_at = excluded.pushed_at, updated_at = excluded.updated_at, language = excluded.language, diff --git a/api/internal/repo/github_user_data/save.go b/api/internal/repo/github_user_data/save.go index cf17baa..acfec9e 100644 --- a/api/internal/repo/github_user_data/save.go +++ b/api/internal/repo/github_user_data/save.go @@ -35,9 +35,10 @@ func (r *GithubDataPsgrRepo) SaveUserData(ctx context.Context, userData domain.G }() _, err = tx.ExecContext(ctx, ` - insert into users (username, name, company, location, bio, public_repos_count, followers_count, following_count, fetched_at) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9) - on conflict (username) do update set + insert into github_data.users (username, username_normalized, name, company, location, bio, public_repos_count, followers_count, following_count, fetched_at) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + on conflict (username_normalized) do update set + username = EXCLUDED.username, name = EXCLUDED.name, company = EXCLUDED.company, location = EXCLUDED.location, @@ -46,7 +47,7 @@ func (r *GithubDataPsgrRepo) SaveUserData(ctx context.Context, userData domain.G followers_count = EXCLUDED.followers_count, following_count = EXCLUDED.following_count, fetched_at = EXCLUDED.fetched_at; - `, userData.Username, userData.Name, userData.Company, userData.Location, userData.Bio, userData.PublicRepos, userData.Followers, userData.Following, userData.FetchedAt) + `, userData.Username, domain.NormalizeGithubUsername(userData.Username), userData.Name, userData.Company, userData.Location, userData.Bio, userData.PublicRepos, userData.Followers, userData.Following, userData.FetchedAt) if err != nil { return r.handleError(err, fn+".insertUser") } @@ -54,9 +55,9 @@ func (r *GithubDataPsgrRepo) SaveUserData(ctx context.Context, userData domain.G // if a new data says that there is no repositories, then delete all existing ones if len(userData.Repositories) == 0 { _, err := tx.ExecContext(ctx, ` - delete from repositories - where owner_username = $1; - `, userData.Username) + delete from github_data.repositories + where owner_username_normalized = $1; + `, domain.NormalizeGithubUsername(userData.Username)) if err != nil { return r.handleError(err, fn+".execDeleteAllRepositoriesFromUser") @@ -97,7 +98,7 @@ func (r *GithubDataPsgrRepo) SaveUserData(ctx context.Context, userData domain.G } deleteArgs := make([]any, len(userData.Repositories)+1) - deleteArgs[0] = userData.Username + deleteArgs[0] = domain.NormalizeGithubUsername(userData.Username) reposCount := len(userData.Repositories) deletePosParams := make([]string, reposCount) @@ -114,8 +115,8 @@ func (r *GithubDataPsgrRepo) SaveUserData(ctx context.Context, userData domain.G } deleteQuery := fmt.Sprintf(` - delete from repositories r - where r.owner_username = $1 + delete from github_data.repositories r + where r.owner_username_normalized = $1 and not exists ( select 1 from (values %s) as v(github_id) diff --git a/api/internal/repo/github_user_data/storage_test.go b/api/internal/repo/github_user_data/storage_test.go index c323239..18eef00 100644 --- a/api/internal/repo/github_user_data/storage_test.go +++ b/api/internal/repo/github_user_data/storage_test.go @@ -48,9 +48,10 @@ func TestSaveUserDataSucess(t *testing.T) { mock.ExpectBegin() mock.ExpectExec(` - insert into users (username, name, company, location, bio, public_repos_count, followers_count, following_count, fetched_at) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9) - on conflict (username) do update set + insert into github_data.users (username, username_normalized, name, company, location, bio, public_repos_count, followers_count, following_count, fetched_at) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + on conflict (username_normalized) do update set + username = EXCLUDED.username, name = EXCLUDED.name, company = EXCLUDED.company, location = EXCLUDED.location, @@ -59,30 +60,30 @@ func TestSaveUserDataSucess(t *testing.T) { followers_count = EXCLUDED.followers_count, following_count = EXCLUDED.following_count, fetched_at = EXCLUDED.fetched_at; - `).WithArgs(userData.Username, userData.Name, userData.Company, userData.Location, userData.Bio, userData.PublicRepos, userData.Followers, userData.Following, userData.FetchedAt).WillReturnResult(sqlmock.NewResult(1, 1)) + `).WithArgs(userData.Username, domain.NormalizeGithubUsername(userData.Username), userData.Name, userData.Company, userData.Location, userData.Bio, userData.PublicRepos, userData.Followers, userData.Following, userData.FetchedAt).WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectExec(` - insert into repositories (github_id, owner_username, pushed_at, updated_at, language, stars_count, is_fork, forks_count) + insert into github_data.repositories (github_id, owner_username_normalized, pushed_at, updated_at, language, stars_count, is_fork, forks_count) values ($1, $2, $3, $4, $5, $6, $7, $8), ($9, $10, $11, $12, $13, $14, $15, $16) on conflict (github_id) do update set - owner_username = excluded.owner_username, + owner_username_normalized = excluded.owner_username_normalized, pushed_at = excluded.pushed_at, updated_at = excluded.updated_at, language = excluded.language, stars_count = excluded.stars_count, is_fork = excluded.is_fork, forks_count = excluded.forks_count; - `).WithArgs(githubRepo1.ID, githubRepo1.OwnerUsername, githubRepo1.PushedAt, githubRepo1.UpdatedAt, githubRepo1.Language, githubRepo1.StarsCount, githubRepo1.Fork, githubRepo1.ForksCount, githubRepo2.ID, githubRepo2.OwnerUsername, githubRepo2.PushedAt, githubRepo2.UpdatedAt, githubRepo2.Language, githubRepo2.StarsCount, githubRepo2.Fork, githubRepo2.ForksCount).WillReturnResult(sqlmock.NewResult(1, 1)) + `).WithArgs(githubRepo1.ID, domain.NormalizeGithubUsername(githubRepo1.OwnerUsername), githubRepo1.PushedAt, githubRepo1.UpdatedAt, githubRepo1.Language, githubRepo1.StarsCount, githubRepo1.Fork, githubRepo1.ForksCount, githubRepo2.ID, domain.NormalizeGithubUsername(githubRepo2.OwnerUsername), githubRepo2.PushedAt, githubRepo2.UpdatedAt, githubRepo2.Language, githubRepo2.StarsCount, githubRepo2.Fork, githubRepo2.ForksCount).WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectExec(` - delete from repositories r - where r.owner_username = $1 + delete from github_data.repositories r + where r.owner_username_normalized = $1 and not exists ( select 1 from (values ($2::bigint), ($3::bigint)) as v(github_id) where v.github_id = r.github_id ); - `).WithArgs(userData.Username, githubRepo1.ID, githubRepo2.ID).WillReturnResult(sqlmock.NewResult(1, 1)) + `).WithArgs(domain.NormalizeGithubUsername(userData.Username), githubRepo1.ID, githubRepo2.ID).WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() @@ -101,9 +102,10 @@ func TestSaveUserDataSucessNoRepos(t *testing.T) { mock.ExpectBegin() mock.ExpectExec(` - insert into users (username, name, company, location, bio, public_repos_count, followers_count, following_count, fetched_at) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9) - on conflict (username) do update set + insert into github_data.users (username, username_normalized, name, company, location, bio, public_repos_count, followers_count, following_count, fetched_at) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + on conflict (username_normalized) do update set + username = EXCLUDED.username, name = EXCLUDED.name, company = EXCLUDED.company, location = EXCLUDED.location, @@ -112,12 +114,12 @@ func TestSaveUserDataSucessNoRepos(t *testing.T) { followers_count = EXCLUDED.followers_count, following_count = EXCLUDED.following_count, fetched_at = EXCLUDED.fetched_at; - `).WithArgs(userData.Username, userData.Name, userData.Company, userData.Location, userData.Bio, userData.PublicRepos, userData.Followers, userData.Following, userData.FetchedAt).WillReturnResult(sqlmock.NewResult(1, 1)) + `).WithArgs(userData.Username, domain.NormalizeGithubUsername(userData.Username), userData.Name, userData.Company, userData.Location, userData.Bio, userData.PublicRepos, userData.Followers, userData.Following, userData.FetchedAt).WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectExec(` - delete from repositories - where owner_username = $1; - `).WithArgs(userData.Username).WillReturnResult(sqlmock.NewResult(1, 1)) + delete from github_data.repositories + where owner_username_normalized = $1; + `).WithArgs(domain.NormalizeGithubUsername(userData.Username)).WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() @@ -134,7 +136,7 @@ func TestGetAllUsernamesSuccess(t *testing.T) { usernameRows.AddRow(un) } mock.ExpectQuery(` - select username from users; + select username from github_data.users; `).WillReturnRows(usernameRows) resUsernames, err := repo.GetAllUsernames(context.TODO()) @@ -147,7 +149,7 @@ func TestGetAllUsernamesNoUsernames(t *testing.T) { mock, repo := getMockAndRepo(t) mock.ExpectQuery(` - select username from users; + select username from github_data.users; `).WillReturnRows(sqlmock.NewRows([]string{"username"})) resUsernames, err := repo.GetAllUsernames(context.TODO()) @@ -159,16 +161,19 @@ func TestGetAllUsernamesNoUsernames(t *testing.T) { func TestGetUserDataSuccess(t *testing.T) { mock, repo := getMockAndRepo(t) - userData := domain.GithubUserData{Username: "Olivia"} + userData := domain.GithubUserData{Username: "OliVia"} repo1 := domain.GithubRepository{ID: 123, OwnerUsername: userData.Username} repo2 := domain.GithubRepository{ID: 3454, OwnerUsername: userData.Username} userData.Repositories = []domain.GithubRepository{repo1, repo2} + userColumns := []string{"username", "name", "company", "location", "bio", "public_repos_count", "followers_count", "following_count", "fetched_at"} - githubRepoColumns := []string{"github_id", "owner_username", "pushed_at", "updated_at", "language", "stars_count", "is_fork", "forks_count"} + + githubRepoColumns := []string{"github_id", "pushed_at", "updated_at", "language", "stars_count", "is_fork", "forks_count"} githubReposRows := sqlmock.NewRows(githubRepoColumns) + for _, githubRepo := range userData.Repositories { - githubReposRows.AddRow(githubRepo.ID, githubRepo.OwnerUsername, githubRepo.PushedAt, githubRepo.UpdatedAt, githubRepo.Language, githubRepo.StarsCount, githubRepo.Fork, githubRepo.ForksCount) + githubReposRows.AddRow(githubRepo.ID, githubRepo.PushedAt, githubRepo.UpdatedAt, githubRepo.Language, githubRepo.StarsCount, githubRepo.Fork, githubRepo.ForksCount) } userRows := sqlmock.NewRows(userColumns) @@ -176,14 +181,14 @@ func TestGetUserDataSuccess(t *testing.T) { mock.ExpectBegin() mock.ExpectQuery(` - select username, name, company, location, bio, public_repos_count, followers_count, following_count, fetched_at from users - where username = $1; - `).WithArgs(userData.Username).WillReturnRows(userRows) + select username, name, company, location, bio, public_repos_count, followers_count, following_count, fetched_at from github_data.users + where username_normalized = $1; + `).WithArgs(domain.NormalizeGithubUsername(userData.Username)).WillReturnRows(userRows) mock.ExpectQuery(` - select github_id, owner_username, pushed_at, updated_at, language, stars_count, is_fork, forks_count from repositories - where owner_username = $1; - `).WithArgs(userData.Username).WillReturnRows(githubReposRows) + select github_id, pushed_at, updated_at, language, stars_count, is_fork, forks_count from github_data.repositories + where owner_username_normalized = $1; + `).WithArgs(domain.NormalizeGithubUsername(userData.Username)).WillReturnRows(githubReposRows) mock.ExpectCommit() resUserData, err := repo.GetUserData(context.TODO(), userData.Username) diff --git a/api/internal/repo/github_user_data/usernames.go b/api/internal/repo/github_user_data/usernames.go index 399372a..08403d7 100644 --- a/api/internal/repo/github_user_data/usernames.go +++ b/api/internal/repo/github_user_data/usernames.go @@ -6,7 +6,7 @@ func (r *GithubDataPsgrRepo) GetAllUsernames(ctx context.Context) ([]string, err fn := "internal.repo.github_user_data.GithubDataPsgrRepo.GetAllUsernames" rows, err := r.db.QueryContext(ctx, ` - select username from users; + select username from github_data.users; `) if err != nil { diff --git a/api/main.go b/api/main.go index d7ee4b4..40d7810 100644 --- a/api/main.go +++ b/api/main.go @@ -67,10 +67,10 @@ func main() { logger.Error("failed to run migrations", "err", err.Error()) os.Exit(1) } - repo := github_data_repo.NewGithubDataPsgrRepo(db, logger) + githubDataRepo := github_data_repo.NewGithubDataPsgrRepo(db, logger) // Create stats service (domain service with cache) - statsService := userstats.NewUserStatsService(repo, githubFetcher, statsCache) + statsService := userstats.NewUserStatsService(githubDataRepo, githubFetcher, statsCache) router := chi.NewRouter() diff --git a/run_tests.sh b/run_tests.sh index 9a6ab22..0d65e1c 100644 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,7 +1,69 @@ #!/bin/sh -cd api -go test -count=1 ./... | grep -Ev "no test files|skipped" -cd ../renderer -go test -count=1 ./... | grep -Ev "no test files|skipped" -cd ../storage -go test -count=1 ./... | grep -Ev "no test files|skipped" + +ROOT_DIR=$(pwd) +RESULTS="/tmp/cov_$$.txt" +rm -f "$RESULTS" + +run_service() { + service=$1 + dir="${ROOT_DIR}/${service}" + printf "\n%s\n" "$service" + + if [ ! -d "$dir" ]; then + printf " not found\n" + echo "$service 0.0" >>"$RESULTS" + return + fi + + cd "$dir" + total=0 + count=0 + tmp="/tmp/gt_$$.txt" + + go test -count=1 -cover ./... 2>&1 | + grep -Ev 'no test files|skipped' >"$tmp" + + while IFS= read -r line; do + [ -z "$line" ] && continue + status=$(printf '%s' "$line" | awk '{print $1}') + pkg=$(printf '%s' "$line" | awk '{print $2}' | awk -F'/' '{print $NF}') + elapsed=$(printf '%s' "$line" | grep -o '[0-9]*\.[0-9]*s' | head -1) + cov=$(printf '%s' "$line" | grep -o 'coverage: [0-9]*\.[0-9]*' | awk '{print $2}') + [ -z "$elapsed" ] && elapsed="-" + [ -z "$cov" ] && cov="0.0" + + if [ "$status" = "ok" ]; then + printf " ok %-30s %6s %s%%\n" "$pkg" "$elapsed" "$cov" + total=$(awk -v a="$total" -v b="$cov" 'BEGIN{printf "%.4f",a+b}') + count=$((count + 1)) + elif [ "$status" = "FAIL" ]; then + printf " FAIL %-30s\n" "$pkg" + count=$((count + 1)) + fi + done <"$tmp" + rm -f "$tmp" + + avg="0.0" + [ "$count" -gt 0 ] && avg=$(awk -v t="$total" -v c="$count" 'BEGIN{printf "%.1f",t/c}') + printf " avg coverage: %s%%\n" "$avg" + + echo "$service $avg" >>"$RESULTS" + cd "$ROOT_DIR" +} + +run_service "api" +run_service "renderer" +run_service "storage" + +printf "\ncoverage summary\n" + +while IFS= read -r line; do + svc=$(printf '%s' "$line" | awk '{print $1}') + val=$(printf '%s' "$line" | awk '{print $2}') + printf " %-10s %s%%\n" "$svc" "$val" +done <"$RESULTS" + +overall=$(awk '{s+=$2; c++} END{printf "%.1f",s/c}' "$RESULTS") +printf " %-10s %s%%\n" "total" "$overall" + +rm -f "$RESULTS"