diff --git a/.github/workflows/cd-release.yml b/.github/workflows/cd-release.yml index 2b5797ad..2697b877 100644 --- a/.github/workflows/cd-release.yml +++ b/.github/workflows/cd-release.yml @@ -370,7 +370,8 @@ jobs: - name: Inject analytics configuration from GitHub Secrets env: - POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} + OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.OTEL_EXPORTER_OTLP_ENDPOINT }} + OTEL_EXPORTER_OTLP_TOKEN: ${{ secrets.OTEL_EXPORTER_OTLP_TOKEN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} run: | echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @@ -384,18 +385,17 @@ jobs: # Show file before injection echo "📄 File content BEFORE injection (showing placeholders):" echo "──────────────────────────────────────────────────────────────────" - grep -E "(POSTHOG_API_KEY_DEFAULT|SENTRY_DSN_DEFAULT)" app/config/analytics_defaults.py || true + grep -E "(OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT|OTEL_EXPORTER_OTLP_TOKEN_DEFAULT|SENTRY_DSN_DEFAULT)" app/config/analytics_defaults.py || true echo "──────────────────────────────────────────────────────────────────" echo "" # Verify secrets are available echo "🔍 Verifying GitHub Secrets availability..." - if [ -z "$POSTHOG_API_KEY" ]; then - echo "⚠️ POSTHOG_API_KEY secret not set (optional - analytics disabled in this build)" - echo " → To enable: Add POSTHOG_API_KEY secret (format: phc_xxxxx) in Settings → Secrets" + if [ -z "$OTEL_EXPORTER_OTLP_ENDPOINT" ] || [ -z "$OTEL_EXPORTER_OTLP_TOKEN" ]; then + echo "⚠️ Grafana OTLP secrets not fully set (telemetry sink disabled in this build)" + echo " → To enable: Add OTEL_EXPORTER_OTLP_ENDPOINT and OTEL_EXPORTER_OTLP_TOKEN in Settings → Secrets" else - echo "✅ POSTHOG_API_KEY secret found in GitHub Secret Store" - echo " → Format: ${POSTHOG_API_KEY:0:8}***${POSTHOG_API_KEY: -4} (${#POSTHOG_API_KEY} characters)" + echo "✅ Grafana OTLP secrets found in GitHub Secret Store" fi if [ -z "$SENTRY_DSN" ]; then @@ -409,7 +409,8 @@ jobs: # Perform replacement (use empty string if secrets not set) echo "🔧 Injecting secrets into application configuration..." - sed -i "s|%%POSTHOG_API_KEY_PLACEHOLDER%%|${POSTHOG_API_KEY:-}|g" app/config/analytics_defaults.py + sed -i "s|%%OTEL_EXPORTER_OTLP_ENDPOINT_PLACEHOLDER%%|${OTEL_EXPORTER_OTLP_ENDPOINT:-}|g" app/config/analytics_defaults.py + sed -i "s|%%OTEL_EXPORTER_OTLP_TOKEN_PLACEHOLDER%%|${OTEL_EXPORTER_OTLP_TOKEN:-}|g" app/config/analytics_defaults.py sed -i "s|%%SENTRY_DSN_PLACEHOLDER%%|${SENTRY_DSN:-}|g" app/config/analytics_defaults.py echo " → Placeholders replaced with secret values (or empty if not set)" echo "" @@ -417,7 +418,7 @@ jobs: # Show file after injection (redacted) echo "📄 File content AFTER injection (secrets redacted):" echo "──────────────────────────────────────────────────────────────────" - grep -E "(POSTHOG_API_KEY_DEFAULT|SENTRY_DSN_DEFAULT)" app/config/analytics_defaults.py | \ + grep -E "(OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT|OTEL_EXPORTER_OTLP_TOKEN_DEFAULT|SENTRY_DSN_DEFAULT)" app/config/analytics_defaults.py | \ sed 's/\(phc_[a-zA-Z0-9]\{8\}\)[a-zA-Z0-9]*\([a-zA-Z0-9]\{4\}\)/\1***\2/g' | \ sed 's|\(https://[^@]*@[^/]*\)|***REDACTED***|g' || true echo "──────────────────────────────────────────────────────────────────" @@ -425,12 +426,17 @@ jobs: # Verify placeholders were replaced echo "🔍 Verifying injection was successful..." - if grep -q "%%POSTHOG_API_KEY_PLACEHOLDER%%" app/config/analytics_defaults.py; then - echo "❌ ERROR: PostHog API key placeholder was NOT replaced!" - echo " The placeholder '%%POSTHOG_API_KEY_PLACEHOLDER%%' is still present in the file." + if grep -q "%%OTEL_EXPORTER_OTLP_ENDPOINT_PLACEHOLDER%%" app/config/analytics_defaults.py; then + echo "❌ ERROR: Grafana endpoint placeholder was NOT replaced!" exit 1 else - echo "✅ PostHog API key placeholder successfully replaced" + echo "✅ Grafana endpoint placeholder successfully replaced" + fi + if grep -q "%%OTEL_EXPORTER_OTLP_TOKEN_PLACEHOLDER%%" app/config/analytics_defaults.py; then + echo "❌ ERROR: Grafana token placeholder was NOT replaced!" + exit 1 + else + echo "✅ Grafana token placeholder successfully replaced" fi if grep -q "%%SENTRY_DSN_PLACEHOLDER%%" app/config/analytics_defaults.py; then @@ -441,19 +447,7 @@ jobs: echo "✅ Sentry DSN placeholder successfully replaced" fi - # Verify PostHog key format only when it was provided (non-empty) - if [ -n "$POSTHOG_API_KEY" ]; then - if ! grep -q 'POSTHOG_API_KEY_DEFAULT = "phc_' app/config/analytics_defaults.py; then - echo "❌ ERROR: PostHog API key format validation FAILED!" - echo " Expected format: phc_* (PostHog Cloud key)" - echo " Please verify the secret value in GitHub Settings." - exit 1 - else - echo "✅ PostHog API key format validated (phc_* pattern confirmed)" - fi - else - echo "✅ PostHog API key skipped (not configured - analytics disabled)" - fi + echo "✅ Grafana OTLP values injected" echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" @@ -461,10 +455,10 @@ jobs: echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" echo "📊 Injected Credentials Summary:" - if [ -n "$POSTHOG_API_KEY" ]; then - echo " • PostHog API Key: phc_***${POSTHOG_API_KEY: -4} ✓" + if [ -n "$OTEL_EXPORTER_OTLP_ENDPOINT" ] && [ -n "$OTEL_EXPORTER_OTLP_TOKEN" ]; then + echo " • Grafana OTLP: configured ✓" else - echo " • PostHog API Key: [Not configured - analytics disabled] ⚠️" + echo " • Grafana OTLP: [Not configured - telemetry sink disabled] ⚠️" fi if [ -n "$SENTRY_DSN" ]; then echo " • Sentry DSN: ${SENTRY_DSN:0:20}*** ✓" @@ -597,10 +591,9 @@ jobs: # See docs/analytics.md for configuration details - SENTRY_DSN=\${SENTRY_DSN:-} - SENTRY_TRACES_RATE=\${SENTRY_TRACES_RATE:-0.0} - - POSTHOG_API_KEY=\${POSTHOG_API_KEY:-} - - POSTHOG_HOST=\${POSTHOG_HOST:-https://app.posthog.com} + - OTEL_EXPORTER_OTLP_ENDPOINT=\${OTEL_EXPORTER_OTLP_ENDPOINT:-} + - OTEL_EXPORTER_OTLP_TOKEN=\${OTEL_EXPORTER_OTLP_TOKEN:-} - ENABLE_TELEMETRY=\${ENABLE_TELEMETRY:-false} - - TELE_URL=\${TELE_URL:-} - TELE_SALT=\${TELE_SALT:-8f4a7b2e9c1d6f3a5e8b4c7d2a9f6e3b1c8d5a7f2e9b4c6d3a8f5e1b7c4d9a2f} # Expose only internally; nginx publishes ports @@ -1337,7 +1330,7 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY echo "### 🔐 Analytics Configuration" >> $GITHUB_STEP_SUMMARY echo "Analytics credentials were **successfully injected** from GitHub Secret Store:" >> $GITHUB_STEP_SUMMARY - echo "- ✅ **PostHog API Key**: Injected from \`POSTHOG_API_KEY\` secret" >> $GITHUB_STEP_SUMMARY + echo "- ✅ **OTLP**: Injected from \`OTEL_EXPORTER_OTLP_ENDPOINT\` + \`OTEL_EXPORTER_OTLP_TOKEN\` secrets" >> $GITHUB_STEP_SUMMARY echo "- ✅ **Sentry DSN**: Injected from \`SENTRY_DSN\` secret" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "> 📍 **Secret Location**: Repository Settings → Secrets and variables → Actions" >> $GITHUB_STEP_SUMMARY diff --git a/README.md b/README.md index ab0a98ba..d8c9e273 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ TimeTracker is built with modern, reliable technologies: - **APScheduler** — Background task scheduling - **Prometheus Client** — Metrics collection - **Sentry SDK** — Error monitoring (optional) -- **PostHog** — Product analytics (optional) +- **Grafana OTLP** — Telemetry sink (optional) ### Development & Testing - **pytest** — Testing framework @@ -798,22 +798,21 @@ TimeTracker includes **optional** analytics and monitoring features to help impr - Helps identify and fix bugs quickly - **Opt-in:** Set `SENTRY_DSN` environment variable -#### 4. **Product Analytics** (Optional - PostHog) +#### 4. **Product Analytics** (Optional - Grafana OTLP) - Tracks feature usage and user behavior patterns with advanced features: - **Person Properties**: Role, auth method, login history - **Feature Flags**: Gradual rollouts, A/B testing, kill switches - **Group Analytics**: Segment by version, platform, deployment - **Rich Context**: Browser, device, environment on every event -- **Opt-in:** Set `POSTHOG_API_KEY` environment variable -- See [POSTHOG_ADVANCED_FEATURES.md](docs/admin/monitoring/POSTHOG_ADVANCED_FEATURES.md) for complete guide +- **Sink config:** Set `GRAFANA_OTLP_ENDPOINT` and `GRAFANA_OTLP_TOKEN` #### 5. **Installation Telemetry** (Optional, Anonymous) -- Sends anonymous installation data via PostHog with: +- Sends anonymous installation data via Grafana OTLP with: - Anonymized fingerprint (SHA-256 hash, cannot be reversed) - Application version - Platform information - **No PII:** No IP addresses, usernames, or business data -- **Opt-in:** Set `ENABLE_TELEMETRY=true` and `POSTHOG_API_KEY` environment variables +- **Opt-in:** Set `ENABLE_TELEMETRY=true` for detailed analytics; base telemetry remains anonymous ### How to Enable Analytics @@ -822,11 +821,11 @@ TimeTracker includes **optional** analytics and monitoring features to help impr SENTRY_DSN=https://your-sentry-dsn@sentry.io/project-id SENTRY_TRACES_RATE=0.1 # 10% sampling for performance traces -# Enable PostHog product analytics (optional) -POSTHOG_API_KEY=your-posthog-api-key -POSTHOG_HOST=https://app.posthog.com +# Configure Grafana Cloud OTLP sink (optional) +GRAFANA_OTLP_ENDPOINT=https://otlp-gateway-prod-eu-west-2.grafana.net/otlp/v1/logs +GRAFANA_OTLP_TOKEN=your-grafana-otlp-token -# Enable anonymous telemetry (optional, uses PostHog) +# Enable detailed analytics (optional) ENABLE_TELEMETRY=true TELE_SALT=your-unique-salt APP_VERSION=1.0.0 diff --git a/app/__init__.py b/app/__init__.py index e5f454f2..ecf6e8a7 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -7,7 +7,6 @@ from datetime import timedelta from urllib.parse import urlparse -import posthog import sentry_sdk from authlib.integrations.flask_client import OAuth from dotenv import load_dotenv @@ -658,18 +657,6 @@ def record_metrics_and_log(response): except Exception as e: app.logger.warning(f"Failed to initialize Sentry: {e}") - # Initialize PostHog for product analytics - # Priority: Env var > Built-in default > Disabled - posthog_api_key = analytics_config.get("posthog_api_key", "") - posthog_host = analytics_config.get("posthog_host", "https://app.posthog.com") - if posthog_api_key: - try: - posthog.project_api_key = posthog_api_key - posthog.host = posthog_host - app.logger.info(f"PostHog product analytics initialized (host: {posthog_host})") - except Exception as e: - app.logger.warning(f"Failed to initialize PostHog: {e}") - # Fail-fast on weak/missing secret in production # Skip validation in testing or debug mode is_testing = app.config.get("TESTING", False) diff --git a/app/config/analytics_defaults.py b/app/config/analytics_defaults.py index 36f58ab7..b1315bb9 100644 --- a/app/config/analytics_defaults.py +++ b/app/config/analytics_defaults.py @@ -14,10 +14,11 @@ DO NOT commit actual keys to this file - they are injected at build time only. """ -# PostHog Configuration -# Replaced by GitHub Actions: POSTHOG_API_KEY_PLACEHOLDER -POSTHOG_API_KEY_DEFAULT = "%%POSTHOG_API_KEY_PLACEHOLDER%%" -POSTHOG_HOST_DEFAULT = "https://us.i.posthog.com" +# OTEL OTLP Configuration +# Replaced by GitHub Actions: OTEL_EXPORTER_OTLP_ENDPOINT_PLACEHOLDER +# Replaced by GitHub Actions: OTEL_EXPORTER_OTLP_TOKEN_PLACEHOLDER +OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT = "%%OTEL_EXPORTER_OTLP_ENDPOINT_PLACEHOLDER%%" +OTEL_EXPORTER_OTLP_TOKEN_DEFAULT = "%%OTEL_EXPORTER_OTLP_TOKEN_PLACEHOLDER%%" # Sentry Configuration # Replaced by GitHub Actions: SENTRY_DSN_PLACEHOLDER @@ -128,8 +129,13 @@ def get_analytics_config(): def is_placeholder(value): return value.startswith("%%") and value.endswith("%%") - # PostHog configuration - use embedded keys (no override) - posthog_api_key = POSTHOG_API_KEY_DEFAULT if not is_placeholder(POSTHOG_API_KEY_DEFAULT) else "" + # OTEL OTLP configuration - use embedded values (no override) + otel_exporter_otlp_endpoint = ( + OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT if not is_placeholder(OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT) else "" + ) + otel_exporter_otlp_token = ( + OTEL_EXPORTER_OTLP_TOKEN_DEFAULT if not is_placeholder(OTEL_EXPORTER_OTLP_TOKEN_DEFAULT) else "" + ) # Sentry configuration - use embedded keys (no override) sentry_dsn = SENTRY_DSN_DEFAULT if not is_placeholder(SENTRY_DSN_DEFAULT) else "" @@ -141,8 +147,8 @@ def is_placeholder(value): # Users control telemetry via the opt-in/opt-out toggle in admin dashboard return { - "posthog_api_key": posthog_api_key, - "posthog_host": POSTHOG_HOST_DEFAULT, # Fixed host, no override + "otel_exporter_otlp_endpoint": otel_exporter_otlp_endpoint, + "otel_exporter_otlp_token": otel_exporter_otlp_token, "sentry_dsn": sentry_dsn, "sentry_traces_rate": float(SENTRY_TRACES_RATE_DEFAULT), # Fixed rate, no override "app_version": app_version, @@ -162,4 +168,6 @@ def is_placeholder(value): return value.startswith("%%") and value.endswith("%%") # Check if keys have been replaced during build - return not is_placeholder(POSTHOG_API_KEY_DEFAULT) + return (not is_placeholder(OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT)) and ( + not is_placeholder(OTEL_EXPORTER_OTLP_TOKEN_DEFAULT) + ) diff --git a/app/models/invoice.py b/app/models/invoice.py index 41923f44..ba271f67 100644 --- a/app/models/invoice.py +++ b/app/models/invoice.py @@ -2,6 +2,7 @@ from decimal import Decimal from app import db +from app.utils.invoice_numbering import generate_next_invoice_number class Invoice(db.Model): @@ -288,127 +289,7 @@ def to_dict(self): @classmethod def generate_invoice_number(cls): """Generate a unique invoice number""" - import json - import os - from datetime import datetime - - from app.models import Settings - - # Get settings for invoice prefix and start number - settings = Settings.get_settings() - # #region agent log - try: - log_data = { - "location": "invoice.py:291", - "message": "Settings.get_settings() returned", - "data": { - "settings_is_none": settings is None, - "settings_has_id": hasattr(settings, "id") and settings.id is not None, - "settings_type": type(settings).__name__, - }, - "timestamp": int(datetime.utcnow().timestamp() * 1000), - "sessionId": "debug-session", - "runId": "run1", - "hypothesisId": "A", - } - log_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".cursor", "debug.log") - with open(log_path, "a", encoding="utf-8") as f: - f.write(json.dumps(log_data) + "\n") - except (OSError, IOError, TypeError, ValueError): - pass - # #endregion - prefix = "INV" # Default prefix - start_number = 1 # Default start number - - if settings: - prefix_raw = getattr(settings, "invoice_prefix", "INV") - start_number_raw = getattr(settings, "invoice_start_number", 1) - # #region agent log - try: - log_data = { - "location": "invoice.py:296", - "message": "Retrieved settings values from object", - "data": { - "prefix_raw": str(prefix_raw), - "start_number_raw": str(start_number_raw), - "prefix_raw_type": type(prefix_raw).__name__, - "start_number_raw_type": type(start_number_raw).__name__, - }, - "timestamp": int(datetime.utcnow().timestamp() * 1000), - "sessionId": "debug-session", - "runId": "run1", - "hypothesisId": "B", - } - log_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".cursor", "debug.log" - ) - with open(log_path, "a", encoding="utf-8") as f: - f.write(json.dumps(log_data) + "\n") - except (OSError, IOError, TypeError, ValueError): - pass - # #endregion - prefix = prefix_raw or "INV" - start_number = start_number_raw or 1 - # Ensure start_number is a valid integer - try: - start_number = int(start_number) - if start_number < 1: - start_number = 1 - except (ValueError, TypeError): - start_number = 1 - # #region agent log - try: - log_data = { - "location": "invoice.py:304", - "message": "Final prefix and start_number values", - "data": { - "prefix": str(prefix), - "start_number": int(start_number), - "settings_was_none": settings is None, - }, - "timestamp": int(datetime.utcnow().timestamp() * 1000), - "sessionId": "debug-session", - "runId": "run1", - "hypothesisId": "C", - } - log_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), ".cursor", "debug.log") - with open(log_path, "a", encoding="utf-8") as f: - f.write(json.dumps(log_data) + "\n") - except (OSError, IOError, TypeError, ValueError): - pass - # #endregion - - # Format: {prefix}-YYYYMMDD-XXX (where XXX is the sequential number) - today = datetime.utcnow() - date_prefix = today.strftime("%Y%m%d") - search_pattern = f"{prefix}-{date_prefix}-%" - - # Find the next available number for today with the custom prefix - existing = cls.query.filter(cls.invoice_number.like(search_pattern)).order_by(cls.invoice_number.desc()).first() - - if existing: - # Extract the number part and increment - try: - # Split by last hyphen to get the number part - parts = existing.invoice_number.rsplit("-", 1) - if len(parts) == 2: - last_num = int(parts[1]) - next_num = last_num + 1 - else: - # Fallback if format is unexpected - next_num = start_number - # Ensure next_num is at least start_number - next_num = max(next_num, start_number) - except (ValueError, IndexError, AttributeError): - next_num = start_number - else: - # No existing invoices for today, use start_number - next_num = start_number - - # Format with appropriate padding based on start_number length - # Use at least 3 digits, but more if start_number requires it - min_digits = max(3, len(str(start_number))) - return f"{prefix}-{date_prefix}-{next_num:0{min_digits}d}" + return generate_next_invoice_number(cls) class InvoiceItem(db.Model): diff --git a/app/models/purchase_order.py b/app/models/purchase_order.py index a9cb1083..4345d5d6 100644 --- a/app/models/purchase_order.py +++ b/app/models/purchase_order.py @@ -6,6 +6,22 @@ from app import db +def _normalize_optional_text(value): + """Normalize optional text input to a trimmed string or None.""" + if value is None: + return None + text = str(value).strip() + return text or None + + +def _normalize_required_text(value, field_name): + """Normalize required text and fail fast for empty values.""" + text = _normalize_optional_text(value) + if not text: + raise ValueError(f"{field_name} is required") + return text + + class PurchaseOrder(db.Model): """PurchaseOrder model - represents a purchase order to a supplier""" @@ -37,6 +53,8 @@ class PurchaseOrder(db.Model): # Relationships items = db.relationship("PurchaseOrderItem", backref="purchase_order", lazy="dynamic", cascade="all, delete-orphan") + supplier = db.relationship("Supplier", backref="purchase_orders", lazy="select") + created_by_user = db.relationship("User", foreign_keys=[created_by], lazy="select") def __init__( self, @@ -49,14 +67,14 @@ def __init__( internal_notes=None, currency_code="EUR", ): - self.po_number = po_number.strip().upper() + self.po_number = _normalize_required_text(po_number, "po_number").upper() self.supplier_id = supplier_id self.order_date = order_date self.expected_delivery_date = expected_delivery_date self.created_by = created_by - self.notes = notes.strip() if notes else None - self.internal_notes = internal_notes.strip() if internal_notes else None - self.currency_code = currency_code.upper() + self.notes = _normalize_optional_text(notes) + self.internal_notes = _normalize_optional_text(internal_notes) + self.currency_code = _normalize_required_text(currency_code, "currency_code").upper() self.status = "draft" self.subtotal = Decimal("0") self.tax_amount = Decimal("0") @@ -126,6 +144,7 @@ def to_dict(self): "id": self.id, "po_number": self.po_number, "supplier_id": self.supplier_id, + "supplier_name": self.supplier.name if self.supplier else None, "status": self.status, "order_date": self.order_date.isoformat() if self.order_date else None, "expected_delivery_date": self.expected_delivery_date.isoformat() if self.expected_delivery_date else None, @@ -139,6 +158,7 @@ def to_dict(self): "created_at": self.created_at.isoformat() if self.created_at else None, "updated_at": self.updated_at.isoformat() if self.updated_at else None, "created_by": self.created_by, + "items": [item.to_dict() for item in self.items], } @@ -185,15 +205,15 @@ def __init__( self.purchase_order_id = purchase_order_id self.stock_item_id = stock_item_id self.supplier_stock_item_id = supplier_stock_item_id - self.description = description.strip() - self.supplier_sku = supplier_sku.strip() if supplier_sku else None + self.description = _normalize_required_text(description, "description") + self.supplier_sku = _normalize_optional_text(supplier_sku) self.quantity_ordered = Decimal(str(quantity_ordered)) self.quantity_received = Decimal("0") self.unit_cost = Decimal(str(unit_cost)) self.line_total = self.quantity_ordered * self.unit_cost self.warehouse_id = warehouse_id - self.notes = notes.strip() if notes else None - self.currency_code = currency_code.upper() + self.notes = _normalize_optional_text(notes) + self.currency_code = _normalize_required_text(currency_code, "currency_code").upper() def __repr__(self): return f"" diff --git a/app/models/settings.py b/app/models/settings.py index f16e7b01..908b4d68 100644 --- a/app/models/settings.py +++ b/app/models/settings.py @@ -4,6 +4,7 @@ from app import db from app.config import Config +from app.utils.invoice_numbering import DEFAULT_INVOICE_PATTERN # Re-entrancy guard: avoid add+commit when get_settings is called from inside a flush/commit _creating_settings = threading.local() @@ -63,6 +64,7 @@ class Settings(db.Model): # Invoice defaults invoice_prefix = db.Column(db.String(50), default="INV", nullable=False) + invoice_number_pattern = db.Column(db.String(120), default=DEFAULT_INVOICE_PATTERN, nullable=False) invoice_start_number = db.Column(db.Integer, default=1000, nullable=False) invoice_terms = db.Column(db.Text, default="Payment is due within 30 days of invoice date.", nullable=False) invoice_notes = db.Column(db.Text, default="Thank you for your business!", nullable=False) @@ -215,6 +217,7 @@ def __init__(self, **kwargs): # Set invoice defaults self.invoice_prefix = kwargs.get("invoice_prefix", "INV") + self.invoice_number_pattern = kwargs.get("invoice_number_pattern", DEFAULT_INVOICE_PATTERN) self.invoice_start_number = kwargs.get("invoice_start_number", 1000) self.invoice_terms = kwargs.get("invoice_terms", "Payment is due within 30 days of invoice date.") self.invoice_notes = kwargs.get("invoice_notes", "Thank you for your business!") @@ -437,6 +440,7 @@ def to_dict(self): "company_tax_id": self.company_tax_id, "company_bank_info": self.company_bank_info, "invoice_prefix": self.invoice_prefix, + "invoice_number_pattern": self.invoice_number_pattern, "invoice_start_number": self.invoice_start_number, "invoice_terms": self.invoice_terms, "invoice_notes": self.invoice_notes, diff --git a/app/models/stock_movement.py b/app/models/stock_movement.py index afaaf7f9..68d43852 100644 --- a/app/models/stock_movement.py +++ b/app/models/stock_movement.py @@ -3,6 +3,8 @@ from datetime import datetime from decimal import Decimal +from sqlalchemy.exc import IntegrityError + from app import db @@ -131,8 +133,18 @@ def record_movement( stock = WarehouseStock.query.filter_by(warehouse_id=warehouse_id, stock_item_id=stock_item_id).first() if not stock: - stock = WarehouseStock(warehouse_id=warehouse_id, stock_item_id=stock_item_id, quantity_on_hand=0) - db.session.add(stock) + try: + with db.session.begin_nested(): + stock = WarehouseStock(warehouse_id=warehouse_id, stock_item_id=stock_item_id, quantity_on_hand=0) + db.session.add(stock) + db.session.flush() + except IntegrityError: + # Another concurrent transaction inserted it first. + stock = WarehouseStock.query.filter_by( + warehouse_id=warehouse_id, stock_item_id=stock_item_id + ).first() + if not stock: + raise # Update stock level stock.adjust_on_hand(quantity) diff --git a/app/repositories/invoice_repository.py b/app/repositories/invoice_repository.py index c14004ba..f8853b74 100644 --- a/app/repositories/invoice_repository.py +++ b/app/repositories/invoice_repository.py @@ -69,42 +69,7 @@ def get_with_relations(self, invoice_id: int) -> Optional[Invoice]: def generate_invoice_number(self) -> str: """Generate a unique invoice number""" - from datetime import datetime - - from app.models import Settings - - # Get settings for invoice prefix and start number - settings = Settings.get_settings() - prefix = "INV" # Default prefix - start_number = 1 # Default start number - - if settings: - prefix = getattr(settings, "invoice_prefix", "INV") or "INV" - start_number = getattr(settings, "invoice_start_number", 1) or 1 - - # Format: {prefix}-YYYYMMDD-XXX - today = datetime.utcnow().strftime("%Y%m%d") - search_pattern = f"{prefix}-{today}-%" - - # Find the highest number for today with the custom prefix - last_invoice = ( - self.model.query.filter(Invoice.invoice_number.like(search_pattern)) - .order_by(Invoice.invoice_number.desc()) - .first() - ) - - if last_invoice: - try: - last_num = int(last_invoice.invoice_number.split("-")[-1]) - next_num = last_num + 1 - # Ensure next_num is at least start_number - next_num = max(next_num, start_number) - except (ValueError, IndexError): - next_num = start_number - else: - next_num = start_number - - return f"{prefix}-{today}-{next_num:03d}" + return Invoice.generate_invoice_number() def mark_as_sent(self, invoice_id: int) -> Optional[Invoice]: """Mark an invoice as sent""" diff --git a/app/routes/admin.py b/app/routes/admin.py index 2911d492..fb9de08f 100644 --- a/app/routes/admin.py +++ b/app/routes/admin.py @@ -42,6 +42,7 @@ from app.utils.db import safe_commit from app.utils.error_handling import safe_file_remove, safe_log from app.utils.installation import get_installation_config +from app.utils.invoice_numbering import sanitize_invoice_pattern, sanitize_invoice_prefix, validate_invoice_pattern from app.utils.permissions import admin_or_permission_required from app.utils.telemetry import get_telemetry_fingerprint, is_telemetry_enabled from app.utils.timezone import get_available_timezones @@ -1057,11 +1058,11 @@ def telemetry_dashboard(): "config": installation_config.get_all_config(), } - # Get PostHog status - posthog_data = { - "enabled": bool(os.getenv("POSTHOG_API_KEY")), - "host": os.getenv("POSTHOG_HOST", "https://app.posthog.com"), - "api_key_set": bool(os.getenv("POSTHOG_API_KEY")), + # Get OTEL OTLP status + grafana_data = { + "enabled": bool(os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")) and bool(os.getenv("OTEL_EXPORTER_OTLP_TOKEN")), + "endpoint": os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", ""), + "token_set": bool(os.getenv("OTEL_EXPORTER_OTLP_TOKEN")), } # Get Sentry status @@ -1075,7 +1076,7 @@ def telemetry_dashboard(): app_module.log_event("admin.telemetry_dashboard_viewed", user_id=current_user.id) app_module.track_event(current_user.id, "admin.telemetry_dashboard_viewed", {}) - return render_template("admin/telemetry.html", telemetry=telemetry_data, posthog=posthog_data, sentry=sentry_data) + return render_template("admin/telemetry.html", telemetry=telemetry_data, grafana=grafana_data, sentry=sentry_data) @admin_bp.route("/admin/telemetry/toggle", methods=["POST"]) @@ -1103,7 +1104,7 @@ def toggle_telemetry(): if new_state: flash(_("Telemetry has been enabled. Thank you for helping us improve!"), "success") else: - flash(_("Telemetry has been disabled."), "info") + flash(_("Detailed analytics has been disabled. Anonymous base telemetry remains active."), "info") return redirect(url_for("admin.telemetry_dashboard")) @@ -1313,8 +1314,21 @@ def settings(): settings_obj.company_bank_info = request.form.get("company_bank_info", "") # Update invoice defaults - invoice_prefix_form = request.form.get("invoice_prefix", "INV") + invoice_prefix_form = sanitize_invoice_prefix(request.form.get("invoice_prefix", "")) + invoice_number_pattern_form = sanitize_invoice_pattern(request.form.get("invoice_number_pattern", "")) invoice_start_number_form = request.form.get("invoice_start_number", 1000) + is_valid_pattern, pattern_error = validate_invoice_pattern(invoice_number_pattern_form) + if not is_valid_pattern: + flash(_("Invalid invoice number pattern: %(reason)s", reason=pattern_error), "error") + system_instance_id = Settings.get_system_instance_id() + return render_template( + "admin/settings.html", + settings=settings_obj, + timezones=timezones, + kiosk_settings=kiosk_settings, + peppol_env_enabled=peppol_env_enabled, + system_instance_id=system_instance_id, + ) # #region agent log try: import json @@ -1324,6 +1338,7 @@ def settings(): "message": "Saving invoice prefix and start number", "data": { "invoice_prefix_form": str(invoice_prefix_form), + "invoice_number_pattern_form": str(invoice_number_pattern_form), "invoice_start_number_form": str(invoice_start_number_form), "settings_obj_id": settings_obj.id if hasattr(settings_obj, "id") else "NO_ID", }, @@ -1339,6 +1354,7 @@ def settings(): pass # #endregion settings_obj.invoice_prefix = invoice_prefix_form + settings_obj.invoice_number_pattern = invoice_number_pattern_form settings_obj.invoice_start_number = int(invoice_start_number_form) settings_obj.invoice_terms = request.form.get("invoice_terms", "Payment is due within 30 days of invoice date.") settings_obj.invoice_notes = request.form.get("invoice_notes", "Thank you for your business!") @@ -1452,6 +1468,7 @@ def settings(): "message": "After commit - settings values", "data": { "invoice_prefix": str(settings_obj.invoice_prefix), + "invoice_number_pattern": str(getattr(settings_obj, "invoice_number_pattern", "")), "invoice_start_number": int(settings_obj.invoice_start_number), "settings_obj_id": settings_obj.id if hasattr(settings_obj, "id") else "NO_ID", }, @@ -4961,7 +4978,15 @@ def create_email_template(): # Validate if not name: flash(_("Template name is required"), "error") - return render_template("admin/email_templates/create.html") + return render_template( + "admin/email_templates/create.html", name=name, description=description, html=html, css=css + ) + + if not html: + flash(_("HTML template content is required"), "error") + return render_template( + "admin/email_templates/create.html", name=name, description=description, html=html, css=css + ) # Check for duplicate name existing = InvoiceTemplate.query.filter_by(name=name).first() @@ -4987,7 +5012,9 @@ def create_email_template(): db.session.add(template) if not safe_commit("create_email_template", {"name": name}): flash(_("Could not create email template due to a database error."), "error") - return render_template("admin/email_templates/create.html") + return render_template( + "admin/email_templates/create.html", name=name, description=description, html=html, css=css + ) flash(_("Email template created successfully"), "success") return redirect(url_for("admin.list_email_templates")) diff --git a/app/routes/api_v1.py b/app/routes/api_v1.py index da96b376..ee3906f6 100644 --- a/app/routes/api_v1.py +++ b/app/routes/api_v1.py @@ -1,10 +1,12 @@ """REST API v1 - Comprehensive API endpoints with token authentication""" from datetime import date, datetime, timedelta -from decimal import InvalidOperation +from decimal import Decimal, InvalidOperation +from uuid import uuid4 from flask import Blueprint, Response, current_app, g, jsonify, request from sqlalchemy import func, or_ +from sqlalchemy.exc import IntegrityError, SQLAlchemyError from app import db, limiter from app.models import ( @@ -4120,9 +4122,6 @@ def get_purchase_order_api(po_id): @require_api_token(("write:inventory", "write:projects")) def create_purchase_order_api(): """Create a purchase order""" - from datetime import datetime - from decimal import Decimal - from app.models import PurchaseOrder, PurchaseOrderItem, Supplier data = request.get_json() or {} @@ -4131,12 +4130,38 @@ def create_purchase_order_api(): if not supplier_id: return jsonify({"error": "supplier_id is required"}), 400 + supplier = Supplier.query.get(supplier_id) + if not supplier: + return jsonify({"error": "supplier_id does not reference an existing supplier"}), 400 + + items = data.get("items", []) + normalized_items = [] try: - # Generate PO number - last_po = PurchaseOrder.query.order_by(PurchaseOrder.id.desc()).first() - next_id = (last_po.id + 1) if last_po else 1 - po_number = f"PO-{datetime.now().strftime('%Y%m%d')}-{next_id:04d}" + for item_data in items: + description = item_data.get("description") + if description is None or not str(description).strip(): + return jsonify({"error": "Each item requires a non-empty description"}), 400 + quantity_ordered = Decimal(str(item_data.get("quantity_ordered", 1))) + unit_cost = Decimal(str(item_data.get("unit_cost", 0))) + if quantity_ordered <= 0: + return jsonify({"error": "quantity_ordered must be greater than zero"}), 400 + if unit_cost < 0: + return jsonify({"error": "unit_cost must be zero or greater"}), 400 + normalized_items.append( + { + "description": str(description).strip(), + "quantity_ordered": quantity_ordered, + "unit_cost": unit_cost, + "stock_item_id": item_data.get("stock_item_id"), + "supplier_stock_item_id": item_data.get("supplier_stock_item_id"), + "supplier_sku": item_data.get("supplier_sku"), + "warehouse_id": item_data.get("warehouse_id"), + } + ) + except (InvalidOperation, ValueError, TypeError): + return jsonify({"error": "Invalid item quantity or unit cost"}), 400 + try: order_date = ( datetime.strptime(data.get("order_date"), "%Y-%m-%d").date() if data.get("order_date") @@ -4149,7 +4174,7 @@ def create_purchase_order_api(): ) purchase_order = PurchaseOrder( - po_number=po_number, + po_number=f"PO-TMP-{uuid4().hex[:12].upper()}", supplier_id=supplier_id, order_date=order_date, created_by=g.api_user.id, @@ -4160,15 +4185,15 @@ def create_purchase_order_api(): ) db.session.add(purchase_order) db.session.flush() + purchase_order.po_number = f"PO-{order_date.strftime('%Y%m%d')}-{purchase_order.id:04d}" # Handle items - items = data.get("items", []) - for item_data in items: + for item_data in normalized_items: item = PurchaseOrderItem( purchase_order_id=purchase_order.id, - description=item_data.get("description", ""), - quantity_ordered=Decimal(str(item_data.get("quantity_ordered", 1))), - unit_cost=Decimal(str(item_data.get("unit_cost", 0))), + description=item_data["description"], + quantity_ordered=item_data["quantity_ordered"], + unit_cost=item_data["unit_cost"], stock_item_id=item_data.get("stock_item_id"), supplier_stock_item_id=item_data.get("supplier_stock_item_id"), supplier_sku=item_data.get("supplier_sku"), @@ -4184,9 +4209,21 @@ def create_purchase_order_api(): jsonify({"message": "Purchase order created successfully", "purchase_order": purchase_order.to_dict()}), 201, ) - except Exception as e: + except IntegrityError: db.session.rollback() - return jsonify({"error": str(e)}), 400 + current_app.logger.exception("Purchase order create conflict or integrity error") + return jsonify({"error": "Could not create purchase order due to data conflict"}), 409 + except (InvalidOperation, ValueError, TypeError): + db.session.rollback() + return jsonify({"error": "Invalid purchase order payload"}), 400 + except SQLAlchemyError: + db.session.rollback() + current_app.logger.exception("Database error while creating purchase order") + return jsonify({"error": "Database error while creating purchase order"}), 500 + except Exception: + db.session.rollback() + current_app.logger.exception("Unexpected error while creating purchase order") + return jsonify({"error": "Unexpected server error while creating purchase order"}), 500 @api_v1_bp.route("/inventory/purchase-orders/", methods=["PUT", "PATCH"]) @@ -4254,10 +4291,21 @@ def update_purchase_order_api(po_id): db.session.commit() return jsonify({"message": "Purchase order updated successfully", "purchase_order": purchase_order.to_dict()}) + except IntegrityError: + db.session.rollback() + current_app.logger.exception("Purchase order update conflict or integrity error") + return jsonify({"error": "Could not update purchase order due to data conflict"}), 409 + except (InvalidOperation, ValueError, TypeError): + db.session.rollback() + return jsonify({"error": "Invalid purchase order payload"}), 400 + except SQLAlchemyError: + db.session.rollback() + current_app.logger.exception("Database error while updating purchase order") + return jsonify({"error": "Database error while updating purchase order"}), 500 except Exception as e: db.session.rollback() current_app.logger.error(f"Error updating purchase order: {e}", exc_info=True) - return jsonify({"error": str(e)}), 400 + return jsonify({"error": "Unexpected server error while updating purchase order"}), 500 @api_v1_bp.route("/inventory/purchase-orders/", methods=["DELETE"]) diff --git a/app/routes/inventory.py b/app/routes/inventory.py index 3451ca5c..61290039 100644 --- a/app/routes/inventory.py +++ b/app/routes/inventory.py @@ -2,6 +2,7 @@ from datetime import datetime, timedelta from decimal import Decimal, InvalidOperation +from uuid import uuid4 from flask import Blueprint, current_app, flash, jsonify, redirect, render_template, request, url_for from flask_babel import gettext as _ @@ -31,6 +32,17 @@ inventory_bp = Blueprint("inventory", __name__) +def _provisional_po_number(): + """Generate a temporary unique PO number before we have a DB id.""" + return f"PO-TMP-{uuid4().hex[:12].upper()}" + + +def _finalize_po_number(purchase_order): + """Assign deterministic PO number based on persisted id.""" + order_date = purchase_order.order_date or datetime.utcnow().date() + purchase_order.po_number = f"PO-{order_date.strftime('%Y%m%d')}-{purchase_order.id:04d}" + + # ==================== Stock Items API (for selection in forms) ==================== @@ -1800,15 +1812,26 @@ def new_purchase_order(): """Create a new purchase order""" if request.method == "POST": try: - # Generate PO number - last_po = PurchaseOrder.query.order_by(PurchaseOrder.id.desc()).first() - next_id = (last_po.id + 1) if last_po else 1 - po_number = f"PO-{datetime.now().strftime('%Y%m%d')}-{next_id:04d}" + supplier_id_raw = request.form.get("supplier_id") + if not supplier_id_raw: + flash(_("Supplier is required."), "error") + return redirect(url_for("inventory.new_purchase_order")) + supplier_id = int(supplier_id_raw) + supplier = Supplier.query.get(supplier_id) + if not supplier: + flash(_("Supplier not found."), "error") + return redirect(url_for("inventory.new_purchase_order")) + + order_date_raw = request.form.get("order_date") + if not order_date_raw: + flash(_("Order date is required."), "error") + return redirect(url_for("inventory.new_purchase_order")) + order_date = datetime.strptime(order_date_raw, "%Y-%m-%d").date() purchase_order = PurchaseOrder( - po_number=po_number, - supplier_id=int(request.form.get("supplier_id")), - order_date=datetime.strptime(request.form.get("order_date"), "%Y-%m-%d").date(), + po_number=_provisional_po_number(), + supplier_id=supplier.id, + order_date=order_date, created_by=current_user.id, expected_delivery_date=( datetime.strptime(request.form.get("expected_delivery_date"), "%Y-%m-%d").date() @@ -1822,6 +1845,7 @@ def new_purchase_order(): db.session.add(purchase_order) db.session.flush() + _finalize_po_number(purchase_order) # Handle items item_descriptions = request.form.getlist("item_description[]") @@ -1835,6 +1859,17 @@ def new_purchase_order(): for i, desc in enumerate(item_descriptions): if desc.strip(): try: + quantity = ( + Decimal(item_quantities[i]) if i < len(item_quantities) and item_quantities[i] else Decimal("1") + ) + unit_cost = ( + Decimal(item_unit_costs[i]) if i < len(item_unit_costs) and item_unit_costs[i] else Decimal("0") + ) + if quantity <= 0: + raise ValueError("quantity must be greater than zero") + if unit_cost < 0: + raise ValueError("unit_cost must be zero or greater") + stock_item_id = ( int(item_stock_ids[i]) if i < len(item_stock_ids) and item_stock_ids[i] else None ) @@ -1852,16 +1887,8 @@ def new_purchase_order(): item = PurchaseOrderItem( purchase_order_id=purchase_order.id, description=desc.strip(), - quantity_ordered=( - Decimal(item_quantities[i]) - if i < len(item_quantities) and item_quantities[i] - else Decimal("1") - ), - unit_cost=( - Decimal(item_unit_costs[i]) - if i < len(item_unit_costs) and item_unit_costs[i] - else Decimal("0") - ), + quantity_ordered=quantity, + unit_cost=unit_cost, stock_item_id=stock_item_id, supplier_stock_item_id=supplier_stock_item_id, supplier_sku=( @@ -1877,7 +1904,9 @@ def new_purchase_order(): current_app.logger.warning(f"Invalid quantity or cost for purchase order item: {e}") purchase_order.calculate_totals() - safe_commit() + if not safe_commit("create_purchase_order", {"supplier_id": supplier.id, "po_number": purchase_order.po_number}): + flash(_("Could not create purchase order due to a database error."), "error") + return redirect(url_for("inventory.new_purchase_order")) log_event( "purchase_order_created", @@ -2001,7 +2030,9 @@ def edit_purchase_order(po_id): current_app.logger.warning(f"Invalid quantity or cost for purchase order item: {e}") purchase_order.calculate_totals() - safe_commit() + if not safe_commit("edit_purchase_order", {"purchase_order_id": po_id}): + flash(_("Could not update purchase order due to a database error."), "error") + return redirect(url_for("inventory.edit_purchase_order", po_id=po_id)) log_event("purchase_order_updated", purchase_order_id=po_id) flash(_("Purchase order updated successfully."), "success") @@ -2040,7 +2071,9 @@ def send_purchase_order(po_id): if request.method == "POST": try: purchase_order.mark_as_sent() - safe_commit() + if not safe_commit("send_purchase_order", {"purchase_order_id": po_id}): + flash(_("Could not update purchase order status due to a database error."), "error") + return redirect(url_for("inventory.view_purchase_order", po_id=po_id)) log_event("purchase_order_sent", purchase_order_id=po_id) flash(_("Purchase order marked as sent."), "success") @@ -2062,7 +2095,9 @@ def cancel_purchase_order(po_id): if request.method == "POST": try: purchase_order.cancel() - safe_commit() + if not safe_commit("cancel_purchase_order", {"purchase_order_id": po_id}): + flash(_("Could not cancel purchase order due to a database error."), "error") + return redirect(url_for("inventory.view_purchase_order", po_id=po_id)) log_event("purchase_order_cancelled", purchase_order_id=po_id) flash(_("Purchase order cancelled successfully."), "success") @@ -2088,7 +2123,9 @@ def delete_purchase_order(po_id): po_number = purchase_order.po_number db.session.delete(purchase_order) - safe_commit() + if not safe_commit("delete_purchase_order", {"purchase_order_id": po_id, "po_number": po_number}): + flash(_("Could not delete purchase order due to a database error."), "error") + return redirect(url_for("inventory.view_purchase_order", po_id=po_id)) log_event("purchase_order_deleted", po_number=po_number) flash(_("Purchase order deleted successfully."), "success") @@ -2130,7 +2167,9 @@ def receive_purchase_order(po_id): ) purchase_order.mark_as_received(received_date) - safe_commit() + if not safe_commit("receive_purchase_order", {"purchase_order_id": po_id}): + flash(_("Could not receive purchase order due to a database error."), "error") + return redirect(url_for("inventory.view_purchase_order", po_id=po_id)) log_event("purchase_order_received", purchase_order_id=po_id) flash(_("Purchase order marked as received and stock updated."), "success") diff --git a/app/routes/invoices.py b/app/routes/invoices.py index a72f7d93..dcfa51e3 100644 --- a/app/routes/invoices.py +++ b/app/routes/invoices.py @@ -599,6 +599,7 @@ def update_invoice_status(invoice_id): if not current_user.is_admin and invoice.created_by != current_user.id: return jsonify({"error": "Permission denied"}), 403 + previous_status = invoice.status new_status = request.form.get("new_status") if new_status not in ["draft", "sent", "paid", "cancelled"]: return jsonify({"error": "Invalid status"}), 400 @@ -631,7 +632,11 @@ def update_invoice_status(invoice_id): reduce_on_sent = os.getenv("INVENTORY_REDUCE_ON_INVOICE_SENT", "true").lower() == "true" reduce_on_paid = os.getenv("INVENTORY_REDUCE_ON_INVOICE_PAID", "false").lower() == "true" - if (new_status == "sent" and reduce_on_sent) or (new_status == "paid" and reduce_on_paid): + should_reduce_stock = ( + (new_status == "sent" and reduce_on_sent and previous_status != "sent") + or (new_status == "paid" and reduce_on_paid and previous_status != "paid") + ) + if should_reduce_stock: for item in invoice.items: if item.is_stock_item and item.stock_item_id and item.warehouse_id: try: @@ -1032,12 +1037,7 @@ def export_invoice_csv(invoice_id): output.seek(0) - # Get invoice prefix from settings, default to "INV" - settings = Settings.get_settings() - prefix = getattr(settings, "invoice_prefix", "INV") if settings else "INV" - if not prefix: - prefix = "INV" - filename = f"{prefix}_{invoice.invoice_number}.csv" + filename = f"{invoice.invoice_number}.csv" return send_file( io.BytesIO(output.getvalue().encode("utf-8")), mimetype="text/csv", as_attachment=True, download_name=filename @@ -1060,7 +1060,7 @@ def export_invoice_ubl(invoice_id): sender = svc._get_sender_party() recipient_party, _ign, _ign = svc._get_recipient_party(invoice) ubl_xml, _ign = build_peppol_ubl_invoice_xml(invoice=invoice, supplier=sender, customer=recipient_party) - fn = f"invoice_{invoice.invoice_number}.xml" + fn = f"{invoice.invoice_number}.xml" return Response( ubl_xml, mimetype="application/xml", headers={"Content-Disposition": f"attachment; filename={fn}"} ) diff --git a/app/routes/setup.py b/app/routes/setup.py index cb706201..16e36dfb 100644 --- a/app/routes/setup.py +++ b/app/routes/setup.py @@ -124,7 +124,7 @@ def initial_setup(): pass flash(_("Setup complete! Thank you for helping us improve TimeTracker."), "success") else: - flash(_("Setup complete! Telemetry is disabled."), "success") + flash(_("Setup complete! Detailed analytics is disabled; anonymous base telemetry remains active."), "success") if google_client_id: flash(_("Google Calendar OAuth credentials have been configured."), "success") diff --git a/app/telemetry/service.py b/app/telemetry/service.py index 71a0565d..4b62dad9 100644 --- a/app/telemetry/service.py +++ b/app/telemetry/service.py @@ -1,37 +1,48 @@ """ -Consent-aware telemetry service. +Consent-aware telemetry service backed by Grafana Cloud OTLP. -- Base telemetry: always-on, minimal schema (install footprint, heartbeat). - Event names: base_telemetry.first_seen, base_telemetry.heartbeat. -- Detailed analytics: only when user has opted in; product events (analytics.* or existing names). +- Base telemetry is always-on and anonymous per installation. +- Detailed analytics is sent only when the user opted in. """ +import json +import logging import os import platform +import base64 +import time from datetime import datetime, timezone from typing import Any, Dict, Optional - -# Lazy imports to avoid circular deps and to keep posthog optional at import time - -# Base telemetry schema keys (no PII). Country omitted unless added server-side later. -BASE_SCHEMA_KEYS = frozenset({ - "install_id", "app_version", "platform", "os_version", "architecture", - "locale", "timezone", "first_seen_at", "last_seen_at", "heartbeat_at", - "release_channel", "deployment_type", -}) +from urllib import request +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + +BASE_SCHEMA_KEYS = frozenset( + { + "install_id", + "app_version", + "platform", + "os_version", + "architecture", + "locale", + "timezone", + "first_seen_at", + "last_seen_at", + "heartbeat_at", + "release_channel", + "deployment_type", + } +) def is_detailed_analytics_enabled() -> bool: - """True if the user has opted in to detailed analytics (feature usage, screens, etc.).""" from app.utils.telemetry import is_telemetry_enabled return is_telemetry_enabled() -def _build_base_telemetry_payload( - event_kind: str, -) -> Dict[str, Any]: - """Build minimal base telemetry payload. No PII.""" +def _build_base_telemetry_payload(event_kind: str) -> Dict[str, Any]: from app.config.analytics_defaults import get_analytics_config from app.utils.installation import get_installation_config @@ -59,41 +70,176 @@ def _build_base_telemetry_payload( return payload -def send_base_telemetry(payload: Dict[str, Any]) -> bool: +def _otlp_enabled() -> bool: + from app.config.analytics_defaults import get_analytics_config + + config = get_analytics_config() + endpoint = ( + config.get("otel_exporter_otlp_endpoint") + or os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + ) + token = config.get("otel_exporter_otlp_token") or os.getenv("OTEL_EXPORTER_OTLP_TOKEN", "") + return bool(endpoint and token) + + +def _build_otlp_auth_header(token: str) -> str: """ - Send base telemetry (always-on, minimal). Schema: install_id, app_version, - platform, os_version, architecture, locale, timezone, first_seen_at, last_seen_at, - heartbeat_at, release_channel, deployment_type. - Sends to PostHog as base_telemetry.first_seen or base_telemetry.heartbeat when payload - includes event_kind or uses distinct event names. Returns True if sent. + Build OTLP Authorization header from a single token input. + Accepted token formats: + - "Basic " + - ":" -> converted to Basic + - "" -> treated as Basic payload """ - try: - import posthog - from app.config.analytics_defaults import get_analytics_config + value = (token or "").strip() + if value.lower().startswith("basic "): + return value + if ":" in value: + encoded = base64.b64encode(value.encode("utf-8")).decode("ascii") + return f"Basic {encoded}" + return f"Basic {value}" - config = get_analytics_config() - posthog_api_key = config.get("posthog_api_key") or os.getenv("POSTHOG_API_KEY", "") - if not posthog_api_key: - return False - if not getattr(posthog, "project_api_key", None) or not posthog.project_api_key: - posthog.project_api_key = posthog_api_key - posthog.host = config.get("posthog_host", os.getenv("POSTHOG_HOST", "https://app.posthog.com")) +def _telemetry_debug_logging_enabled() -> bool: + return (os.getenv("OTEL_DEBUG_LOGGING", "false") or "").strip().lower() in {"1", "true", "yes", "on"} - install_id = payload.get("install_id") - if not install_id: - return False - event_name = payload.get("_event", "base_telemetry.heartbeat") - props = {k: v for k, v in payload.items() if k != "_event"} - posthog.capture(distinct_id=install_id, event=event_name, properties=props) - return True - except Exception: +def _remove_pii(properties: Dict[str, Any]) -> Dict[str, Any]: + pii_keys = {"email", "username", "ip", "ip_address", "full_name", "name", "password", "token"} + return {k: v for k, v in properties.items() if k.lower() not in pii_keys} + + +def _to_otlp_any_value(value: Any) -> Dict[str, Any]: + if isinstance(value, bool): + return {"boolValue": value} + if isinstance(value, int): + return {"intValue": str(value)} + if isinstance(value, float): + return {"doubleValue": value} + return {"stringValue": str(value)} + + +def _build_otlp_logs_payload( + event_name: str, + identity: str, + detailed: bool, + safe_props: Dict[str, Any], + service_version: str, +) -> Dict[str, Any]: + now_nanos = str(int(time.time() * 1_000_000_000)) + resource_attributes = [ + {"key": "service.name", "value": {"stringValue": "timetracker"}}, + {"key": "service.version", "value": {"stringValue": str(service_version or "unknown")}}, + {"key": "deployment.environment", "value": {"stringValue": os.getenv("FLASK_ENV", "production")}}, + ] + record_attributes = [ + {"key": "event_name", "value": {"stringValue": event_name}}, + {"key": "identity", "value": {"stringValue": str(identity)}}, + {"key": "detailed", "value": {"boolValue": bool(detailed)}}, + ] + for key, value in safe_props.items(): + record_attributes.append({"key": str(key), "value": _to_otlp_any_value(value)}) + + return { + "resourceLogs": [ + { + "resource": {"attributes": resource_attributes}, + "scopeLogs": [ + { + "scope": {"name": "timetracker.telemetry"}, + "logRecords": [ + { + "timeUnixNano": now_nanos, + "severityText": "INFO", + "body": {"stringValue": event_name}, + "attributes": record_attributes, + } + ], + } + ], + } + ] + } + + +def _send_otlp_event(event_name: str, identity: str, properties: Dict[str, Any], detailed: bool) -> bool: + from app.config.analytics_defaults import get_analytics_config + + config = get_analytics_config() + endpoint = config.get("otel_exporter_otlp_endpoint") or os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + token = config.get("otel_exporter_otlp_token") or os.getenv("OTEL_EXPORTER_OTLP_TOKEN", "") + + if not endpoint or not token: + if _telemetry_debug_logging_enabled(): + logger.info( + "telemetry.skip event=%s reason=missing_otlp_config endpoint_set=%s token_set=%s", + event_name, + bool(endpoint), + bool(token), + ) return False + # Support OTEL-style base endpoint by auto-targeting logs path. + endpoint = endpoint.rstrip("/") + if endpoint.endswith("/otlp"): + endpoint = f"{endpoint}/v1/logs" + elif not endpoint.endswith("/v1/logs"): + endpoint = f"{endpoint}/v1/logs" + + safe_props = _remove_pii(properties) if detailed else properties + payload = _build_otlp_logs_payload( + event_name=event_name, + identity=str(identity), + detailed=detailed, + safe_props=safe_props, + service_version=str(config.get("app_version", "unknown")), + ) + body = json.dumps(payload).encode("utf-8") + auth_header = _build_otlp_auth_header(token) + headers = { + "Content-Type": "application/json", + "Authorization": auth_header, + } + if _telemetry_debug_logging_enabled(): + parsed = urlparse(endpoint) + auth_mode = "basic_from_colon" if ":" in token and not token.lower().startswith("basic ") else "basic_direct" + logger.info( + "telemetry.send event=%s detailed=%s endpoint=%s://%s%s auth_mode=%s identity_len=%s props_count=%s", + event_name, + detailed, + parsed.scheme or "https", + parsed.netloc, + parsed.path, + auth_mode, + len(str(identity)), + len(safe_props), + ) + + req = request.Request( + endpoint, + data=body, + method="POST", + headers=headers, + ) + try: + with request.urlopen(req, timeout=5) as response: + if _telemetry_debug_logging_enabled(): + logger.info("telemetry.ok event=%s status=%s", event_name, getattr(response, "status", "unknown")) + return True + except Exception as exc: + logger.warning("telemetry.fail event=%s error=%s", event_name, exc) + return False + + +def send_base_telemetry(payload: Dict[str, Any]) -> bool: + install_id = payload.get("install_id") + if not install_id: + return False + event_name = payload.get("_event", "base_telemetry.heartbeat") + props = {k: v for k, v in payload.items() if k != "_event"} + return _send_otlp_event(event_name=event_name, identity=str(install_id), properties=props, detailed=False) + def send_base_first_seen() -> bool: - """Send base_telemetry.first_seen once per install. Idempotent.""" from app.utils.installation import get_installation_config inst = get_installation_config() @@ -109,91 +255,43 @@ def send_base_first_seen() -> bool: def send_base_heartbeat() -> bool: - """Send base_telemetry.heartbeat (e.g. daily). Updates last_seen_at.""" payload = _build_base_telemetry_payload("heartbeat") payload["_event"] = "base_telemetry.heartbeat" return send_base_telemetry(payload) def identify_user(user_id: Any, properties: Optional[Dict[str, Any]] = None) -> None: - """Identify user in analytics backend. Only when opted in and PostHog configured.""" if not is_detailed_analytics_enabled(): return - try: - import posthog - from app.config.analytics_defaults import get_analytics_config - - config = get_analytics_config() - posthog_api_key = config.get("posthog_api_key") or os.getenv("POSTHOG_API_KEY", "") - if not posthog_api_key: - return - if not getattr(posthog, "project_api_key", None) or not posthog.project_api_key: - posthog.project_api_key = posthog_api_key - posthog.host = config.get("posthog_host", os.getenv("POSTHOG_HOST", "https://app.posthog.com")) - posthog.identify(distinct_id=str(user_id), properties=properties or {}) - except Exception: - pass + _send_otlp_event("analytics.identify", str(user_id), properties or {}, detailed=True) -def send_analytics_event( - user_id: Any, - event_name: str, - properties: Optional[Dict[str, Any]] = None, -) -> None: - """ - Send a product analytics event. Only sent when detailed analytics is opted in - and PostHog is configured. Adds install_id and context. - """ +def send_analytics_event(user_id: Any, event_name: str, properties: Optional[Dict[str, Any]] = None) -> None: if not is_detailed_analytics_enabled(): return + from app.config.analytics_defaults import get_analytics_config + from app.utils.installation import get_installation_config + + config = get_analytics_config() + enhanced = dict(properties or {}) + enhanced["install_id"] = get_installation_config().get_install_id() + enhanced["environment"] = os.getenv("FLASK_ENV", "production") + enhanced["app_version"] = config.get("app_version") + enhanced["deployment_method"] = "docker" if os.path.exists("/.dockerenv") else "native" + try: - import posthog - from app.config.analytics_defaults import get_analytics_config - from app.utils.installation import get_installation_config - - config = get_analytics_config() - posthog_api_key = config.get("posthog_api_key") or os.getenv("POSTHOG_API_KEY", "") - if not posthog_api_key: - return - - if not getattr(posthog, "project_api_key", None) or not posthog.project_api_key: - posthog.project_api_key = posthog_api_key - posthog.host = config.get("posthog_host", os.getenv("POSTHOG_HOST", "https://app.posthog.com")) - - enhanced = dict(properties or {}) - enhanced["install_id"] = get_installation_config().get_install_id() - enhanced["environment"] = os.getenv("FLASK_ENV", "production") - enhanced["app_version"] = config.get("app_version") - enhanced["deployment_method"] = "docker" if os.path.exists("/.dockerenv") else "native" - - try: - from flask import request - - if request: - enhanced["$current_url"] = request.url - enhanced["$host"] = request.host - enhanced["$pathname"] = request.path - enhanced["$browser"] = getattr(request.user_agent, "browser", None) - enhanced["$device_type"] = ( - "mobile" - if getattr(request.user_agent, "platform", None) in ["android", "iphone"] - else "desktop" - ) - enhanced["$os"] = getattr(request.user_agent, "platform", None) - except Exception: - pass - - posthog.capture(distinct_id=str(user_id), event=event_name, properties=enhanced) + from flask import request as flask_request + + if flask_request: + enhanced["current_url"] = flask_request.url + enhanced["host"] = flask_request.host + enhanced["pathname"] = flask_request.path + enhanced["browser"] = getattr(flask_request.user_agent, "browser", None) + enhanced["device_type"] = ( + "mobile" if getattr(flask_request.user_agent, "platform", None) in ["android", "iphone"] else "desktop" + ) + enhanced["os"] = getattr(flask_request.user_agent, "platform", None) except Exception: pass - -def send_base_telemetry(payload: Dict[str, Any]) -> bool: - """ - Send base telemetry (always-on, minimal). Schema: install_id, app_version, - platform, os_version, architecture, locale, timezone, first_seen_at, last_seen_at, - heartbeat_at, release_channel, deployment_type; country server-derived if possible. - Implemented in Phase 2; for now no-op if no sink configured. - """ - # Phase 2 will implement the sink (PostHog base event or custom endpoint) - return False + _send_otlp_event(event_name=event_name, identity=str(user_id), properties=enhanced, detailed=True) diff --git a/app/templates/admin/email_templates/create.html b/app/templates/admin/email_templates/create.html index 3953b015..c1ec5200 100644 --- a/app/templates/admin/email_templates/create.html +++ b/app/templates/admin/email_templates/create.html @@ -76,7 +76,7 @@
- + @@ -100,7 +100,10 @@ Use Jinja2 syntax for variables: {{ '{{ invoice.invoice_number }}' }}, {{ '{{ company_name }}' }}, {{ '{{ custom_message }}' }}

- + +
@@ -244,20 +247,6 @@

{{ _('Ot initialValue: originalHtml || '' }); - // Update hidden textarea on form submit - const emailTemplateForm = document.getElementById('emailTemplateForm'); - if (emailTemplateForm) { - emailTemplateForm.addEventListener('submit', function() { - if (htmlEditor) { - try { - htmlTextarea.value = htmlEditor.getHTML(); - } catch (e) { - htmlTextarea.value = htmlEditor.getMarkdown(); - } - } - }); - } - // Auto-update preview on content change htmlEditor.on('change', function() { // Debounce preview updates @@ -291,7 +280,50 @@

{{ _('Ot } } +function setHtmlValidationError(isVisible) { + const errorElement = document.getElementById('htmlValidationError'); + if (!errorElement) { + return; + } + errorElement.classList.toggle('hidden', !isVisible); +} + +function syncEditorHtmlToTextarea() { + const htmlTextarea = document.getElementById('html'); + if (!htmlTextarea) { + return ''; + } + + let htmlContent = ''; + if (htmlEditor) { + try { + htmlContent = htmlEditor.getHTML(); + } catch (e) { + htmlContent = htmlEditor.getMarkdown(); + } + } else { + const fallbackTextarea = document.getElementById('html_fallback'); + htmlContent = fallbackTextarea ? fallbackTextarea.value : htmlTextarea.value; + } + + htmlTextarea.value = htmlContent || ''; + return htmlTextarea.value.trim(); +} + function setupEventListeners() { + const emailTemplateForm = document.getElementById('emailTemplateForm'); + if (emailTemplateForm) { + emailTemplateForm.addEventListener('submit', function(event) { + const htmlContent = syncEditorHtmlToTextarea(); + if (!htmlContent) { + event.preventDefault(); + setHtmlValidationError(true); + return; + } + setHtmlValidationError(false); + }); + } + // CSS editor change handler const cssTextarea = document.getElementById('css'); if (cssTextarea) { @@ -429,6 +461,8 @@

{{ _('Ot return; } } + + setHtmlValidationError(false); updatePreviewContent(htmlContent); } diff --git a/app/templates/admin/email_templates/edit.html b/app/templates/admin/email_templates/edit.html index 61f2b857..2c0044e7 100644 --- a/app/templates/admin/email_templates/edit.html +++ b/app/templates/admin/email_templates/edit.html @@ -74,7 +74,7 @@

- + @@ -98,7 +98,10 @@ Use Jinja2 syntax for variables: {{ '{{ invoice.invoice_number }}' }}, {{ '{{ company_name }}' }}, {{ '{{ custom_message }}' }}

- + +
@@ -242,20 +245,6 @@

{{ _('Ot initialValue: originalHtml || '' }); - // Update hidden textarea on form submit - const emailTemplateForm = document.getElementById('emailTemplateForm'); - if (emailTemplateForm) { - emailTemplateForm.addEventListener('submit', function() { - if (htmlEditor) { - try { - htmlTextarea.value = htmlEditor.getHTML(); - } catch (e) { - htmlTextarea.value = htmlEditor.getMarkdown(); - } - } - }); - } - // Auto-update preview on content change htmlEditor.on('change', function() { clearTimeout(window.previewTimeout); @@ -288,7 +277,50 @@

{{ _('Ot } } +function setHtmlValidationError(isVisible) { + const errorElement = document.getElementById('htmlValidationError'); + if (!errorElement) { + return; + } + errorElement.classList.toggle('hidden', !isVisible); +} + +function syncEditorHtmlToTextarea() { + const htmlTextarea = document.getElementById('html'); + if (!htmlTextarea) { + return ''; + } + + let htmlContent = ''; + if (htmlEditor) { + try { + htmlContent = htmlEditor.getHTML(); + } catch (e) { + htmlContent = htmlEditor.getMarkdown(); + } + } else { + const fallbackTextarea = document.getElementById('html_fallback'); + htmlContent = fallbackTextarea ? fallbackTextarea.value : htmlTextarea.value; + } + + htmlTextarea.value = htmlContent || ''; + return htmlTextarea.value.trim(); +} + function setupEventListeners() { + const emailTemplateForm = document.getElementById('emailTemplateForm'); + if (emailTemplateForm) { + emailTemplateForm.addEventListener('submit', function(event) { + const htmlContent = syncEditorHtmlToTextarea(); + if (!htmlContent) { + event.preventDefault(); + setHtmlValidationError(true); + return; + } + setHtmlValidationError(false); + }); + } + // CSS editor change handler const cssTextarea = document.getElementById('css'); if (cssTextarea) { @@ -430,6 +462,8 @@

{{ _('Ot return; } } + + setHtmlValidationError(false); updatePreviewContent(htmlContent); } diff --git a/app/templates/admin/settings.html b/app/templates/admin/settings.html index 1c133a64..abd6a5c1 100644 --- a/app/templates/admin/settings.html +++ b/app/templates/admin/settings.html @@ -244,6 +244,13 @@

{{ _('Invoice Defaults') }}

+
+ + +

+ Available tokens: {SEQ}, {YYYY}, {YY}, {MM}, {DD}, {PREFIX}. Leave blank to generate sequence-only numbers. +

+
diff --git a/app/templates/admin/telemetry.html b/app/templates/admin/telemetry.html index 537c35d1..a82ca1c0 100644 --- a/app/templates/admin/telemetry.html +++ b/app/templates/admin/telemetry.html @@ -65,11 +65,11 @@

📊 Telemetr {% endif %}

- +
-

📈 PostHog (Product Analytics)

- {% if posthog.enabled %} +

📈 Grafana Cloud OTLP

+ {% if grafana.enabled %} Configured {% else %} Not Configured @@ -78,26 +78,26 @@

📈 PostHog
-

API Key

-

{{ 'Set' if posthog.api_key_set else 'Not Set' }}

+

Token

+

{{ 'Set' if grafana.token_set else 'Not Set' }}

-

Host

-

{{ posthog.host }}

+

OTLP Endpoint

+

{{ grafana.endpoint or 'Not Set' }}

- {% if posthog.enabled %} + {% if grafana.enabled %}

- PostHog is tracking: User behavior events like timer starts, project creation, etc. - Uses internal user IDs only (no PII). + Grafana OTLP is receiving telemetry: anonymous base install events and consented detailed analytics. + Direct personal identifiers are excluded.

{% else %}

- To enable PostHog, set POSTHOG_API_KEY in your environment variables. + To enable OTLP telemetry, set OTEL_EXPORTER_OTLP_ENDPOINT and OTEL_EXPORTER_OTLP_TOKEN in your environment variables.

{% endif %} diff --git a/app/templates/base.html b/app/templates/base.html index 9cddddb6..7e0a2d42 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -30,9 +30,9 @@ diff --git a/app/templates/invoices/edit.html b/app/templates/invoices/edit.html index 70082b3f..af0696fe 100644 --- a/app/templates/invoices/edit.html +++ b/app/templates/invoices/edit.html @@ -71,8 +71,8 @@

@@ -110,10 +110,10 @@

{% endif %} -
+
-
+
{% if item.time_entry_ids %} {% else %} @@ -230,9 +230,9 @@