From f7abaa02620d76c560329298628f661bdb6127f0 Mon Sep 17 00:00:00 2001 From: Mihai Criveti Date: Mon, 18 Aug 2025 09:15:50 +0100 Subject: [PATCH 1/6] Metadata / creation dates Signed-off-by: Mihai Criveti --- mcpgateway/admin.py | 53 ++- ...a0c4_add_comprehensive_metadata_to_all_.py | 82 +++++ mcpgateway/db.py | 78 +++++ mcpgateway/main.py | 40 ++- mcpgateway/schemas.py | 75 +++++ mcpgateway/services/tool_service.py | 57 +++- mcpgateway/static/admin.js | 69 ++++ mcpgateway/utils/metadata_capture.py | 267 +++++++++++++++ .../integration/test_metadata_integration.py | 287 +++++++++++++++++ .../mcpgateway/utils/test_metadata_capture.py | 303 ++++++++++++++++++ 10 files changed, 1303 insertions(+), 8 deletions(-) create mode 100644 mcpgateway/alembic/versions/34492f99a0c4_add_comprehensive_metadata_to_all_.py create mode 100644 mcpgateway/utils/metadata_capture.py create mode 100644 tests/integration/test_metadata_integration.py create mode 100644 tests/unit/mcpgateway/utils/test_metadata_capture.py diff --git a/mcpgateway/admin.py b/mcpgateway/admin.py index 002802fd3..d8e973283 100644 --- a/mcpgateway/admin.py +++ b/mcpgateway/admin.py @@ -27,6 +27,7 @@ from pathlib import Path import time from typing import Any, cast, Dict, List, Optional, Union +import uuid # Third-Party from fastapi import APIRouter, Depends, HTTPException, Request, Response @@ -40,6 +41,7 @@ # First-Party from mcpgateway.config import settings from mcpgateway.db import get_db, GlobalConfig +from mcpgateway.db import Tool as DbTool from mcpgateway.models import LogLevel from mcpgateway.schemas import ( GatewayCreate, @@ -80,6 +82,7 @@ from mcpgateway.services.tool_service import ToolError, ToolNotFoundError, ToolService from mcpgateway.utils.create_jwt_token import get_jwt_token from mcpgateway.utils.error_formatter import ErrorFormatter +from mcpgateway.utils.metadata_capture import MetadataCapture from mcpgateway.utils.passthrough_headers import PassthroughHeadersError from mcpgateway.utils.retry_manager import ResilientHttpClient from mcpgateway.utils.security_cookies import set_auth_cookie @@ -1975,7 +1978,20 @@ async def admin_add_tool( try: tool = ToolCreate(**tool_data) LOGGER.debug(f"Validated tool data: {tool.model_dump(by_alias=True)}") - await tool_service.register_tool(db, tool) + + # Extract creation metadata + metadata = MetadataCapture.extract_creation_metadata(request, user) + + await tool_service.register_tool( + db, + tool, + created_by=metadata["created_by"], + created_from_ip=metadata["created_from_ip"], + created_via=metadata["created_via"], + created_user_agent=metadata["created_user_agent"], + import_batch_id=metadata["import_batch_id"], + federation_source=metadata["federation_source"], + ) return JSONResponse( content={"message": "Tool registered successfully!", "success": True}, status_code=200, @@ -2198,7 +2214,23 @@ async def admin_edit_tool( LOGGER.debug(f"Tool update data built: {tool_data}") try: tool = ToolUpdate(**tool_data) # Pydantic validation happens here - await tool_service.update_tool(db, tool_id, tool) + + # Get current tool to extract current version + current_tool = db.get(DbTool, tool_id) + current_version = getattr(current_tool, "version", 0) if current_tool else 0 + + # Extract modification metadata + mod_metadata = MetadataCapture.extract_modification_metadata(request, user, current_version) + + await tool_service.update_tool( + db, + tool_id, + tool, + modified_by=mod_metadata["modified_by"], + modified_from_ip=mod_metadata["modified_from_ip"], + modified_via=mod_metadata["modified_via"], + modified_user_agent=mod_metadata["modified_user_agent"], + ) return JSONResponse(content={"message": "Edit tool successfully", "success": True}, status_code=200) except IntegrityError as ex: error_message = ErrorFormatter.format_database_error(ex) @@ -4458,11 +4490,26 @@ async def admin_import_tools( created, errors = [], [] # ---------- import loop ---------- + # Generate import batch ID for this bulk operation + import_batch_id = str(uuid.uuid4()) + + # Extract base metadata for bulk import + base_metadata = MetadataCapture.extract_creation_metadata(request, user, import_batch_id=import_batch_id) + for i, item in enumerate(payload): name = (item or {}).get("name") try: tool = ToolCreate(**item) # pydantic validation - await tool_service.register_tool(db, tool) + await tool_service.register_tool( + db, + tool, + created_by=base_metadata["created_by"], + created_from_ip=base_metadata["created_from_ip"], + created_via="import", # Override to show this is bulk import + created_user_agent=base_metadata["created_user_agent"], + import_batch_id=import_batch_id, + federation_source=base_metadata["federation_source"], + ) created.append({"index": i, "name": name}) except IntegrityError as ex: # The formatter can itself throw; guard it. diff --git a/mcpgateway/alembic/versions/34492f99a0c4_add_comprehensive_metadata_to_all_.py b/mcpgateway/alembic/versions/34492f99a0c4_add_comprehensive_metadata_to_all_.py new file mode 100644 index 000000000..eb81bba2a --- /dev/null +++ b/mcpgateway/alembic/versions/34492f99a0c4_add_comprehensive_metadata_to_all_.py @@ -0,0 +1,82 @@ +"""add_comprehensive_metadata_to_all_entities + +Revision ID: 34492f99a0c4 +Revises: eb17fd368f9d +Create Date: 2025-08-18 08:06:17.141169 + +""" + +# Standard +from typing import Sequence, Union + +# Third-Party +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision: str = "34492f99a0c4" +down_revision: Union[str, Sequence[str], None] = "eb17fd368f9d" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add comprehensive metadata columns to all entity tables for audit tracking.""" + tables = ["tools", "resources", "prompts", "servers", "gateways"] + + for table in tables: + # Creation metadata (nullable=True for backwards compatibility) + op.add_column(table, sa.Column("created_by", sa.String(), nullable=True)) + op.add_column(table, sa.Column("created_from_ip", sa.String(), nullable=True)) + op.add_column(table, sa.Column("created_via", sa.String(), nullable=True)) + op.add_column(table, sa.Column("created_user_agent", sa.Text(), nullable=True)) + + # Modification metadata (nullable=True for backwards compatibility) + op.add_column(table, sa.Column("modified_by", sa.String(), nullable=True)) + op.add_column(table, sa.Column("modified_from_ip", sa.String(), nullable=True)) + op.add_column(table, sa.Column("modified_via", sa.String(), nullable=True)) + op.add_column(table, sa.Column("modified_user_agent", sa.Text(), nullable=True)) + + # Source tracking (nullable=True for backwards compatibility) + op.add_column(table, sa.Column("import_batch_id", sa.String(), nullable=True)) + op.add_column(table, sa.Column("federation_source", sa.String(), nullable=True)) + op.add_column(table, sa.Column("version", sa.Integer(), nullable=False, server_default="1")) + + # Create indexes for query performance (PostgreSQL compatible, SQLite ignores) + try: + op.create_index(f"idx_{table}_created_by", table, ["created_by"]) + op.create_index(f"idx_{table}_created_at", table, ["created_at"]) + op.create_index(f"idx_{table}_modified_at", table, ["modified_at"]) + op.create_index(f"idx_{table}_created_via", table, ["created_via"]) + except Exception: # nosec B110 - database compatibility + # SQLite doesn't support all index types, skip silently + pass + + +def downgrade() -> None: + """Remove comprehensive metadata columns from all entity tables.""" + tables = ["tools", "resources", "prompts", "servers", "gateways"] + + for table in tables: + # Drop indexes first (if they exist) + try: + op.drop_index(f"idx_{table}_created_by", table) + op.drop_index(f"idx_{table}_created_at", table) + op.drop_index(f"idx_{table}_modified_at", table) + op.drop_index(f"idx_{table}_created_via", table) + except Exception: # nosec B110 - database compatibility + # Indexes might not exist on SQLite + pass + + # Drop metadata columns + op.drop_column(table, "version") + op.drop_column(table, "federation_source") + op.drop_column(table, "import_batch_id") + op.drop_column(table, "modified_user_agent") + op.drop_column(table, "modified_via") + op.drop_column(table, "modified_from_ip") + op.drop_column(table, "modified_by") + op.drop_column(table, "created_user_agent") + op.drop_column(table, "created_via") + op.drop_column(table, "created_from_ip") + op.drop_column(table, "created_by") diff --git a/mcpgateway/db.py b/mcpgateway/db.py index e1f0573a8..90d723c85 100644 --- a/mcpgateway/db.py +++ b/mcpgateway/db.py @@ -348,6 +348,21 @@ class Tool(Base): jsonpath_filter: Mapped[str] = mapped_column(default="") tags: Mapped[List[str]] = mapped_column(JSON, default=list, nullable=False) + # Comprehensive metadata for audit tracking + created_by: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_from_ip: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_via: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_user_agent: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + modified_by: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_from_ip: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_via: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_user_agent: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + import_batch_id: Mapped[Optional[str]] = mapped_column(String, nullable=True) + federation_source: Mapped[Optional[str]] = mapped_column(String, nullable=True) + version: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + # Request type and authentication fields auth_type: Mapped[Optional[str]] = mapped_column(default=None) # "basic", "bearer", or None auth_value: Mapped[Optional[str]] = mapped_column(default=None) @@ -593,6 +608,22 @@ class Resource(Base): updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now, onupdate=utc_now) is_active: Mapped[bool] = mapped_column(default=True) tags: Mapped[List[str]] = mapped_column(JSON, default=list, nullable=False) + + # Comprehensive metadata for audit tracking + created_by: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_from_ip: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_via: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_user_agent: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + modified_by: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_from_ip: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_via: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_user_agent: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + import_batch_id: Mapped[Optional[str]] = mapped_column(String, nullable=True) + federation_source: Mapped[Optional[str]] = mapped_column(String, nullable=True) + version: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + metrics: Mapped[List["ResourceMetric"]] = relationship("ResourceMetric", back_populates="resource", cascade="all, delete-orphan") # Content storage - can be text or binary @@ -804,6 +835,22 @@ class Prompt(Base): updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now, onupdate=utc_now) is_active: Mapped[bool] = mapped_column(default=True) tags: Mapped[List[str]] = mapped_column(JSON, default=list, nullable=False) + + # Comprehensive metadata for audit tracking + created_by: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_from_ip: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_via: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_user_agent: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + modified_by: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_from_ip: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_via: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_user_agent: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + import_batch_id: Mapped[Optional[str]] = mapped_column(String, nullable=True) + federation_source: Mapped[Optional[str]] = mapped_column(String, nullable=True) + version: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + metrics: Mapped[List["PromptMetric"]] = relationship("PromptMetric", back_populates="prompt", cascade="all, delete-orphan") gateway_id: Mapped[Optional[str]] = mapped_column(ForeignKey("gateways.id")) @@ -972,6 +1019,22 @@ class Server(Base): updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now, onupdate=utc_now) is_active: Mapped[bool] = mapped_column(default=True) tags: Mapped[List[str]] = mapped_column(JSON, default=list, nullable=False) + + # Comprehensive metadata for audit tracking + created_by: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_from_ip: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_via: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_user_agent: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + modified_by: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_from_ip: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_via: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_user_agent: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + import_batch_id: Mapped[Optional[str]] = mapped_column(String, nullable=True) + federation_source: Mapped[Optional[str]] = mapped_column(String, nullable=True) + version: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + metrics: Mapped[List["ServerMetric"]] = relationship("ServerMetric", back_populates="server", cascade="all, delete-orphan") # Many-to-many relationships for associated items @@ -1108,6 +1171,21 @@ class Gateway(Base): last_seen: Mapped[Optional[datetime]] tags: Mapped[List[str]] = mapped_column(JSON, default=list, nullable=False) + # Comprehensive metadata for audit tracking + created_by: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_from_ip: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_via: Mapped[Optional[str]] = mapped_column(String, nullable=True) + created_user_agent: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + modified_by: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_from_ip: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_via: Mapped[Optional[str]] = mapped_column(String, nullable=True) + modified_user_agent: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + import_batch_id: Mapped[Optional[str]] = mapped_column(String, nullable=True) + federation_source: Mapped[Optional[str]] = mapped_column(String, nullable=True) + version: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + # Header passthrough configuration passthrough_headers: Mapped[Optional[List[str]]] = mapped_column(JSON, nullable=True) # Store list of strings as JSON array diff --git a/mcpgateway/main.py b/mcpgateway/main.py index 8e7e03175..e5e748b30 100644 --- a/mcpgateway/main.py +++ b/mcpgateway/main.py @@ -58,6 +58,7 @@ from mcpgateway.config import jsonpath_modifier, settings from mcpgateway.db import Prompt as DbPrompt from mcpgateway.db import PromptMetric, refresh_slugs_on_startup, SessionLocal +from mcpgateway.db import Tool as DbTool from mcpgateway.handlers.sampling import SamplingHandler from mcpgateway.middleware.security_headers import SecurityHeadersMiddleware from mcpgateway.models import InitializeResult, ListResourceTemplatesResult, LogLevel, ResourceContent, Root @@ -103,6 +104,7 @@ from mcpgateway.transports.streamablehttp_transport import SessionManagerWrapper, streamable_http_auth from mcpgateway.utils.db_isready import wait_for_db_ready from mcpgateway.utils.error_formatter import ErrorFormatter +from mcpgateway.utils.metadata_capture import MetadataCapture from mcpgateway.utils.passthrough_headers import set_global_passthrough_headers from mcpgateway.utils.redis_isready import wait_for_redis_ready from mcpgateway.utils.retry_manager import ResilientHttpClient @@ -1245,12 +1247,13 @@ async def list_tools( @tool_router.post("", response_model=ToolRead) @tool_router.post("/", response_model=ToolRead) -async def create_tool(tool: ToolCreate, db: Session = Depends(get_db), user: str = Depends(require_auth)) -> ToolRead: +async def create_tool(tool: ToolCreate, request: Request, db: Session = Depends(get_db), user: str = Depends(require_auth)) -> ToolRead: """ Creates a new tool in the system. Args: tool (ToolCreate): The data needed to create the tool. + request (Request): The FastAPI request object for metadata extraction. db (Session): The database session dependency. user (str): The authenticated user making the request. @@ -1261,8 +1264,21 @@ async def create_tool(tool: ToolCreate, db: Session = Depends(get_db), user: str HTTPException: If the tool name already exists or other validation errors occur. """ try: + + # Extract metadata from request + metadata = MetadataCapture.extract_creation_metadata(request, user) + logger.debug(f"User {user} is creating a new tool") - return await tool_service.register_tool(db, tool) + return await tool_service.register_tool( + db, + tool, + created_by=metadata["created_by"], + created_from_ip=metadata["created_from_ip"], + created_via=metadata["created_via"], + created_user_agent=metadata["created_user_agent"], + import_batch_id=metadata["import_batch_id"], + federation_source=metadata["federation_source"], + ) except Exception as ex: logger.error(f"Error while creating tool: {ex}") if isinstance(ex, ToolNameConflictError): @@ -1324,6 +1340,7 @@ async def get_tool( async def update_tool( tool_id: str, tool: ToolUpdate, + request: Request, db: Session = Depends(get_db), user: str = Depends(require_auth), ) -> ToolRead: @@ -1333,6 +1350,7 @@ async def update_tool( Args: tool_id (str): The ID of the tool to update. tool (ToolUpdate): The updated tool information. + request (Request): The FastAPI request object for metadata extraction. db (Session): The database session dependency. user (str): The authenticated user making the request. @@ -1343,8 +1361,24 @@ async def update_tool( HTTPException: If an error occurs during the update. """ try: + + # Get current tool to extract current version + current_tool = db.get(DbTool, tool_id) + current_version = getattr(current_tool, "version", 0) if current_tool else 0 + + # Extract modification metadata + mod_metadata = MetadataCapture.extract_modification_metadata(request, user, current_version) + logger.debug(f"User {user} is updating tool with ID {tool_id}") - return await tool_service.update_tool(db, tool_id, tool) + return await tool_service.update_tool( + db, + tool_id, + tool, + modified_by=mod_metadata["modified_by"], + modified_from_ip=mod_metadata["modified_from_ip"], + modified_via=mod_metadata["modified_via"], + modified_user_agent=mod_metadata["modified_user_agent"], + ) except Exception as ex: if isinstance(ex, ToolNotFoundError): raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(ex)) diff --git a/mcpgateway/schemas.py b/mcpgateway/schemas.py index 9f0a68c1e..24238937e 100644 --- a/mcpgateway/schemas.py +++ b/mcpgateway/schemas.py @@ -836,6 +836,21 @@ class ToolRead(BaseModelWithConfigDict): original_name_slug: str tags: List[str] = Field(default_factory=list, description="Tags for categorizing the tool") + # Comprehensive metadata for audit tracking + created_by: Optional[str] = Field(None, description="Username who created this entity") + created_from_ip: Optional[str] = Field(None, description="IP address of creator") + created_via: Optional[str] = Field(None, description="Creation method: ui|api|import|federation") + created_user_agent: Optional[str] = Field(None, description="User agent of creation request") + + modified_by: Optional[str] = Field(None, description="Username who last modified this entity") + modified_from_ip: Optional[str] = Field(None, description="IP address of last modifier") + modified_via: Optional[str] = Field(None, description="Modification method") + modified_user_agent: Optional[str] = Field(None, description="User agent of modification request") + + import_batch_id: Optional[str] = Field(None, description="UUID of bulk import batch") + federation_source: Optional[str] = Field(None, description="Source gateway for federated entities") + version: Optional[int] = Field(1, description="Entity version for change tracking") + class ToolInvocation(BaseModelWithConfigDict): """Schema for tool invocation requests. @@ -1258,6 +1273,21 @@ class ResourceRead(BaseModelWithConfigDict): metrics: ResourceMetrics tags: List[str] = Field(default_factory=list, description="Tags for categorizing the resource") + # Comprehensive metadata for audit tracking + created_by: Optional[str] = Field(None, description="Username who created this entity") + created_from_ip: Optional[str] = Field(None, description="IP address of creator") + created_via: Optional[str] = Field(None, description="Creation method: ui|api|import|federation") + created_user_agent: Optional[str] = Field(None, description="User agent of creation request") + + modified_by: Optional[str] = Field(None, description="Username who last modified this entity") + modified_from_ip: Optional[str] = Field(None, description="IP address of last modifier") + modified_via: Optional[str] = Field(None, description="Modification method") + modified_user_agent: Optional[str] = Field(None, description="User agent of modification request") + + import_batch_id: Optional[str] = Field(None, description="UUID of bulk import batch") + federation_source: Optional[str] = Field(None, description="Source gateway for federated entities") + version: Optional[int] = Field(1, description="Entity version for change tracking") + class ResourceSubscription(BaseModelWithConfigDict): """Schema for resource subscriptions. @@ -1715,6 +1745,21 @@ class PromptRead(BaseModelWithConfigDict): tags: List[str] = Field(default_factory=list, description="Tags for categorizing the prompt") metrics: PromptMetrics + # Comprehensive metadata for audit tracking + created_by: Optional[str] = Field(None, description="Username who created this entity") + created_from_ip: Optional[str] = Field(None, description="IP address of creator") + created_via: Optional[str] = Field(None, description="Creation method: ui|api|import|federation") + created_user_agent: Optional[str] = Field(None, description="User agent of creation request") + + modified_by: Optional[str] = Field(None, description="Username who last modified this entity") + modified_from_ip: Optional[str] = Field(None, description="IP address of last modifier") + modified_via: Optional[str] = Field(None, description="Modification method") + modified_user_agent: Optional[str] = Field(None, description="User agent of modification request") + + import_batch_id: Optional[str] = Field(None, description="UUID of bulk import batch") + federation_source: Optional[str] = Field(None, description="Source gateway for federated entities") + version: Optional[int] = Field(1, description="Entity version for change tracking") + class PromptInvocation(BaseModelWithConfigDict): """Schema for prompt invocation requests. @@ -2265,6 +2310,21 @@ class GatewayRead(BaseModelWithConfigDict): auth_header_value: Optional[str] = Field(None, description="vallue for custom headers authentication") tags: List[str] = Field(default_factory=list, description="Tags for categorizing the gateway") + # Comprehensive metadata for audit tracking + created_by: Optional[str] = Field(None, description="Username who created this entity") + created_from_ip: Optional[str] = Field(None, description="IP address of creator") + created_via: Optional[str] = Field(None, description="Creation method: ui|api|import|federation") + created_user_agent: Optional[str] = Field(None, description="User agent of creation request") + + modified_by: Optional[str] = Field(None, description="Username who last modified this entity") + modified_from_ip: Optional[str] = Field(None, description="IP address of last modifier") + modified_via: Optional[str] = Field(None, description="Modification method") + modified_user_agent: Optional[str] = Field(None, description="User agent of modification request") + + import_batch_id: Optional[str] = Field(None, description="UUID of bulk import batch") + federation_source: Optional[str] = Field(None, description="Source gateway for federated entities") + version: Optional[int] = Field(1, description="Entity version for change tracking") + slug: str = Field(None, description="Slug for gateway endpoint URL") # This will be the main method to automatically populate fields @@ -2851,6 +2911,21 @@ class ServerRead(BaseModelWithConfigDict): metrics: ServerMetrics tags: List[str] = Field(default_factory=list, description="Tags for categorizing the server") + # Comprehensive metadata for audit tracking + created_by: Optional[str] = Field(None, description="Username who created this entity") + created_from_ip: Optional[str] = Field(None, description="IP address of creator") + created_via: Optional[str] = Field(None, description="Creation method: ui|api|import|federation") + created_user_agent: Optional[str] = Field(None, description="User agent of creation request") + + modified_by: Optional[str] = Field(None, description="Username who last modified this entity") + modified_from_ip: Optional[str] = Field(None, description="IP address of last modifier") + modified_via: Optional[str] = Field(None, description="Modification method") + modified_user_agent: Optional[str] = Field(None, description="User agent of modification request") + + import_batch_id: Optional[str] = Field(None, description="UUID of bulk import batch") + federation_source: Optional[str] = Field(None, description="Source gateway for federated entities") + version: Optional[int] = Field(1, description="Entity version for change tracking") + @model_validator(mode="before") @classmethod def populate_associated_ids(cls, values): diff --git a/mcpgateway/services/tool_service.py b/mcpgateway/services/tool_service.py index 381727806..f891b2bcb 100644 --- a/mcpgateway/services/tool_service.py +++ b/mcpgateway/services/tool_service.py @@ -308,12 +308,28 @@ async def _record_tool_metric(self, db: Session, tool: DbTool, start_time: float db.add(metric) db.commit() - async def register_tool(self, db: Session, tool: ToolCreate) -> ToolRead: + async def register_tool( + self, + db: Session, + tool: ToolCreate, + created_by: Optional[str] = None, + created_from_ip: Optional[str] = None, + created_via: Optional[str] = None, + created_user_agent: Optional[str] = None, + import_batch_id: Optional[str] = None, + federation_source: Optional[str] = None, + ) -> ToolRead: """Register a new tool. Args: db: Database session. tool: Tool creation schema. + created_by: Username who created this tool. + created_from_ip: IP address of creator. + created_via: Creation method (ui, api, import, federation). + created_user_agent: User agent of creation request. + import_batch_id: UUID for bulk import operations. + federation_source: Source gateway for federated tools. Returns: Created tool information. @@ -368,6 +384,14 @@ async def register_tool(self, db: Session, tool: ToolCreate) -> ToolRead: auth_value=auth_value, gateway_id=tool.gateway_id, tags=tool.tags or [], + # Metadata fields + created_by=created_by, + created_from_ip=created_from_ip, + created_via=created_via, + created_user_agent=created_user_agent, + import_batch_id=import_batch_id, + federation_source=federation_source, + version=1, ) db.add(db_tool) db.commit() @@ -863,7 +887,16 @@ async def connect_to_streamablehttp_server(server_url: str): span.set_attribute("duration.ms", (time.monotonic() - start_time) * 1000) await self._record_tool_metric(db, tool, start_time, success, error_message) - async def update_tool(self, db: Session, tool_id: str, tool_update: ToolUpdate) -> ToolRead: + async def update_tool( + self, + db: Session, + tool_id: str, + tool_update: ToolUpdate, + modified_by: Optional[str] = None, + modified_from_ip: Optional[str] = None, + modified_via: Optional[str] = None, + modified_user_agent: Optional[str] = None, + ) -> ToolRead: """ Update an existing tool. @@ -871,6 +904,10 @@ async def update_tool(self, db: Session, tool_id: str, tool_update: ToolUpdate) db (Session): The SQLAlchemy database session. tool_id (str): The unique identifier of the tool. tool_update (ToolUpdate): Tool update schema with new data. + modified_by (Optional[str]): Username who modified this tool. + modified_from_ip (Optional[str]): IP address of modifier. + modified_via (Optional[str]): Modification method (ui, api). + modified_user_agent (Optional[str]): User agent of modification request. Returns: The updated ToolRead object. @@ -933,6 +970,22 @@ async def update_tool(self, db: Session, tool_id: str, tool_update: ToolUpdate) if tool_update.tags is not None: tool.tags = tool_update.tags + # Update modification metadata + if modified_by is not None: + tool.modified_by = modified_by + if modified_from_ip is not None: + tool.modified_from_ip = modified_from_ip + if modified_via is not None: + tool.modified_via = modified_via + if modified_user_agent is not None: + tool.modified_user_agent = modified_user_agent + + # Increment version + if hasattr(tool, "version") and tool.version is not None: + tool.version += 1 + else: + tool.version = 1 + tool.updated_at = datetime.now(timezone.utc) db.commit() db.refresh(tool) diff --git a/mcpgateway/static/admin.js b/mcpgateway/static/admin.js index d8fed398b..0b5e3c572 100644 --- a/mcpgateway/static/admin.js +++ b/mcpgateway/static/admin.js @@ -4914,6 +4914,43 @@ async function viewTool(toolId) {
  • Last Execution Time:
    • +
    + Metadata: +
    +
    + Created By: + +
    +
    + Created At: + +
    +
    + Created From: + +
    +
    + Created Via: + +
    +
    + Last Modified By: + +
    +
    + Last Modified At: + +
    +
    + Version: + +
    +
    + Import Batch: + +
    +
    +
    `; @@ -5010,6 +5047,38 @@ async function viewTool(toolId) { ".metric-last-time", tool.metrics?.lastExecutionTime ?? "N/A", ); + + // Set metadata fields safely with appropriate fallbacks for legacy entities + setTextSafely( + ".metadata-created-by", + tool.createdBy || "Legacy Entity", + ); + setTextSafely( + ".metadata-created-at", + tool.createdAt + ? new Date(tool.createdAt).toLocaleString() + : "Pre-metadata", + ); + setTextSafely( + ".metadata-created-from", + tool.createdFromIp || "Unknown", + ); + setTextSafely( + ".metadata-created-via", + tool.createdVia || "Unknown", + ); + setTextSafely(".metadata-modified-by", tool.modifiedBy || "N/A"); + setTextSafely( + ".metadata-modified-at", + tool.modifiedAt + ? new Date(tool.modifiedAt).toLocaleString() + : "N/A", + ); + setTextSafely(".metadata-version", tool.version || "1"); + setTextSafely( + ".metadata-import-batch", + tool.importBatchId || "N/A", + ); } openModal("tool-modal"); diff --git a/mcpgateway/utils/metadata_capture.py b/mcpgateway/utils/metadata_capture.py new file mode 100644 index 000000000..600273c70 --- /dev/null +++ b/mcpgateway/utils/metadata_capture.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- +"""Metadata capture utilities for comprehensive audit tracking. + +Copyright 2025 +SPDX-License-Identifier: Apache-2.0 +Authors: Mihai Criveti + +This module provides utilities for capturing comprehensive metadata during +entity creation and modification operations. It extracts request context +information such as authenticated user, IP address, user agent, and source +type for audit trail purposes. + +Examples: + >>> from mcpgateway.utils.metadata_capture import MetadataCapture + >>> from types import SimpleNamespace + >>> # Create mock request for testing + >>> request = SimpleNamespace() + >>> request.client = SimpleNamespace() + >>> request.client.host = "192.168.1.1" + >>> request.headers = {"user-agent": "test/1.0"} + >>> request.url = SimpleNamespace() + >>> request.url.path = "/admin/tools" + >>> # Metadata capture during entity creation + >>> metadata = MetadataCapture.extract_creation_metadata(request, user="admin") + >>> metadata["created_by"] + 'admin' + >>> metadata["created_via"] + 'ui' +""" + +# Standard +from typing import Dict, Optional + +# Third-Party +from fastapi import Request + + +class MetadataCapture: + """Utilities for capturing comprehensive metadata during entity operations.""" + + @staticmethod + def extract_request_context(request: Request) -> Dict[str, Optional[str]]: + """Extract basic request context information. + + Args: + request: FastAPI request object + + Returns: + Dict containing IP address, user agent, and source type + + Examples: + >>> # Mock request for testing + >>> from types import SimpleNamespace + >>> mock_request = SimpleNamespace() + >>> mock_request.client = SimpleNamespace() + >>> mock_request.client.host = "192.168.1.100" + >>> mock_request.headers = {"user-agent": "Mozilla/5.0"} + >>> mock_request.url = SimpleNamespace() + >>> mock_request.url.path = "/admin/tools" + >>> context = MetadataCapture.extract_request_context(mock_request) + >>> context["from_ip"] + '192.168.1.100' + >>> context["via"] + 'ui' + """ + # Extract IP address (handle various proxy scenarios) + client_ip = None + if request.client: + client_ip = request.client.host + + # Check for forwarded headers (reverse proxy support) + forwarded_for = request.headers.get("x-forwarded-for") + if forwarded_for: + # Take the first IP in the chain (original client) + client_ip = forwarded_for.split(",")[0].strip() + + # Extract user agent + user_agent = request.headers.get("user-agent") + + # Determine source type based on URL path + via = "api" # default + if hasattr(request, "url") and hasattr(request.url, "path"): + path = str(request.url.path) + if "/admin/" in path: + via = "ui" + + return { + "from_ip": client_ip, + "user_agent": user_agent, + "via": via, + } + + @staticmethod + def extract_creation_metadata( + request: Request, + user: str, + import_batch_id: Optional[str] = None, + federation_source: Optional[str] = None, + ) -> Dict[str, Optional[str]]: + """Extract complete metadata for entity creation. + + Args: + request: FastAPI request object + user: Authenticated username (or "anonymous" if auth disabled) + import_batch_id: Optional UUID for bulk import operations + federation_source: Optional source gateway for federated entities + + Returns: + Dict containing all creation metadata fields + + Examples: + >>> from types import SimpleNamespace + >>> mock_request = SimpleNamespace() + >>> mock_request.client = SimpleNamespace() + >>> mock_request.client.host = "10.0.0.1" + >>> mock_request.headers = {"user-agent": "curl/7.68.0"} + >>> mock_request.url = SimpleNamespace() + >>> mock_request.url.path = "/tools" + >>> metadata = MetadataCapture.extract_creation_metadata(mock_request, "admin") + >>> metadata["created_by"] + 'admin' + >>> metadata["created_via"] + 'api' + >>> metadata["created_from_ip"] + '10.0.0.1' + """ + context = MetadataCapture.extract_request_context(request) + + return { + "created_by": user, + "created_from_ip": context["from_ip"], + "created_via": context["via"], + "created_user_agent": context["user_agent"], + "import_batch_id": import_batch_id, + "federation_source": federation_source, + "version": 1, + } + + @staticmethod + def extract_modification_metadata( + request: Request, + user: str, + current_version: int = 1, + ) -> Dict[str, Optional[str]]: + """Extract metadata for entity modification. + + Args: + request: FastAPI request object + user: Authenticated username (or "anonymous" if auth disabled) + current_version: Current entity version (will be incremented) + + Returns: + Dict containing modification metadata fields + + Examples: + >>> from types import SimpleNamespace + >>> mock_request = SimpleNamespace() + >>> mock_request.client = SimpleNamespace() + >>> mock_request.client.host = "172.16.0.1" + >>> mock_request.headers = {"user-agent": "HTTPie/2.4.0"} + >>> mock_request.url = SimpleNamespace() + >>> mock_request.url.path = "/admin/tools/123/edit" + >>> metadata = MetadataCapture.extract_modification_metadata(mock_request, "alice", 2) + >>> metadata["modified_by"] + 'alice' + >>> metadata["modified_via"] + 'ui' + >>> metadata["version"] + 3 + """ + context = MetadataCapture.extract_request_context(request) + + return { + "modified_by": user, + "modified_from_ip": context["from_ip"], + "modified_via": context["via"], + "modified_user_agent": context["user_agent"], + "version": current_version + 1, + } + + @staticmethod + def determine_source_from_context( + import_batch_id: Optional[str] = None, + federation_source: Optional[str] = None, + via: str = "api", + ) -> str: + """Determine the source type based on available context. + + Args: + import_batch_id: UUID for bulk import operations + federation_source: Source gateway for federated entities + via: Basic source type (api, ui) + + Returns: + More specific source description + + Examples: + >>> MetadataCapture.determine_source_from_context(via="ui") + 'ui' + >>> MetadataCapture.determine_source_from_context(import_batch_id="123", via="api") + 'import' + >>> MetadataCapture.determine_source_from_context(federation_source="gateway-1", via="api") + 'federation' + """ + if import_batch_id: + return "import" + elif federation_source: + return "federation" + else: + return via + + @staticmethod + def sanitize_user_agent(user_agent: Optional[str]) -> Optional[str]: + """Sanitize user agent string for safe storage and display. + + Args: + user_agent: Raw user agent string from request headers + + Returns: + Sanitized user agent string or None + + Examples: + >>> MetadataCapture.sanitize_user_agent("Mozilla/5.0 (Linux)") + 'Mozilla/5.0 (Linux)' + >>> MetadataCapture.sanitize_user_agent(None) + >>> len(MetadataCapture.sanitize_user_agent("x" * 2000)) <= 503 + True + """ + if not user_agent: + return None + + # Truncate excessively long user agents + if len(user_agent) > 500: + user_agent = user_agent[:500] + "..." + + # Remove any potentially dangerous characters + user_agent = user_agent.replace("\n", " ").replace("\r", " ").replace("\t", " ") + + return user_agent.strip() + + @staticmethod + def validate_ip_address(ip_address: Optional[str]) -> Optional[str]: + """Validate and sanitize IP address for storage. + + Args: + ip_address: IP address string from request + + Returns: + Validated IP address or None + + Examples: + >>> MetadataCapture.validate_ip_address("192.168.1.1") + '192.168.1.1' + >>> MetadataCapture.validate_ip_address("::1") + '::1' + >>> MetadataCapture.validate_ip_address(None) + >>> MetadataCapture.validate_ip_address("invalid-ip") + 'invalid-ip' + """ + if not ip_address: + return None + + # Basic validation - store as-is but limit length + if len(ip_address) > 45: # Max length for IPv6 + return ip_address[:45] + + return ip_address.strip() diff --git a/tests/integration/test_metadata_integration.py b/tests/integration/test_metadata_integration.py new file mode 100644 index 000000000..77886f490 --- /dev/null +++ b/tests/integration/test_metadata_integration.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- +"""Integration tests for metadata tracking feature. + +Copyright 2025 +SPDX-License-Identifier: Apache-2.0 +Authors: Mihai Criveti + +This module tests the complete metadata tracking functionality across +the entire application stack, including API endpoints, database storage, +and UI integration. +""" + +# Standard +import asyncio +from datetime import datetime +import json +import uuid +from typing import Dict + +# Third-Party +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +# First-Party +from mcpgateway.db import Base, get_db, Tool as DbTool +from mcpgateway.main import app +from mcpgateway.schemas import ToolCreate +from mcpgateway.services.tool_service import ToolService +from mcpgateway.utils.verify_credentials import require_auth + + +@pytest.fixture +def test_app(): + """Create test app with in-memory database.""" + # Create in-memory SQLite database for testing + engine = create_engine("sqlite:///:memory:", connect_args={"check_same_thread": False}) + TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + + Base.metadata.create_all(bind=engine) + + def override_get_db(): + try: + db = TestingSessionLocal() + yield db + finally: + db.close() + + app.dependency_overrides[get_db] = override_get_db + app.dependency_overrides[require_auth] = lambda: "test_user" + + yield app + + # Cleanup + app.dependency_overrides.clear() + + +@pytest.fixture +def client(test_app): + """Create test client.""" + return TestClient(test_app) + + +class TestMetadataIntegration: + """Integration tests for metadata tracking across the application.""" + + def test_tool_creation_api_metadata(self, client): + """Test that tool creation via API captures metadata correctly.""" + unique_name = f"api_test_tool_{uuid.uuid4().hex[:8]}" + tool_data = { + "name": unique_name, + "url": "http://example.com/api", + "description": "Tool created via API", + "integration_type": "REST", + "request_type": "GET" + } + + response = client.post("/tools", json=tool_data) + assert response.status_code == 200 + + tool = response.json() + + # Verify metadata was captured + assert tool["createdBy"] == "test_user" + assert tool["createdVia"] == "api" # Should detect API call + assert tool["version"] == 1 + assert tool["createdFromIp"] is not None # Should capture some IP + + # Verify metadata is properly serialized + assert "createdAt" in tool + # modifiedAt is only set after modifications, not during creation + + def test_tool_creation_admin_ui_metadata(self, client): + """Test that tool creation via admin UI works with metadata.""" + tool_data = { + "name": f"admin_ui_test_tool_{uuid.uuid4().hex[:8]}", + "url": "http://example.com/admin", + "description": "Tool created via admin UI", + "integrationType": "REST", + "requestType": "GET" + } + + # Simulate admin UI request + response = client.post("/admin/tools", data=tool_data) + + # Admin endpoint might return different status codes, just verify it doesn't crash + assert response.status_code in [200, 400, 422, 500] # Allow various responses + + # The important thing is that the metadata capture code doesn't break the endpoint + + def test_tool_update_metadata(self, client): + """Test that tool updates capture modification metadata.""" + # First create a tool + tool_data = { + "name": f"update_test_tool_{uuid.uuid4().hex[:8]}", + "url": "http://example.com/test", + "description": "Tool for update testing", + "integration_type": "REST", + "request_type": "GET" + } + + create_response = client.post("/tools", json=tool_data) + assert create_response.status_code == 200 + tool_id = create_response.json()["id"] + + # Now update the tool + update_data = { + "description": "Updated description" + } + + update_response = client.put(f"/tools/{tool_id}", json=update_data) + assert update_response.status_code == 200 + + updated_tool = update_response.json() + + # Verify modification metadata + assert updated_tool["modifiedBy"] == "test_user" + assert updated_tool["modifiedVia"] == "api" + assert updated_tool["version"] == 2 # Should increment + assert updated_tool["description"] == "Updated description" + + def test_metadata_backwards_compatibility(self, client): + """Test that metadata works with legacy entities.""" + # Create a tool and then manually remove metadata to simulate legacy entity + tool_data = { + "name": f"legacy_simulation_tool_{uuid.uuid4().hex[:8]}", + "url": "http://example.com/legacy", + "description": "Simulated legacy tool", + "integration_type": "REST", + "request_type": "GET" + } + + response = client.post("/tools", json=tool_data) + assert response.status_code == 200 + tool = response.json() + + # Even "legacy" simulation should have metadata since we're testing new code + # But verify that optional fields handle None gracefully + assert tool["createdBy"] is not None # Should have metadata + assert "version" in tool + assert tool["version"] >= 1 + + def test_auth_disabled_metadata(self, client, test_app): + """Test metadata capture when authentication is disabled.""" + # Override auth to return anonymous + test_app.dependency_overrides[require_auth] = lambda: "anonymous" + + tool_data = { + "name": f"anonymous_test_tool_{uuid.uuid4().hex[:8]}", + "url": "http://example.com/anon", + "description": "Tool created anonymously", + "integration_type": "REST", + "request_type": "GET" + } + + response = client.post("/tools", json=tool_data) + assert response.status_code == 200 + + tool = response.json() + + # Verify anonymous metadata + assert tool["createdBy"] == "anonymous" + assert tool["version"] == 1 + assert tool["createdVia"] == "api" + + def test_metadata_fields_in_tool_read_schema(self, client): + """Test that all metadata fields are present in API responses.""" + tool_data = { + "name": f"schema_test_tool_{uuid.uuid4().hex[:8]}", + "url": "http://example.com/schema", + "description": "Tool for schema testing", + "integration_type": "REST", + "request_type": "GET" + } + + response = client.post("/tools", json=tool_data) + assert response.status_code == 200 + + tool = response.json() + + # Verify all metadata fields are present + expected_fields = [ + "createdBy", "createdFromIp", "createdVia", "createdUserAgent", + "modifiedBy", "modifiedFromIp", "modifiedVia", "modifiedUserAgent", + "importBatchId", "federationSource", "version" + ] + + for field in expected_fields: + assert field in tool, f"Missing metadata field: {field}" + + def test_tool_list_includes_metadata(self, client): + """Test that tool list endpoint includes metadata fields.""" + # Create a tool first + tool_data = { + "name": f"list_test_tool_{uuid.uuid4().hex[:8]}", + "url": "http://example.com/list", + "description": "Tool for list testing", + "integration_type": "REST", + "request_type": "GET" + } + + client.post("/tools", json=tool_data) + + # List tools + response = client.get("/tools") + assert response.status_code == 200 + + tools = response.json() + assert len(tools) > 0 + + # Verify metadata is included in list response + tool = tools[0] + assert "createdBy" in tool + assert "version" in tool + + @pytest.mark.asyncio + async def test_service_layer_metadata_handling(self): + """Test metadata handling at the service layer.""" + from mcpgateway.db import SessionLocal + from mcpgateway.utils.metadata_capture import MetadataCapture + from types import SimpleNamespace + + # Create mock request + mock_request = SimpleNamespace() + mock_request.client = SimpleNamespace() + mock_request.client.host = "test-ip" + mock_request.headers = {"user-agent": "test-agent"} + mock_request.url = SimpleNamespace() + mock_request.url.path = "/admin/tools" + + # Extract metadata + metadata = MetadataCapture.extract_creation_metadata(mock_request, "service_test_user") + + # Create tool data + tool_data = ToolCreate( + name=f"service_layer_test_{uuid.uuid4().hex[:8]}", + url="http://example.com/service", + description="Service layer test tool", + integration_type="REST", + request_type="GET" + ) + + # Test service creation with metadata + service = ToolService() + db = SessionLocal() + + try: + tool_read = await service.register_tool( + db, + tool_data, + created_by=metadata["created_by"], + created_from_ip=metadata["created_from_ip"], + created_via=metadata["created_via"], + created_user_agent=metadata["created_user_agent"], + ) + + # Verify metadata was stored + assert tool_read.created_by == "service_test_user" + assert tool_read.created_from_ip == "test-ip" + assert tool_read.created_via == "ui" + assert tool_read.created_user_agent == "test-agent" + assert tool_read.version == 1 + + finally: + db.close() \ No newline at end of file diff --git a/tests/unit/mcpgateway/utils/test_metadata_capture.py b/tests/unit/mcpgateway/utils/test_metadata_capture.py new file mode 100644 index 000000000..8ffc4ed70 --- /dev/null +++ b/tests/unit/mcpgateway/utils/test_metadata_capture.py @@ -0,0 +1,303 @@ +# -*- coding: utf-8 -*- +"""Unit tests for metadata capture utilities. + +Copyright 2025 +SPDX-License-Identifier: Apache-2.0 +Authors: Mihai Criveti + +This module tests the metadata capture functionality for comprehensive +audit tracking of entity creation and modification operations. +""" + +# Standard +from types import SimpleNamespace +from unittest.mock import MagicMock + +# Third-Party +import pytest + +# First-Party +from mcpgateway.utils.metadata_capture import MetadataCapture + + +class TestMetadataCapture: + """Test cases for MetadataCapture utility class.""" + + def test_extract_request_context_basic(self): + """Test basic request context extraction.""" + # Create mock request + request = SimpleNamespace() + request.client = SimpleNamespace() + request.client.host = "192.168.1.100" + request.headers = {"user-agent": "Mozilla/5.0 (Linux)"} + request.url = SimpleNamespace() + request.url.path = "/tools" + + context = MetadataCapture.extract_request_context(request) + + assert context["from_ip"] == "192.168.1.100" + assert context["user_agent"] == "Mozilla/5.0 (Linux)" + assert context["via"] == "api" + + def test_extract_request_context_admin_ui(self): + """Test request context extraction for admin UI.""" + request = SimpleNamespace() + request.client = SimpleNamespace() + request.client.host = "10.0.0.1" + request.headers = {"user-agent": "Chrome/90.0"} + request.url = SimpleNamespace() + request.url.path = "/admin/tools" + + context = MetadataCapture.extract_request_context(request) + + assert context["from_ip"] == "10.0.0.1" + assert context["via"] == "ui" + + def test_extract_request_context_proxy_headers(self): + """Test IP extraction with proxy headers.""" + request = SimpleNamespace() + request.client = SimpleNamespace() + request.client.host = "127.0.0.1" + request.headers = { + "user-agent": "curl/7.68.0", + "x-forwarded-for": "203.0.113.1, 192.168.1.1, 127.0.0.1" + } + request.url = SimpleNamespace() + request.url.path = "/api/tools" + + context = MetadataCapture.extract_request_context(request) + + # Should use first IP from X-Forwarded-For + assert context["from_ip"] == "203.0.113.1" + assert context["user_agent"] == "curl/7.68.0" + + def test_extract_request_context_no_client(self): + """Test request context when client info is missing.""" + request = SimpleNamespace() + request.client = None + request.headers = {"user-agent": "test/1.0"} + request.url = SimpleNamespace() + request.url.path = "/tools" + + context = MetadataCapture.extract_request_context(request) + + assert context["from_ip"] is None + assert context["user_agent"] == "test/1.0" + assert context["via"] == "api" + + def test_extract_creation_metadata(self): + """Test complete creation metadata extraction.""" + request = SimpleNamespace() + request.client = SimpleNamespace() + request.client.host = "172.16.0.5" + request.headers = {"user-agent": "HTTPie/2.4.0"} + request.url = SimpleNamespace() + request.url.path = "/admin/servers" + + metadata = MetadataCapture.extract_creation_metadata( + request, + "admin", + import_batch_id="batch-123", + federation_source="gateway-prod" + ) + + assert metadata["created_by"] == "admin" + assert metadata["created_from_ip"] == "172.16.0.5" + assert metadata["created_via"] == "ui" + assert metadata["created_user_agent"] == "HTTPie/2.4.0" + assert metadata["import_batch_id"] == "batch-123" + assert metadata["federation_source"] == "gateway-prod" + assert metadata["version"] == 1 + + def test_extract_creation_metadata_anonymous_user(self): + """Test creation metadata with anonymous user.""" + request = SimpleNamespace() + request.client = SimpleNamespace() + request.client.host = "192.168.1.1" + request.headers = {"user-agent": "test-client"} + request.url = SimpleNamespace() + request.url.path = "/tools" + + metadata = MetadataCapture.extract_creation_metadata(request, "anonymous") + + assert metadata["created_by"] == "anonymous" + assert metadata["created_via"] == "api" + assert metadata["version"] == 1 + + def test_extract_modification_metadata(self): + """Test modification metadata extraction.""" + request = SimpleNamespace() + request.client = SimpleNamespace() + request.client.host = "10.1.1.1" + request.headers = {"user-agent": "PostmanRuntime/7.28.0"} + request.url = SimpleNamespace() + request.url.path = "/tools/123" + + metadata = MetadataCapture.extract_modification_metadata(request, "alice", 3) + + assert metadata["modified_by"] == "alice" + assert metadata["modified_from_ip"] == "10.1.1.1" + assert metadata["modified_via"] == "api" + assert metadata["modified_user_agent"] == "PostmanRuntime/7.28.0" + assert metadata["version"] == 4 # current_version + 1 + + def test_determine_source_from_context_import(self): + """Test source determination for bulk import.""" + source = MetadataCapture.determine_source_from_context( + import_batch_id="batch-456", + via="api" + ) + assert source == "import" + + def test_determine_source_from_context_federation(self): + """Test source determination for federation.""" + source = MetadataCapture.determine_source_from_context( + federation_source="gateway-east", + via="api" + ) + assert source == "federation" + + def test_determine_source_from_context_normal(self): + """Test source determination for normal operations.""" + source = MetadataCapture.determine_source_from_context(via="ui") + assert source == "ui" + + def test_sanitize_user_agent_normal(self): + """Test normal user agent sanitization.""" + result = MetadataCapture.sanitize_user_agent("Mozilla/5.0 (Windows NT 10.0)") + assert result == "Mozilla/5.0 (Windows NT 10.0)" + + def test_sanitize_user_agent_none(self): + """Test user agent sanitization with None input.""" + result = MetadataCapture.sanitize_user_agent(None) + assert result is None + + def test_sanitize_user_agent_empty(self): + """Test user agent sanitization with empty string.""" + result = MetadataCapture.sanitize_user_agent("") + assert result is None + + def test_sanitize_user_agent_long(self): + """Test user agent sanitization with overly long input.""" + long_ua = "x" * 1000 + result = MetadataCapture.sanitize_user_agent(long_ua) + assert len(result) == 503 # 500 + "..." + assert result.endswith("...") + + def test_sanitize_user_agent_with_special_chars(self): + """Test user agent sanitization with special characters.""" + ua_with_newlines = "Mozilla/5.0\n(Linux;\r\tX11)" + result = MetadataCapture.sanitize_user_agent(ua_with_newlines) + assert "\n" not in result + assert "\r" not in result + assert "\t" not in result + assert result == "Mozilla/5.0 (Linux; X11)" + + def test_validate_ip_address_ipv4(self): + """Test IP address validation for IPv4.""" + result = MetadataCapture.validate_ip_address("192.168.1.1") + assert result == "192.168.1.1" + + def test_validate_ip_address_ipv6(self): + """Test IP address validation for IPv6.""" + result = MetadataCapture.validate_ip_address("2001:db8::1") + assert result == "2001:db8::1" + + def test_validate_ip_address_none(self): + """Test IP address validation with None.""" + result = MetadataCapture.validate_ip_address(None) + assert result is None + + def test_validate_ip_address_empty(self): + """Test IP address validation with empty string.""" + result = MetadataCapture.validate_ip_address("") + assert result is None + + def test_validate_ip_address_long(self): + """Test IP address validation with overly long input.""" + long_ip = "x" * 100 + result = MetadataCapture.validate_ip_address(long_ip) + assert len(result) == 45 # Truncated to max IPv6 length + + def test_validate_ip_address_with_whitespace(self): + """Test IP address validation with whitespace.""" + result = MetadataCapture.validate_ip_address(" 192.168.1.1 ") + assert result == "192.168.1.1" + + def test_extract_creation_metadata_all_none(self): + """Test creation metadata with all optional parameters None.""" + request = SimpleNamespace() + request.client = SimpleNamespace() + request.client.host = "192.168.1.1" + request.headers = {"user-agent": "test"} + request.url = SimpleNamespace() + request.url.path = "/tools" + + metadata = MetadataCapture.extract_creation_metadata( + request, + "user", + import_batch_id=None, + federation_source=None + ) + + assert metadata["created_by"] == "user" + assert metadata["import_batch_id"] is None + assert metadata["federation_source"] is None + + def test_extract_modification_metadata_default_version(self): + """Test modification metadata with default version.""" + request = SimpleNamespace() + request.client = SimpleNamespace() + request.client.host = "10.0.0.1" + request.headers = {"user-agent": "test"} + request.url = SimpleNamespace() + request.url.path = "/tools/123" + + metadata = MetadataCapture.extract_modification_metadata(request, "bob") + + assert metadata["modified_by"] == "bob" + assert metadata["version"] == 2 # 1 + 1 + + def test_edge_case_no_url_attribute(self): + """Test edge case where request has no url attribute.""" + request = SimpleNamespace() + request.client = SimpleNamespace() + request.client.host = "192.168.1.1" + request.headers = {"user-agent": "test"} + # No url attribute + + context = MetadataCapture.extract_request_context(request) + + assert context["from_ip"] == "192.168.1.1" + assert context["via"] == "api" # default when no path available + + def test_edge_case_no_headers(self): + """Test edge case where request has no headers.""" + request = SimpleNamespace() + request.client = SimpleNamespace() + request.client.host = "192.168.1.1" + request.headers = {} + request.url = SimpleNamespace() + request.url.path = "/tools" + + context = MetadataCapture.extract_request_context(request) + + assert context["user_agent"] is None + assert context["from_ip"] == "192.168.1.1" + + def test_edge_case_malformed_forwarded_header(self): + """Test edge case with malformed X-Forwarded-For header.""" + request = SimpleNamespace() + request.client = SimpleNamespace() + request.client.host = "127.0.0.1" + request.headers = { + "user-agent": "test", + "x-forwarded-for": "malformed" + } + request.url = SimpleNamespace() + request.url.path = "/tools" + + context = MetadataCapture.extract_request_context(request) + + # Should still extract the forwarded IP even if malformed + assert context["from_ip"] == "malformed" \ No newline at end of file From 3e72edfc9b0c20322c9ff58c78c14611bdd7c20f Mon Sep 17 00:00:00 2001 From: Mihai Criveti Date: Mon, 18 Aug 2025 10:00:18 +0100 Subject: [PATCH 2/6] Metadata / creation dates Signed-off-by: Mihai Criveti --- mcpgateway/admin.py | 28 +- ...a0c4_add_comprehensive_metadata_to_all_.py | 1 + mcpgateway/main.py | 30 +- mcpgateway/services/gateway_service.py | 41 ++- mcpgateway/services/prompt_service.py | 26 +- mcpgateway/static/admin.js | 189 ++++++++++ mcpgateway/templates/admin.html | 322 ++++++++++++++---- mcpgateway/utils/metadata_capture.py | 38 ++- .../integration/test_metadata_integration.py | 80 ++--- .../mcpgateway/utils/test_metadata_capture.py | 84 +++-- 10 files changed, 689 insertions(+), 150 deletions(-) diff --git a/mcpgateway/admin.py b/mcpgateway/admin.py index d8e973283..8b56055ce 100644 --- a/mcpgateway/admin.py +++ b/mcpgateway/admin.py @@ -2690,7 +2690,17 @@ async def admin_add_gateway(request: Request, db: Session = Depends(get_db), use return JSONResponse(content={"success": False, "message": "; ".join(error_ctx)}, status_code=422) try: - await gateway_service.register_gateway(db, gateway) + # Extract creation metadata + metadata = MetadataCapture.extract_creation_metadata(request, user) + + await gateway_service.register_gateway( + db, + gateway, + created_by=metadata["created_by"], + created_from_ip=metadata["created_from_ip"], + created_via=metadata["created_via"], + created_user_agent=metadata["created_user_agent"], + ) return JSONResponse( content={"message": "Gateway registered successfully!", "success": True}, status_code=200, @@ -3606,7 +3616,7 @@ async def admin_add_prompt(request: Request, db: Session = Depends(get_db), user try: args_json = "[]" args_value = form.get("arguments") - if isinstance(args_value, str): + if isinstance(args_value, str) and args_value.strip(): args_json = args_value arguments = json.loads(args_json) prompt = PromptCreate( @@ -3616,7 +3626,19 @@ async def admin_add_prompt(request: Request, db: Session = Depends(get_db), user arguments=arguments, tags=tags, ) - await prompt_service.register_prompt(db, prompt) + # Extract creation metadata + metadata = MetadataCapture.extract_creation_metadata(request, user) + + await prompt_service.register_prompt( + db, + prompt, + created_by=metadata["created_by"], + created_from_ip=metadata["created_from_ip"], + created_via=metadata["created_via"], + created_user_agent=metadata["created_user_agent"], + import_batch_id=metadata["import_batch_id"], + federation_source=metadata["federation_source"], + ) return JSONResponse( content={"message": "Prompt registered successfully!", "success": True}, status_code=200, diff --git a/mcpgateway/alembic/versions/34492f99a0c4_add_comprehensive_metadata_to_all_.py b/mcpgateway/alembic/versions/34492f99a0c4_add_comprehensive_metadata_to_all_.py index eb81bba2a..0c1eae9dc 100644 --- a/mcpgateway/alembic/versions/34492f99a0c4_add_comprehensive_metadata_to_all_.py +++ b/mcpgateway/alembic/versions/34492f99a0c4_add_comprehensive_metadata_to_all_.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """add_comprehensive_metadata_to_all_entities Revision ID: 34492f99a0c4 diff --git a/mcpgateway/main.py b/mcpgateway/main.py index e5e748b30..799445d41 100644 --- a/mcpgateway/main.py +++ b/mcpgateway/main.py @@ -1775,6 +1775,7 @@ async def list_prompts( @prompt_router.post("/", response_model=PromptRead) async def create_prompt( prompt: PromptCreate, + request: Request, db: Session = Depends(get_db), user: str = Depends(require_auth), ) -> PromptRead: @@ -1783,6 +1784,7 @@ async def create_prompt( Args: prompt (PromptCreate): Payload describing the prompt to create. + request (Request): The FastAPI request object for metadata extraction. db (Session): Active SQLAlchemy session. user (str): Authenticated username. @@ -1796,7 +1798,19 @@ async def create_prompt( """ logger.debug(f"User: {user} requested to create prompt: {prompt}") try: - return await prompt_service.register_prompt(db, prompt) + # Extract metadata from request + metadata = MetadataCapture.extract_creation_metadata(request, user) + + return await prompt_service.register_prompt( + db, + prompt, + created_by=metadata["created_by"], + created_from_ip=metadata["created_from_ip"], + created_via=metadata["created_via"], + created_user_agent=metadata["created_user_agent"], + import_batch_id=metadata["import_batch_id"], + federation_source=metadata["federation_source"], + ) except Exception as e: if isinstance(e, PromptNameConflictError): # If the prompt name already exists, return a 409 Conflict error @@ -2087,6 +2101,7 @@ async def list_gateways( @gateway_router.post("/", response_model=GatewayRead) async def register_gateway( gateway: GatewayCreate, + request: Request, db: Session = Depends(get_db), user: str = Depends(require_auth), ) -> GatewayRead: @@ -2095,6 +2110,7 @@ async def register_gateway( Args: gateway: Gateway creation data. + request: The FastAPI request object for metadata extraction. db: Database session. user: Authenticated user. @@ -2103,7 +2119,17 @@ async def register_gateway( """ logger.debug(f"User '{user}' requested to register gateway: {gateway}") try: - return await gateway_service.register_gateway(db, gateway) + # Extract metadata from request + metadata = MetadataCapture.extract_creation_metadata(request, user) + + return await gateway_service.register_gateway( + db, + gateway, + created_by=metadata["created_by"], + created_from_ip=metadata["created_from_ip"], + created_via=metadata["created_via"], + created_user_agent=metadata["created_user_agent"], + ) except Exception as ex: if isinstance(ex, GatewayConnectionError): return JSONResponse(content={"message": "Unable to connect to gateway"}, status_code=status.HTTP_503_SERVICE_UNAVAILABLE) diff --git a/mcpgateway/services/gateway_service.py b/mcpgateway/services/gateway_service.py index e85dced2a..79ab646d5 100644 --- a/mcpgateway/services/gateway_service.py +++ b/mcpgateway/services/gateway_service.py @@ -390,12 +390,24 @@ async def shutdown(self) -> None: self._active_gateways.clear() logger.info("Gateway service shutdown complete") - async def register_gateway(self, db: Session, gateway: GatewayCreate) -> GatewayRead: + async def register_gateway( + self, + db: Session, + gateway: GatewayCreate, + created_by: Optional[str] = None, + created_from_ip: Optional[str] = None, + created_via: Optional[str] = None, + created_user_agent: Optional[str] = None, + ) -> GatewayRead: """Register a new gateway. Args: db: Database session gateway: Gateway creation schema + created_by: Username who created this gateway + created_from_ip: IP address of creator + created_via: Creation method (ui, api, federation) + created_user_agent: User agent of creation request Returns: Created gateway information @@ -463,6 +475,13 @@ async def register_gateway(self, db: Session, gateway: GatewayCreate) -> Gateway jsonpath_filter=tool.jsonpath_filter, auth_type=auth_type, auth_value=auth_value, + # Federation metadata + created_by=created_by or "system", + created_from_ip=created_from_ip, + created_via="federation", # These are federated tools + created_user_agent=created_user_agent, + federation_source=gateway.name, + version=1, ) for tool in tools ] @@ -475,6 +494,13 @@ async def register_gateway(self, db: Session, gateway: GatewayCreate) -> Gateway description=resource.description, mime_type=resource.mime_type, template=resource.template, + # Federation metadata + created_by=created_by or "system", + created_from_ip=created_from_ip, + created_via="federation", # These are federated resources + created_user_agent=created_user_agent, + federation_source=gateway.name, + version=1, ) for resource in resources ] @@ -486,6 +512,13 @@ async def register_gateway(self, db: Session, gateway: GatewayCreate) -> Gateway description=prompt.description, template=prompt.template if hasattr(prompt, "template") else "", argument_schema={}, # Use argument_schema instead of arguments + # Federation metadata + created_by=created_by or "system", + created_from_ip=created_from_ip, + created_via="federation", # These are federated prompts + created_user_agent=created_user_agent, + federation_source=gateway.name, + version=1, ) for prompt in prompts ] @@ -505,6 +538,12 @@ async def register_gateway(self, db: Session, gateway: GatewayCreate) -> Gateway tools=tools, resources=db_resources, prompts=db_prompts, + # Gateway metadata + created_by=created_by, + created_from_ip=created_from_ip, + created_via=created_via or "api", + created_user_agent=created_user_agent, + version=1, ) # Add to DB diff --git a/mcpgateway/services/prompt_service.py b/mcpgateway/services/prompt_service.py index 608b8b277..ebe38bd02 100644 --- a/mcpgateway/services/prompt_service.py +++ b/mcpgateway/services/prompt_service.py @@ -240,12 +240,28 @@ def _convert_db_prompt(self, db_prompt: DbPrompt) -> Dict[str, Any]: "tags": db_prompt.tags or [], } - async def register_prompt(self, db: Session, prompt: PromptCreate) -> PromptRead: + async def register_prompt( + self, + db: Session, + prompt: PromptCreate, + created_by: Optional[str] = None, + created_from_ip: Optional[str] = None, + created_via: Optional[str] = None, + created_user_agent: Optional[str] = None, + import_batch_id: Optional[str] = None, + federation_source: Optional[str] = None, + ) -> PromptRead: """Register a new prompt template. Args: db: Database session prompt: Prompt creation schema + created_by: Username who created this prompt + created_from_ip: IP address of creator + created_via: Creation method (ui, api, import, federation) + created_user_agent: User agent of creation request + import_batch_id: UUID for bulk import operations + federation_source: Source gateway for federated prompts Returns: Created prompt information @@ -298,6 +314,14 @@ async def register_prompt(self, db: Session, prompt: PromptCreate) -> PromptRead template=prompt.template, argument_schema=argument_schema, tags=prompt.tags, + # Metadata fields + created_by=created_by, + created_from_ip=created_from_ip, + created_via=created_via, + created_user_agent=created_user_agent, + import_batch_id=import_batch_id, + federation_source=federation_source, + version=1, ) # Add to DB diff --git a/mcpgateway/static/admin.js b/mcpgateway/static/admin.js index 0b5e3c572..7439af4e4 100644 --- a/mcpgateway/static/admin.js +++ b/mcpgateway/static/admin.js @@ -2580,6 +2580,67 @@ async function viewPrompt(promptName) { container.appendChild(metricsDiv); } + // Add metadata section + const metadataDiv = document.createElement("div"); + metadataDiv.className = "mt-6 border-t pt-4"; + + const metadataTitle = document.createElement("strong"); + metadataTitle.textContent = "Metadata:"; + metadataDiv.appendChild(metadataTitle); + + const metadataGrid = document.createElement("div"); + metadataGrid.className = "grid grid-cols-2 gap-4 mt-2 text-sm"; + + const metadataFields = [ + { + label: "Created By", + value: prompt.createdBy || "Legacy Entity", + }, + { + label: "Created At", + value: prompt.createdAt + ? new Date(prompt.createdAt).toLocaleString() + : "Pre-metadata", + }, + { + label: "Created From", + value: prompt.createdFromIp || "Unknown", + }, + { label: "Created Via", value: prompt.createdVia || "Unknown" }, + { + label: "Last Modified By", + value: prompt.modifiedBy || "N/A", + }, + { + label: "Last Modified At", + value: prompt.modifiedAt + ? new Date(prompt.modifiedAt).toLocaleString() + : "N/A", + }, + { label: "Version", value: prompt.version || "1" }, + { label: "Import Batch", value: prompt.importBatchId || "N/A" }, + ]; + + metadataFields.forEach((field) => { + const fieldDiv = document.createElement("div"); + + const labelSpan = document.createElement("span"); + labelSpan.className = + "font-medium text-gray-600 dark:text-gray-400"; + labelSpan.textContent = field.label + ":"; + + const valueSpan = document.createElement("span"); + valueSpan.className = "ml-2"; + valueSpan.textContent = field.value; + + fieldDiv.appendChild(labelSpan); + fieldDiv.appendChild(valueSpan); + metadataGrid.appendChild(fieldDiv); + }); + + metadataDiv.appendChild(metadataGrid); + container.appendChild(metadataDiv); + // Replace content safely promptDetailsDiv.innerHTML = ""; promptDetailsDiv.appendChild(container); @@ -2792,6 +2853,73 @@ async function viewGateway(gatewayId) { statusP.appendChild(statusSpan); container.appendChild(statusP); + // Add metadata section + const metadataDiv = document.createElement("div"); + metadataDiv.className = "mt-6 border-t pt-4"; + + const metadataTitle = document.createElement("strong"); + metadataTitle.textContent = "Metadata:"; + metadataDiv.appendChild(metadataTitle); + + const metadataGrid = document.createElement("div"); + metadataGrid.className = "grid grid-cols-2 gap-4 mt-2 text-sm"; + + const metadataFields = [ + { + label: "Created By", + value: gateway.createdBy || "Legacy Entity", + }, + { + label: "Created At", + value: gateway.createdAt + ? new Date(gateway.createdAt).toLocaleString() + : "Pre-metadata", + }, + { + label: "Created From", + value: gateway.createdFromIp || "Unknown", + }, + { + label: "Created Via", + value: gateway.createdVia || "Unknown", + }, + { + label: "Last Modified By", + value: gateway.modifiedBy || "N/A", + }, + { + label: "Last Modified At", + value: gateway.modifiedAt + ? new Date(gateway.modifiedAt).toLocaleString() + : "N/A", + }, + { label: "Version", value: gateway.version || "1" }, + { + label: "Import Batch", + value: gateway.importBatchId || "N/A", + }, + ]; + + metadataFields.forEach((field) => { + const fieldDiv = document.createElement("div"); + + const labelSpan = document.createElement("span"); + labelSpan.className = + "font-medium text-gray-600 dark:text-gray-400"; + labelSpan.textContent = field.label + ":"; + + const valueSpan = document.createElement("span"); + valueSpan.className = "ml-2"; + valueSpan.textContent = field.value; + + fieldDiv.appendChild(labelSpan); + fieldDiv.appendChild(valueSpan); + metadataGrid.appendChild(fieldDiv); + }); + + metadataDiv.appendChild(metadataGrid); + container.appendChild(metadataDiv); + gatewayDetailsDiv.innerHTML = ""; gatewayDetailsDiv.appendChild(container); } @@ -3052,6 +3180,67 @@ async function viewServer(serverId) { statusP.appendChild(statusSpan); container.appendChild(statusP); + // Add metadata section + const metadataDiv = document.createElement("div"); + metadataDiv.className = "mt-6 border-t pt-4"; + + const metadataTitle = document.createElement("strong"); + metadataTitle.textContent = "Metadata:"; + metadataDiv.appendChild(metadataTitle); + + const metadataGrid = document.createElement("div"); + metadataGrid.className = "grid grid-cols-2 gap-4 mt-2 text-sm"; + + const metadataFields = [ + { + label: "Created By", + value: server.createdBy || "Legacy Entity", + }, + { + label: "Created At", + value: server.createdAt + ? new Date(server.createdAt).toLocaleString() + : "Pre-metadata", + }, + { + label: "Created From", + value: server.createdFromIp || "Unknown", + }, + { label: "Created Via", value: server.createdVia || "Unknown" }, + { + label: "Last Modified By", + value: server.modifiedBy || "N/A", + }, + { + label: "Last Modified At", + value: server.modifiedAt + ? new Date(server.modifiedAt).toLocaleString() + : "N/A", + }, + { label: "Version", value: server.version || "1" }, + { label: "Import Batch", value: server.importBatchId || "N/A" }, + ]; + + metadataFields.forEach((field) => { + const fieldDiv = document.createElement("div"); + + const labelSpan = document.createElement("span"); + labelSpan.className = + "font-medium text-gray-600 dark:text-gray-400"; + labelSpan.textContent = field.label + ":"; + + const valueSpan = document.createElement("span"); + valueSpan.className = "ml-2"; + valueSpan.textContent = field.value; + + fieldDiv.appendChild(labelSpan); + fieldDiv.appendChild(valueSpan); + metadataGrid.appendChild(fieldDiv); + }); + + metadataDiv.appendChild(metadataGrid); + container.appendChild(metadataDiv); + serverDetailsDiv.innerHTML = ""; serverDetailsDiv.appendChild(container); } diff --git a/mcpgateway/templates/admin.html b/mcpgateway/templates/admin.html index 2953d620f..be530d5da 100644 --- a/mcpgateway/templates/admin.html +++ b/mcpgateway/templates/admin.html @@ -359,13 +359,17 @@

    -

    +

    Configuration Export & Import

    -

    +

    📤 Export Configuration

    @@ -373,82 +377,167 @@

    -
    -
    -