cocalc-api/test: fix edgecases in api and improve testing

Author: haraldschilly

.github/workflows/make-and-test.yml

@@ -278,6 +278,7 @@ jobs:
         run: |
           export COCALC_API_KEY=$(cat src/api_key.txt)
           export COCALC_HOST=http://localhost:5000
+          export CI=true
           cd src/python/cocalc-api && make ci
         env:
           PGDATABASE: smc

src/packages/conat/hub/api/projects.ts

@@ -1,6 +1,10 @@
 import { authFirstRequireAccount } from "./util";
 import { type CreateProjectOptions } from "@cocalc/util/db-schema/projects";
 import { type UserCopyOptions } from "@cocalc/util/db-schema/projects";
+import {
+  type ProjectState,
+  type ProjectStatus,
+} from "@cocalc/util/db-schema/projects";
 
 export const projects = {
   createProject: authFirstRequireAccount,
@@ -13,6 +17,9 @@ export const projects = {
   start: authFirstRequireAccount,
   stop: authFirstRequireAccount,
   deleteProject: authFirstRequireAccount,
+  touch: authFirstRequireAccount,
+  state: authFirstRequireAccount,
+  status: authFirstRequireAccount,
 };
 
 export type AddCollaborator =
@@ -104,5 +111,18 @@ export interface Projects {
 
   start: (opts: { account_id: string; project_id: string }) => Promise<void>;
   stop: (opts: { account_id: string; project_id: string }) => Promise<void>;
-  deleteProject: (opts: { account_id: string; project_id: string }) => Promise<void>;
+  deleteProject: (opts: {
+    account_id: string;
+    project_id: string;
+  }) => Promise<void>;
+
+  touch: (opts: { account_id: string; project_id: string }) => Promise<void>;
+  state: (opts: {
+    account_id: string;
+    project_id: string;
+  }) => Promise<ProjectState>;
+  status: (opts: {
+    account_id: string;
+    project_id: string;
+  }) => Promise<ProjectStatus>;
 }

src/packages/server/accounts/get-name.ts

@@ -37,8 +37,8 @@ type Names = {
 function canonicalName(row) {
   // some accounts have these null for some reason sometimes, but it is nice if client code can assume not null.
   let { first_name = "", last_name = "", profile } = row;
-  first_name = first_name.trim();
-  last_name = last_name.trim();
+  first_name = (first_name ?? "").trim();
+  last_name = (last_name ?? "").trim();
   if (!first_name && !last_name) {
     // Also ensure both are not empty so you can always see something.  I think the frontend and/or api doesn't
     // allow a user to make their name empty, but *just in case* we do this.

src/packages/server/conat/api/projects.ts

@@ -106,3 +106,45 @@ export async function stop({
 
 import deleteProject from "@cocalc/server/projects/delete";
 export { deleteProject };
+
+export async function touch({
+  account_id,
+  project_id,
+}: {
+  account_id: string;
+  project_id: string;
+}): Promise<void> {
+  if (!(await isCollaborator({ account_id, project_id }))) {
+    throw Error("must be collaborator on project to touch it");
+  }
+  const project = await getProject(project_id);
+  await project.touch(account_id);
+}
+
+export async function state({
+  account_id,
+  project_id,
+}: {
+  account_id: string;
+  project_id: string;
+}): Promise<any> {
+  if (!(await isCollaborator({ account_id, project_id }))) {
+    throw Error("must be collaborator on project to get state");
+  }
+  const project = await getProject(project_id);
+  return await project.state();
+}
+
+export async function status({
+  account_id,
+  project_id,
+}: {
+  account_id: string;
+  project_id: string;
+}): Promise<any> {
+  if (!(await isCollaborator({ account_id, project_id }))) {
+    throw Error("must be collaborator on project to get status");
+  }
+  const project = await getProject(project_id);
+  return await project.status();
+}

src/python/cocalc-api/Makefile

@@ -37,8 +37,11 @@ test:
 test-verbose:
 	uv run pytest -v
 
-ci:
-	uv run pytest --junitxml=test-results.xml --cov=src --cov-report=term-missing --cov-report=html
+ci-create-project:
+	@uv run python3 scripts/ci-create-project.py
+
+ci: ci-create-project
+	export COCALC_PROJECT_ID=$$($(MAKE) ci-create-project) && uv run pytest --junitxml=test-results.xml --cov=src --cov-report=term-missing --cov-report=html
 
 coverage:
 	uv run pytest --cov=src --cov-report=term-missing --cov-report=html

src/python/cocalc-api/pyproject.toml

@@ -51,6 +51,7 @@ omit = [
     "*/__pycache__/*",
     "*/venv/*",
     "*/env/*",
+    "*/mcp/*",
 ]
 
 [tool.coverage.report]

src/python/cocalc-api/scripts/ci-create-project.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""
+Create a test project for CI tests using the cocalc-api Hub.
+
+This script creates a temporary project for running tests that require a project ID
+(e.g., cocalc-api tests with account-scoped API keys).
+
+Outputs the project ID to stdout so it can be captured by the CI workflow.
+"""
+
+import os
+import sys
+import httpx
+
+
+def main():
+    """Create a test project and output its ID."""
+    # Get configuration from environment
+    api_key = os.environ.get("COCALC_API_KEY")
+    host = os.environ.get("COCALC_HOST", "http://localhost:5000")
+
+    if not api_key:
+        print("Error: COCALC_API_KEY environment variable not set", file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        # Create HTTP client with API key authentication
+        client = httpx.Client(
+            auth=(api_key, ""),
+            headers={"Content-Type": "application/json"},
+            timeout=120.0
+        )
+
+        # Call the hub API to create a project
+        resp = client.post(
+            f"{host}/api/conat/hub",
+            json={
+                "name": "projects.createProject",
+                "args": [{"title": "CI Test Project"}]
+            }
+        )
+
+        result = resp.json()
+
+        # Check for errors in the response
+        if "error" in result:
+            print(f"Error creating project: {result['error']}", file=sys.stderr)
+            sys.exit(1)
+
+        # The result should be the project ID
+        project_id = result
+        if not project_id or project_id == "None":
+            print(f"Error: Invalid project ID returned: {result}", file=sys.stderr)
+            sys.exit(1)
+
+        # Output the project ID to stdout
+        print(project_id)
+
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

src/python/cocalc-api/src/cocalc_api/hub.py

@@ -147,7 +147,7 @@ def user_search(self, query: str) -> UserSearchResult:
               'created': 1756056224470,
               'email_address_verified': None}]
         """
-        ...
+        ...  # pragma: no cover
 
 
 class Projects:
@@ -246,7 +246,7 @@ def add_collaborator(self, project_id: str | list[str], account_id: str | list[s
         Returns:
             dict[str, Any]: JSON response from the API.
         """
-        ...
+        ...  # pragma: no cover
 
     @api_method("projects.removeCollaborator", opts=True)
     def remove_collaborator(self, project_id: str, account_id: str) -> dict[str, Any]:
@@ -260,7 +260,7 @@ def remove_collaborator(self, project_id: str, account_id: str) -> dict[str, Any
         Returns:
             dict[str, Any]: JSON response from the API.
         """
-        ...
+        ...  # pragma: no cover
 
     @api_method("projects.start")
     def start(self, project_id: str) -> dict[str, Any]:
@@ -295,6 +295,55 @@ def delete(self, project_id: str) -> dict[str, Any]:
         """
         ...
 
+    @api_method("projects.touch")
+    def touch(self, project_id: str) -> dict[str, Any]:
+        """
+        Signal that the project is in use by updating its last_edited timestamp.
+        This also ensures the project is started.
+
+        Args:
+            project_id (str): Project ID of the project to touch.
+
+        Returns:
+            dict[str, Any]: API response indicating success.
+        """
+        ...
+
+    @api_method("projects.state")
+    def state(self, project_id: str) -> dict[str, Any]:
+        """
+        Get the current state of a project (running, stopped, starting, etc.).
+
+        Args:
+            project_id (str): Project ID of the project.
+
+        Returns:
+            dict[str, Any]: Project state object containing:
+                - state: "running" | "stopped" | "starting" | "restarting" | "error"
+                - ip: IP address where project is running (if running)
+                - error: Error message (if in error state)
+                - time: Timestamp of last state change
+        """
+        ...
+
+    @api_method("projects.status")
+    def status(self, project_id: str) -> dict[str, Any]:
+        """
+        Get detailed status information about a project.
+
+        Args:
+            project_id (str): Project ID of the project.
+
+        Returns:
+            dict[str, Any]: Project status object containing:
+                - project.pid: PID of project server process
+                - start_ts: Timestamp when project started
+                - version: Project code version
+                - disk_MB: Disk usage in MB
+                - memory: Memory usage information
+        """
+        ...
+
 
 class Jupyter:
 
@@ -324,7 +373,7 @@ def kernels(self, project_id: Optional[str] = None) -> list[dict[str, Any]]:
             >>> 'python3' in kernel_names
             True
         """
-        ...
+        ...  # pragma: no cover
 
     @api_method("jupyter.execute", timeout_seconds=True)
     def execute(
@@ -365,7 +414,7 @@ def execute(
             ...     project_id='6e75dbf1-0342-4249-9dce-6b21648656e9')
             {'output': [{'data': {'text/plain': '3^4 * 5^2'}}], ...}
         """
-        ...
+        ...  # pragma: no cover
 
 
 class Sync:
@@ -385,7 +434,7 @@ def history(self, project_id: str, path: str) -> list[dict[str, Any]]:  # type:
         Returns:
             list[dict[str, Any]]: Array of patches in a compressed diff-match-patch format, along with time and user data.
         """
-        ...
+        ...  # pragma: no cover
 
 
 class Database:
@@ -420,7 +469,7 @@ def query(self, query: dict[str, Any]) -> dict[str, Any]:
             >>> hub.db.query({"accounts":{"first_name":None}})
             {'accounts': {'first_name': 'W'}}
         """
-        ...
+        ...  # pragma: no cover
 
 
 class Messages:
@@ -442,7 +491,7 @@ def send(self, subject: str, body: str, to_ids: list[str], reply_id: Optional[in
         Returns:
             int: ID of the message.
         """
-        ...
+        ...  # pragma: no cover
 
     @api_method("messages.get")
     def get(
@@ -462,7 +511,7 @@ def get(
         Returns:
             list[MessageType]: List of messages.
         """
-        ...
+        ...  # pragma: no cover
 
 
 """

src/python/cocalc-api/src/cocalc_api/mcp/mcp_debug.py

@@ -43,8 +43,7 @@ async def debug_mcp_server() -> None:
             read_stream, write_stream = client
             async with ClientSession(read_stream, write_stream) as session:
                 # Initialize the connection
-                print("\n=== MCP Server Debug Information ===\n",
-                      file=sys.stderr)
+                print("\n=== MCP Server Debug Information ===\n", file=sys.stderr)
 
                 # Get capabilities
                 info = await session.initialize()
@@ -105,16 +104,13 @@ async def debug_mcp_server() -> None:
     except Exception as e:
         error_str = str(e)
         if "COCALC_API_KEY" in error_str or "not set" in error_str:
-            print("Error: COCALC_API_KEY environment variable is not set",
-                  file=sys.stderr)
+            print("Error: COCALC_API_KEY environment variable is not set", file=sys.stderr)
             print("\nUsage:", file=sys.stderr)
             print("  export COCALC_API_KEY='sk-...'", file=sys.stderr)
             print("  make mcp-debug", file=sys.stderr)
         elif "project_id" in error_str.lower():
-            print("Error: Project-scoped API key requires COCALC_PROJECT_ID",
-                  file=sys.stderr)
-            print("\nFor project-scoped API keys, provide the project ID:",
-                  file=sys.stderr)
+            print("Error: Project-scoped API key requires COCALC_PROJECT_ID", file=sys.stderr)
+            print("\nFor project-scoped API keys, provide the project ID:", file=sys.stderr)
             print("  export COCALC_API_KEY='sk-...'", file=sys.stderr)
             print("  export COCALC_PROJECT_ID='uuid-...'", file=sys.stderr)
             print("  make mcp-debug", file=sys.stderr)