diff --git a/server/HACKING.md b/server/HACKING.md index 032feb448..0561da3d5 100644 --- a/server/HACKING.md +++ b/server/HACKING.md @@ -16,7 +16,7 @@ This will setup Testflinger running on port 5000, along with MongoDB and Vault. To get this running on your system: ```shell -docker-compose up -d --build +docker compose up -d --build ``` If you want to manage client credentials in your local database, you can use diff --git a/server/devel/create_sample_data.py b/server/devel/create_sample_data.py index d6620554a..cc7a97a17 100755 --- a/server/devel/create_sample_data.py +++ b/server/devel/create_sample_data.py @@ -63,7 +63,15 @@ def _server_validator(server: str) -> str: "--queues", type=int, default=10, - help="Number of queues to create", + help="Number of queues to distribute amongst jobs and agents", + ) + + parser.add_argument( + "-d", + "--advertised-queues", + type=int, + default=1, + help="Number of advertised queues to create", ) parser.add_argument( @@ -112,7 +120,9 @@ def __iter__(self): "state": random.choice(("waiting", "test", "provision")), } if self.queue_list: - agent_data["queues"] = [random.choice(self.queue_list)] + agent_data["queues"] = random.sample( + self.queue_list, random.randint(1, len(self.queue_list)) + ) yield (f"{self.prefix}{agent_num}", agent_data) @@ -241,7 +251,10 @@ def main(): testflinger_client = TestflingerClient(server_url=args.server) queues = QueueDataGenerator(num_queues=args.queues) - testflinger_client.post_queue_data(queues=queues) + # configure "advertised" queues: + testflinger_client.post_queue_data( + random.sample(tuple(queues), random.randint(1, args.advertised_queues)) + ) logging.info("Created %s queues", args.queues) valid_queue_names = extract_queue_names(queues=queues) diff --git a/server/devel/openapi_app.py b/server/devel/openapi_app.py index 1215fe6d6..441afa7ad 100644 --- a/server/devel/openapi_app.py +++ b/server/devel/openapi_app.py @@ -18,22 +18,24 @@ This module creates a lightweight Testflinger server instance specifically for extracting the OpenAPI specification without requiring MongoDB or other -external dependencies. -It's used by the APIFlask to generate the API spec in JSON format. +external dependencies. +It's used by the APIFlask to generate the API spec in JSON format. (https://apiflask.com/openapi/#the-flask-spec-command) Usage: uvx --with tox-uv tox -e schema """ -from dataclasses import dataclass +from dataclasses import dataclass from testflinger.application import create_flask_app + @dataclass class OpenAPIConfig: """Config for Testing.""" TESTING = True + # Create and expose the app in TESTING mode for Flask CLI to use app = create_flask_app(OpenAPIConfig) diff --git a/server/scripts/generate_openapi_schema.py b/server/scripts/generate_openapi_schema.py index 293a32511..0f89cfe8a 100755 --- a/server/scripts/generate_openapi_schema.py +++ b/server/scripts/generate_openapi_schema.py @@ -53,7 +53,7 @@ def generate_schema() -> dict: def normalize_json(data: dict) -> str: """Normalize JSON to compact form for comparison""" - return json.dumps(data, sort_keys=True, separators=(',', ':')) + return json.dumps(data, sort_keys=True, separators=(",", ":")) def diff_schemas(local_schema_path: Path) -> bool: @@ -70,7 +70,10 @@ def diff_schemas(local_schema_path: Path) -> bool: generated = generate_schema() if not local_schema_path.exists(): - print(f"Error: Expected schema file not found: {local_schema_path}", file=sys.stderr) + print( + f"Error: Expected schema file not found: {local_schema_path}", + file=sys.stderr, + ) return False with local_schema_path.open() as f: @@ -83,7 +86,10 @@ def diff_schemas(local_schema_path: Path) -> bool: if generated_normalized != local_normalized: print("Error: OpenAPI schema is out of date", file=sys.stderr) print("", file=sys.stderr) - print("To update the schema, run from server/ directory:", file=sys.stderr) + print( + "To update the schema, run from server/ directory:", + file=sys.stderr, + ) print(f" uvx --with tox-uv tox run -e schema", file=sys.stderr) return False @@ -99,13 +105,13 @@ def main(): "-o", type=Path, default=None, - help="Write schema to specified file (default is stdout)" + help="Write schema to specified file (default is stdout)", ) parser.add_argument( "--diff", "-d", type=Path, - help="Compare generated schema with the specified file for validation" + help="Compare generated schema with the specified file for validation", ) args = parser.parse_args() @@ -124,7 +130,7 @@ def main(): args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open("w") as f: json.dump(schema, f, indent=2, sort_keys=True) - f.write("\n") # trailing linebreak + f.write("\n") # trailing linebreak print(f" Schema written to: {args.output}") else: print(json.dumps(schema, indent=2, sort_keys=True)) diff --git a/server/src/testflinger/database.py b/server/src/testflinger/database.py index 7f708ef4d..d6910cb6a 100644 --- a/server/src/testflinger/database.py +++ b/server/src/testflinger/database.py @@ -280,6 +280,18 @@ def get_jobs_on_queue(queue: str) -> list[dict]: return list(jobs) +def get_num_incomplete_jobs_on_queue(queue: str) -> int: + """Get the number of incomplete jobs on a specific queue.""" + return mongo.db.jobs.count_documents( + { + "job_data.job_queue": queue, + "result_data.job_state": { + "$nin": ["complete", "completed", "cancelled"] + }, + } + ) + + def calculate_percentiles(data: list) -> dict: """ Calculate the percentiles of the wait times for each queue. diff --git a/server/src/testflinger/templates/agent_detail.html b/server/src/testflinger/templates/agent_detail.html index 9b65b035c..c32ce8341 100644 --- a/server/src/testflinger/templates/agent_detail.html +++ b/server/src/testflinger/templates/agent_detail.html @@ -32,16 +32,29 @@

Agent Detail - {{ agent.name }}

Queues

- + + + + + + + + + + {% for queue in agent.queues %} + + + + + + {% endfor %} + +
NameNumber of jobsDescription
+ {{ queue.name }} + {% if agent.restricted_to[queue.name]|default([]) %} + (restricted to: {{ agent.restricted_to[queue.name]|join(", ") }}) + {% endif %} + {{ queue.numjobs }}{{ queue.description }}

Provision History

@@ -98,7 +111,7 @@

Agent Log

{{ agent.log|join('\n') }} - +
{% endblock content %} diff --git a/server/src/testflinger/views.py b/server/src/testflinger/views.py index 67e20f6da..f12c16a6b 100644 --- a/server/src/testflinger/views.py +++ b/server/src/testflinger/views.py @@ -91,6 +91,20 @@ def agent_detail(agent_id): for queue in agent_info.get("queues", []) } + queue_info = [] + for queue_name in agent_info.pop("queues", []): + queue_data = mongo.db.queues.find_one({"name": queue_name}) + if not queue_data: + # If it's not an advertised queue, create some dummy data + queue_data = {"description": ""} + queue_data["name"] = queue_name + queue_data["numjobs"] = database.get_num_incomplete_jobs_on_queue( + queue_name + ) + queue_info.append(queue_data) + + agent_info["queues"] = queue_info + # We want to include the start/stop dates so that default values # can be filled in for the date pickers agent_info["start"] = start_date @@ -186,13 +200,8 @@ def queues_data(): # Get job counts for each queue for queue in queue_data: - queue["numjobs"] = mongo.db.jobs.count_documents( - { - "job_data.job_queue": queue["name"], - "result_data.job_state": { - "$nin": ["complete", "completed", "cancelled"] - }, - } + queue["numjobs"] = database.get_num_incomplete_jobs_on_queue( + queue["name"] ) return queue_data diff --git a/server/tests/test_views.py b/server/tests/test_views.py index 3f89656ef..0969bafd9 100644 --- a/server/tests/test_views.py +++ b/server/tests/test_views.py @@ -56,16 +56,34 @@ def test_queues(): ) mongo.db.jobs.insert_many( [ - {"job_data": {"job_queue": "advertised_queue1"}}, - {"job_data": {"job_queue": "advertised_queue1"}}, - {"job_data": {"job_queue": "advertised_queue1"}}, - {"job_data": {"job_queue": "advertised_queue2"}}, - {"job_data": {"job_queue": "advertised_queue2"}}, + { + "job_data": {"job_queue": "advertised_queue1"}, + "result_data": {"job_state": "waiting"}, + }, + { + "job_data": {"job_queue": "advertised_queue1"}, + "result_data": {"job_state": "running"}, + }, + { + "job_data": {"job_queue": "advertised_queue1"}, + "result_data": {"job_state": "waiting"}, + }, + { + "job_data": {"job_queue": "advertised_queue2"}, + "result_data": {"job_state": "running"}, + }, + { + "job_data": {"job_queue": "advertised_queue2"}, + "result_data": {"job_state": "waiting"}, + }, ] ) # Get the data from the function we use to generate the view - with patch("testflinger.views.mongo", mongo): + with ( + patch("testflinger.views.mongo", mongo), + patch("testflinger.database.mongo", mongo), + ): data = queues_data() # Make sure we found all the queues, not just advertised ones @@ -149,6 +167,49 @@ def test_agent_detail_with_restricted_to(testapp): assert "(restricted to: test-client-id)" in html +def test_agent_detail_with_non_advertised_queue(testapp): + """Test agent detail with advertised and non-advertised queues.""" + mongo = mongomock.MongoClient() + # Insert one advertised queue + mongo.db.queues.insert_one( + {"name": "advertised_queue", "description": "advertised description"} + ) + # Agent listens to both advertised and non-advertised queues + mongo.db.agents.insert_one( + { + "name": "agent1", + "queues": ["advertised_queue", "non_advertised_queue"], + "updated_at": datetime.now(tz=timezone.utc), + } + ) + mongo.db.jobs.insert_many( + [ + { + "job_data": {"job_queue": "advertised_queue"}, + "result_data": {"job_state": "waiting"}, + }, + { + "job_data": {"job_queue": "non_advertised_queue"}, + "result_data": {"job_state": "running"}, + }, + ] + ) + + with ( + patch("testflinger.views.mongo", mongo), + patch("testflinger.database.mongo", mongo), + ): + with testapp.test_request_context(): + response = agent_detail("agent1") + + html = str(response) + # Should include both advertised and non-advertised queues + assert "advertised_queue" in html + assert "non_advertised_queue" in html + # Non-advertised queue creates dummy data with empty description + assert "advertised description" in html + + def test_job_not_found(testapp): """ Test that the job_detail fails gracefully when