From eda15aa2672a1c28448db719c2b3219219adaf8e Mon Sep 17 00:00:00 2001 From: Aidan Holland Date: Tue, 11 Mar 2025 03:07:49 -0400 Subject: [PATCH 1/3] feat: Add Censys Platform API support --- README.md | 28 ++-- censys/cli/commands/__init__.py | 13 +- censys/cli/commands/config.py | 19 ++- censys/cli/commands/platform.py | 128 ++++++++++++++++ censys/common/base.py | 12 +- censys/common/config.py | 9 +- censys/common/exceptions.py | 39 +++++ censys/platform/__init__.py | 20 +++ censys/platform/client.py | 57 +++++++ censys/platform/v3/__init__.py | 15 ++ censys/platform/v3/api.py | 139 +++++++++++++++++ censys/platform/v3/certificates.py | 92 ++++++++++++ censys/platform/v3/hosts.py | 141 ++++++++++++++++++ censys/platform/v3/search.py | 114 ++++++++++++++ censys/platform/v3/webproperties.py | 100 +++++++++++++ docs/index.rst | 1 + docs/usage-cli.rst | 21 ++- docs/usage-platform.rst | 190 ++++++++++++++++++++++++ examples/README.md | 23 +++ examples/platform/bulk_view_hosts.py | 57 +++++++ examples/platform/host_timeline.py | 55 +++++++ examples/platform/platform_client.py | 116 +++++++++++++++ examples/platform/search.py | 86 +++++++++++ examples/platform/unified_search.py | 155 +++++++++++++++++++ examples/platform/view_certificates.py | 52 +++++++ examples/platform/view_host.py | 56 +++++++ examples/platform/view_webproperties.py | 64 ++++++++ tests/cli/test_config.py | 2 +- tests/platform/__init__.py | 1 + tests/platform/test_client.py | 67 +++++++++ tests/platform/v3/__init__.py | 1 + tests/platform/v3/test_api.py | 118 +++++++++++++++ tests/platform/v3/test_certificates.py | 81 ++++++++++ tests/platform/v3/test_hosts.py | 122 +++++++++++++++ tests/platform/v3/test_search.py | 123 +++++++++++++++ tests/platform/v3/test_webproperties.py | 98 ++++++++++++ tests/search/__init__.py | 0 37 files changed, 2384 insertions(+), 31 deletions(-) create mode 100644 censys/cli/commands/platform.py create mode 100644 censys/platform/__init__.py create mode 100644 censys/platform/client.py create mode 100644 censys/platform/v3/__init__.py create mode 100644 censys/platform/v3/api.py create mode 100644 censys/platform/v3/certificates.py create mode 100644 censys/platform/v3/hosts.py create mode 100644 censys/platform/v3/search.py create mode 100644 censys/platform/v3/webproperties.py create mode 100644 docs/usage-platform.rst create mode 100644 examples/platform/bulk_view_hosts.py create mode 100644 examples/platform/host_timeline.py create mode 100644 examples/platform/platform_client.py create mode 100644 examples/platform/search.py create mode 100644 examples/platform/unified_search.py create mode 100644 examples/platform/view_certificates.py create mode 100644 examples/platform/view_host.py create mode 100644 examples/platform/view_webproperties.py create mode 100644 tests/platform/__init__.py create mode 100644 tests/platform/test_client.py create mode 100644 tests/platform/v3/__init__.py create mode 100644 tests/platform/v3/test_api.py create mode 100644 tests/platform/v3/test_certificates.py create mode 100644 tests/platform/v3/test_hosts.py create mode 100644 tests/platform/v3/test_search.py create mode 100644 tests/platform/v3/test_webproperties.py create mode 100644 tests/search/__init__.py diff --git a/README.md b/README.md index 19573077..e11c2665 100644 --- a/README.md +++ b/README.md @@ -11,16 +11,13 @@ An easy-to-use and lightweight API wrapper for Censys APIs ([censys.io](https:// > **Notice:** The Censys Search v1 endpoints are deprecated as of Nov. 30, 2021. Please begin using v2 endpoints to query hosts and certificates and check out our [support center](https://support.censys.io/hc/en-us/sections/360013076551-Censys-Search-2-0) for resources. -> [!IMPORTANT] -> This library does not support the new Censys Platform, however a new set of SDKs that do support the platform are coming soon. -> Please refer to the [platform API refrence docs](https://docs.censys.com/reference/get-started#/) in the mean time. - ## Features -- [Search Censys data](https://censys-python.readthedocs.io/en/stable/usage-v2.html) -- [Bulk Certificate lookups](https://censys-python.readthedocs.io/en/stable/usage-v2.html#bulk-view) -- [Download Bulk Data](https://censys-python.readthedocs.io/en/stable/usage-v1.html#data) -- [Manage assets, events, and seeds in Censys ASM](https://censys-python.readthedocs.io/en/stable/usage-asm.html) +- [Platform API](https://censys-python.readthedocs.io/en/stable/usage-platform.html) +- [Legacy Search API](https://censys-python.readthedocs.io/en/stable/usage-v2.html) +- [Helpful utilities like Bulk Certificate lookups](https://censys-python.readthedocs.io/en/stable/usage-v2.html#bulk-view) +- [Data Download API](https://censys-python.readthedocs.io/en/stable/usage-v1.html#data) +- [Censys ASM API](https://censys-python.readthedocs.io/en/stable/usage-asm.html) - [Command-line interface](https://censys-python.readthedocs.io/en/stable/cli.html) @@ -59,7 +56,17 @@ Optionally, you can enable tab completion for the CLI by adding this line to you eval "$(register-python-argcomplete censys)" ``` -To configure your search credentials run `censys config` or set both `CENSYS_API_ID` and `CENSYS_API_SECRET` environment variables. +To configure the platform API, run `censys platform config` or set both `CENSYS_PLATFORM_TOKEN` and `CENSYS_ORGANIZATION_ID` environment variables. + +```sh +$ censys platform config + +Censys Platform Token: XXX +Censys Organization ID: XXX +Do you want color output? [y/n]: y +``` + +If you wish to use the legacy Censys Search API, you can configure your search credentials run `censys config` or set both `CENSYS_API_ID` and `CENSYS_API_SECRET` environment variables. ```sh $ censys config @@ -99,6 +106,9 @@ The examples located in the [`examples/`](examples/) directory are a great place - [Discussions](https://github.com/censys/censys-python/discussions) - [Censys Homepage](https://censys.io/) - [Censys Search](https://search.censys.io/) + - [Censys Search API](https://search.censys.io/api) +- [Censys Platform](https://platform.censys.io/) + - [Censys Platform API](https://docs.censys.io/reference/get-started#/) ## Contributing diff --git a/censys/cli/commands/__init__.py b/censys/cli/commands/__init__.py index 2fa8c4c2..22fcc10d 100644 --- a/censys/cli/commands/__init__.py +++ b/censys/cli/commands/__init__.py @@ -1,5 +1,14 @@ """Censys CLI commands.""" -from . import account, asm, config, hnri, search, subdomains, view +from . import account, asm, config, hnri, platform, search, subdomains, view -__all__ = ["account", "asm", "config", "hnri", "search", "subdomains", "view"] +__all__ = [ + "account", + "asm", + "config", + "hnri", + "platform", + "search", + "subdomains", + "view", +] diff --git a/censys/cli/commands/config.py b/censys/cli/commands/config.py index 8f299221..d102c2be 100644 --- a/censys/cli/commands/config.py +++ b/censys/cli/commands/config.py @@ -42,35 +42,42 @@ def cli_config(_: argparse.Namespace): # pragma: no cover api_id_prompt = f"{api_id_prompt} [cyan]({redacted_id})[/cyan]" api_secret_prompt = f"{api_secret_prompt} [cyan]({redacted_secret})[/cyan]" + console.print("[bold]Search API Credentials[/bold]") api_id = Prompt.ask(api_id_prompt, console=console) or api_id api_secret = Prompt.ask(api_secret_prompt, console=console) or api_secret if not (api_id and api_secret): - console.print("Please enter valid credentials") + console.print("Please enter valid Search API credentials") sys.exit(1) api_id = api_id.strip() api_secret = api_secret.strip() + console.print( + "\n[bold]Note:[/bold] For Platform API configuration, please use [cyan]censys platform config[/cyan] command." + ) + color = Confirm.ask( - "Do you want color output?", default=True, show_default=False, console=console + "\nDo you want color output?", default=True, show_default=False, console=console ) config.set(DEFAULT, "color", "auto" if color else "") try: + # Test Search API credentials client = CensysSearchAPIv2(api_id, api_secret) account = client.account() email = account.get("email") - console.print(f"\nSuccessfully authenticated for {email}") + console.print(f"\nSuccessfully authenticated for Search API: {email}") - # Assumes that login was successfully + # Update config config.set(DEFAULT, "api_id", api_id) config.set(DEFAULT, "api_secret", api_secret) write_config(config) + console.print("Configuration saved successfully") sys.exit(0) except CensysUnauthorizedException: - console.print("Failed to authenticate") + console.print("Failed to authenticate with Search API") sys.exit(1) except PermissionError as e: console.print(e) @@ -91,6 +98,6 @@ def include(parent_parser: argparse._SubParsersAction, parents: dict): config_parser = parent_parser.add_parser( "config", description="Configure Censys Search API Settings", - help="configure Censys search API settings", + help="configure Censys Search API settings", ) config_parser.set_defaults(func=cli_config) diff --git a/censys/cli/commands/platform.py b/censys/cli/commands/platform.py new file mode 100644 index 00000000..8901883b --- /dev/null +++ b/censys/cli/commands/platform.py @@ -0,0 +1,128 @@ +"""Censys Platform CLI commands.""" + +import argparse +import os +import sys + +from rich.prompt import Confirm, Prompt + +from censys.cli.utils import console +from censys.common.config import DEFAULT, get_config, write_config + + +def cli_platform_config(_: argparse.Namespace): # pragma: no cover + """Platform config subcommand. + + Args: + _: Argparse Namespace. + """ + platform_token_prompt = "Censys Platform Token" + platform_org_id_prompt = "Censys Organization ID" + + config = get_config() + platform_token = config.get(DEFAULT, "platform_token", fallback="") + platform_org_id = config.get(DEFAULT, "platform_org_id", fallback="") + + platform_token_env = os.getenv("CENSYS_PLATFORM_TOKEN") + platform_org_id_env = os.getenv("CENSYS_ORGANIZATION_ID") + + if platform_token_env is not None: + console.print( + "Please note environment variable CENSYS_PLATFORM_TOKEN " + "will take priority over configured Platform API token." + ) + platform_token = platform_token_env or platform_token + + if platform_org_id_env is not None: + console.print( + "Please note environment variable CENSYS_ORGANIZATION_ID " + "will take priority over configured Platform Organization ID." + ) + platform_org_id = platform_org_id_env or platform_org_id + + if platform_token: + # Only show last 6 characters of the token + visible_chars = 6 + if len(platform_token) > visible_chars: + redacted_token = ( + "*" * (len(platform_token) - visible_chars) + + platform_token[-visible_chars:] + ) + platform_token_prompt = ( + f"{platform_token_prompt} [cyan]({redacted_token})[/cyan]" + ) + + if platform_org_id: + redacted_org_id = ( + "*" * (len(platform_org_id) - 4) + platform_org_id[-4:] + if len(platform_org_id) > 4 + else platform_org_id + ) + platform_org_id_prompt = ( + f"{platform_org_id_prompt} [cyan]({redacted_org_id})[/cyan]" + ) + + console.print("[bold]Platform API Credentials[/bold]") + platform_token = ( + Prompt.ask(platform_token_prompt, console=console) or platform_token + ) + platform_token = platform_token.strip() if platform_token else "" + + if not platform_token: + console.print("Please enter a valid Platform API token") + sys.exit(1) + + console.print("\n[bold]Platform Organization ID[/bold]") + console.print("Required for Platform API access") + platform_org_id = ( + Prompt.ask(platform_org_id_prompt, console=console) or platform_org_id + ) + platform_org_id = platform_org_id.strip() if platform_org_id else "" + + if not platform_org_id: + console.print( + "[yellow]Warning: Organization ID is required for Platform API access[/yellow]" + ) + + color = Confirm.ask( + "\nDo you want color output?", default=True, show_default=False, console=console + ) + config.set(DEFAULT, "color", "auto" if color else "") + + try: + # Save Platform API token and Organization ID + config.set(DEFAULT, "platform_token", platform_token) + config.set(DEFAULT, "platform_org_id", platform_org_id) + write_config(config) + console.print("Platform API configuration saved successfully") + sys.exit(0) + except PermissionError as e: + console.print(e) + console.print( + "Cannot write config file to directory. " + + "Please set the `CENSYS_CONFIG_PATH` environmental variable to a writeable location." + ) + sys.exit(1) + + +def include(parent_parser: argparse._SubParsersAction, parents: dict): + """Include this subcommand into the parent parser. + + Args: + parent_parser (argparse._SubParsersAction): Parent parser. + parents (dict): Parent arg parsers. + """ + platform_parser = parent_parser.add_parser( + "platform", + description="Censys Platform API Commands", + help="platform API commands", + ) + platform_subparsers = platform_parser.add_subparsers() + + # Platform config command + config_parser = platform_subparsers.add_parser( + "config", + description="Configure Censys Platform API Settings", + help="configure Censys Platform API settings", + ) + config_parser.set_defaults(func=cli_platform_config) diff --git a/censys/common/base.py b/censys/common/base.py index 5abfa64c..60c7d0f6 100644 --- a/censys/common/base.py +++ b/censys/common/base.py @@ -182,7 +182,7 @@ def _call_method( @_backoff_wrapper def _make_call( self, - method: Callable[..., Response], + method: Callable, endpoint: str, args: Optional[dict] = None, data: Optional[Any] = None, @@ -190,13 +190,10 @@ def _make_call( ) -> dict: """Make API call. - Wrapper functions for all our REST API calls checking for errors - and decoding the responses. - Args: - method (Callable): Method to send HTTP request. - endpoint (str): The path of API endpoint. - args (dict): Optional; URL args that are mapped to params. + method (Callable): Method to call. + endpoint (str): Endpoint to call. + args (dict, optional): Arguments to pass to method. data (Any): Optional; JSON data to serialize with request. **kwargs: Arbitrary keyword arguments to pass to method. @@ -244,6 +241,7 @@ def _make_call( "statusCode", "unknown" ) details = json_data.get("details", "unknown") + except (ValueError, json.decoder.JSONDecodeError) as error: raise CensysJSONDecodeException( status_code=res.status_code, diff --git a/censys/common/config.py b/censys/common/config.py index bb53776e..f32d69bf 100644 --- a/censys/common/config.py +++ b/censys/common/config.py @@ -2,6 +2,7 @@ import configparser import os +from contextlib import suppress from pathlib import Path DEFAULT = "DEFAULT" @@ -51,13 +52,15 @@ def write_config(config: configparser.ConfigParser) -> None: def get_config() -> configparser.ConfigParser: - """Reads and returns config. + """Gets and parses configuration. Returns: - configparser.ConfigParser: Config for Censys. + configparser.ConfigParser: Config parser. """ config = configparser.ConfigParser(defaults=default_config, default_section=DEFAULT) + # Attempt to read and parse configuration config_path = get_config_path() if os.path.isfile(config_path): - config.read(config_path) + with suppress(configparser.Error), open(config_path) as config_file: + config.read_file(config_file) return config diff --git a/censys/common/exceptions.py b/censys/common/exceptions.py index d9082cd3..6cabb3bf 100644 --- a/censys/common/exceptions.py +++ b/censys/common/exceptions.py @@ -56,6 +56,20 @@ def __repr__(self) -> str: __str__ = __repr__ +class CensysPlatformException(CensysAPIException): + """Base Exception for the Censys Platform API.""" + + def __repr__(self) -> str: + """Representation of CensysPlatformException. + + Returns: + str: Printable representation. + """ + return f"{self.status_code} ({self.const}): {self.message or self.body}" + + __str__ = __repr__ + + class CensysAsmException(CensysAPIException): """Base Exception for the Censys ASM API.""" @@ -363,3 +377,28 @@ class CensysExceptionMapper: 500: CensysInternalServerException, } """Map of status code to Search Exception.""" + + PLATFORM_EXCEPTIONS: Dict[int, Type[CensysPlatformException]] = { + 401: CensysUnauthorizedException, + 403: CensysUnauthorizedException, + 404: CensysNotFoundException, + 429: CensysRateLimitExceededException, + 500: CensysInternalServerException, + } + """Map of status code to Platform Exception.""" + + @staticmethod + def exception_for_status_code(status_code: int) -> Type[CensysAPIException]: + """Return the appropriate exception class for the given status code. + + Args: + status_code (int): HTTP status code. + + Returns: + Type[CensysAPIException]: The exception class to raise. + """ + if status_code in CensysExceptionMapper.PLATFORM_EXCEPTIONS: + return CensysExceptionMapper.PLATFORM_EXCEPTIONS[status_code] + if status_code in CensysExceptionMapper.SEARCH_EXCEPTIONS: + return CensysExceptionMapper.SEARCH_EXCEPTIONS[status_code] + return CensysAPIException diff --git a/censys/platform/__init__.py b/censys/platform/__init__.py new file mode 100644 index 00000000..30d97efa --- /dev/null +++ b/censys/platform/__init__.py @@ -0,0 +1,20 @@ +"""An easy-to-use and lightweight API wrapper for Censys Platform API.""" + +from .client import CensysPlatformClient +from .v3 import ( + CensysCertificates, + CensysHosts, + CensysPlatformAPIv3, + CensysSearch, + CensysWebProperties, +) + +__copyright__ = "Copyright 2024 Censys, Inc." +__all__ = [ + "CensysPlatformClient", + "CensysPlatformAPIv3", + "CensysCertificates", + "CensysHosts", + "CensysSearch", + "CensysWebProperties", +] diff --git a/censys/platform/client.py b/censys/platform/client.py new file mode 100644 index 00000000..7066414f --- /dev/null +++ b/censys/platform/client.py @@ -0,0 +1,57 @@ +"""Interact with all Censys Platform APIs through a unified client.""" + +from typing import Optional + +from .v3 import CensysCertificates, CensysHosts, CensysSearch, CensysWebProperties + + +class CensysPlatformClient: + """Client for interacting with all Censys Platform APIs. + + Examples: + Initialize the Platform client: + + >>> from censys.platform import CensysPlatformClient + >>> client = CensysPlatformClient() + + Access all Platform API resources through a single client: + + >>> hosts = client.hosts # CensysHosts instance + >>> certs = client.certificates # CensysCertificates instance + >>> web = client.webproperties # CensysWebProperties instance + >>> search = client.search # CensysSearch instance + + Perform operations with the client: + + >>> host_results = client.hosts.search("host.services: (port = 22 and protocol = 'SSH')") + >>> cert_results = client.certificates.search("certificate.issuer.common_name: *Google*") + """ + + hosts: CensysHosts + certificates: CensysCertificates + webproperties: CensysWebProperties + search: CensysSearch + + def __init__( + self, + token: Optional[str] = None, + organization_id: Optional[str] = None, + **kwargs, + ): + """Initialize the CensysPlatformClient. + + Args: + token (str, optional): Personal Access Token for the Censys Platform API. + organization_id (str, optional): Organization ID for the Censys Platform API. + **kwargs: Optional keyword arguments to pass to the API clients. + """ + self.hosts = CensysHosts(token=token, organization_id=organization_id, **kwargs) + self.certificates = CensysCertificates( + token=token, organization_id=organization_id, **kwargs + ) + self.webproperties = CensysWebProperties( + token=token, organization_id=organization_id, **kwargs + ) + self.search = CensysSearch( + token=token, organization_id=organization_id, **kwargs + ) diff --git a/censys/platform/v3/__init__.py b/censys/platform/v3/__init__.py new file mode 100644 index 00000000..b661790b --- /dev/null +++ b/censys/platform/v3/__init__.py @@ -0,0 +1,15 @@ +"""Interact with the Censys Platform v3 APIs.""" + +from .api import CensysPlatformAPIv3 +from .certificates import CensysCertificates +from .hosts import CensysHosts +from .search import CensysSearch +from .webproperties import CensysWebProperties + +__all__ = [ + "CensysPlatformAPIv3", + "CensysCertificates", + "CensysHosts", + "CensysSearch", + "CensysWebProperties", +] diff --git a/censys/platform/v3/api.py b/censys/platform/v3/api.py new file mode 100644 index 00000000..35964919 --- /dev/null +++ b/censys/platform/v3/api.py @@ -0,0 +1,139 @@ +"""Base for interacting with the Censys Platform API.""" + +from typing import Any, Optional, Type + +from requests.models import Response + +from censys.common.base import CensysAPIBase +from censys.common.config import DEFAULT, get_config +from censys.common.exceptions import CensysExceptionMapper, CensysPlatformException + + +class CensysPlatformAPIv3(CensysAPIBase): + """This class is the base class for the Platform API. + + Examples: + >>> c = CensysPlatformAPIv3(token="your_platform_pat") + """ + + DEFAULT_URL: str = "https://api.platform.censys.io" + """Default Platform API base URL.""" + INDEX_NAME: str = "" + ASSET_NAME: str = "" + API_VERSION: str = "v3" + ASSET_VERSION: str = "v1" + + def __init__( + self, + token: Optional[str] = None, + organization_id: Optional[str] = None, + **kwargs, + ): + """Inits CensysPlatformAPIv3. + + Args: + token (str, optional): Personal Access Token for Censys Platform API. + organization_id (str, optional): Organization ID for Censys Platform API. + **kwargs: Optional kwargs. + + Raises: + ValueError: If no Personal Access Token is provided. + """ + # For backward compatibility, but these aren't used + api_id = kwargs.pop("api_id", None) + api_secret = kwargs.pop("api_secret", None) + + if api_id or api_secret: + import warnings + + warnings.warn( + "api_id and api_secret are deprecated for Platform API. " + "Please use token (PAT) instead.", + DeprecationWarning, + stacklevel=2, + ) + + url = kwargs.pop("url", self.DEFAULT_URL) + CensysAPIBase.__init__(self, url=url, **kwargs) + + # Gets config file + config = get_config() + + self.base_url = url + self.organization_id = organization_id + + # Try to get Personal Access Token if not provided + self._token = ( + token + or kwargs.get("token") + or config.get(DEFAULT, "platform_token", fallback=None) + ) + self.organization_id = organization_id or config.get( + DEFAULT, "platform_org_id", fallback=None + ) + + # Determine the Accept header based on asset type + accept_header = "application/json" + if self.ASSET_NAME: + accept_header = f"application/vnd.censys.api.{self.API_VERSION}.{self.ASSET_NAME}.{self.ASSET_VERSION}+json" + + # Set up authentication headers using token + if not self._token: + raise ValueError("Personal Access Token is required for Platform API.") + + if not self.organization_id: + raise ValueError("Organization ID is required for Platform API.") + + self._session.headers.update( + { + "Authorization": f"Bearer {self._token}", + "X-Organization-ID": self.organization_id, + "Accept": accept_header, + "Content-Type": "application/json", + } + ) + + def _get_exception_class( # type: ignore + self, res: Response + ) -> Type[CensysPlatformException]: + """Get the exception class for the response status code. + + Args: + res (Response): Response object. + + Returns: + Type[CensysPlatformException]: The exception class to raise. + """ + return CensysExceptionMapper.exception_for_status_code(res.status_code) # type: ignore + + def _get(self, endpoint: str, args: Optional[dict] = None, **kwargs: Any) -> dict: + """Get data from a REST API endpoint. + + Args: + endpoint (str): The endpoint to send the request to. + args (dict, optional): The parameters to be sent in the request. + **kwargs: Optional keyword args. + + Returns: + dict: The returned data from the endpoint. + """ + # Add organization_id to args if it exists and args is provided + if self.organization_id and args is not None: + args = {**args, "organization_id": self.organization_id} + elif self.organization_id: + args = {"organization_id": self.organization_id} + return super()._get(endpoint, args, **kwargs) + + def view(self, resource_id: str, **kwargs: Any) -> dict: + """View details of a specific resource. + + Args: + resource_id (str): The resource ID to fetch. + **kwargs: Optional keyword args. + + Returns: + dict: The resource details. + """ + # Organization ID is already in the headers, no need to add to query params + params = kwargs.pop("params", {}) or {} + return self._get(f"{self.INDEX_NAME}/{resource_id}", params=params, **kwargs) diff --git a/censys/platform/v3/certificates.py b/censys/platform/v3/certificates.py new file mode 100644 index 00000000..fc0a26d0 --- /dev/null +++ b/censys/platform/v3/certificates.py @@ -0,0 +1,92 @@ +"""Interact with the Censys Platform Certificate API.""" + +from typing import Any, Dict, List, Optional + +from .api import CensysPlatformAPIv3 + + +class CensysCertificates(CensysPlatformAPIv3): + """Interacts with the Censys Platform Certificate API. + + Examples: + Inits Censys Platform Certificates. + + >>> from censys.platform import CensysCertificates + >>> c = CensysCertificates() + + Get certificate details. + + >>> c.view("a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1") + { + "certificate": { + "fingerprint_sha256": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", + ... + }, + ... + } + + Get multiple certificates. + + >>> c.bulk_view(["a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2"]) + { + "result": [ + { + "certificate": { + "fingerprint_sha256": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", + ... + } + }, + { + "certificate": { + "fingerprint_sha256": "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2", + ... + } + } + ] + } + """ + + INDEX_NAME = "v3/global/asset/certificate" + ASSET_NAME = "certificate" + + def __init__( + self, + token: Optional[str] = None, + organization_id: Optional[str] = None, + **kwargs, + ): + """Inits CensysCertificates. + + Args: + token (str, optional): Personal Access Token for Censys Platform API. + organization_id (str, optional): Organization ID to use for API requests. + **kwargs: Optional kwargs. + """ + CensysPlatformAPIv3.__init__( + self, token=token, organization_id=organization_id, **kwargs + ) + + def view(self, certificate_id: str, **kwargs: Any) -> Dict[str, Any]: + """Get a certificate by ID. + + Args: + certificate_id (str): The certificate ID to fetch. + **kwargs: Optional keyword args. + + Returns: + Dict[str, Any]: The certificate details. + """ + return self._get(f"{self.INDEX_NAME}/{certificate_id}", **kwargs) + + def bulk_view(self, certificate_ids: List[str], **kwargs: Any) -> Dict[str, Any]: + """Get multiple certificates by ID. + + Args: + certificate_ids (List[str]): The certificate IDs to fetch. + **kwargs: Optional keyword args. + + Returns: + Dict[str, Any]: The certificates details. + """ + params = {"certificate_ids": certificate_ids} + return self._get(self.INDEX_NAME, params=params, **kwargs) diff --git a/censys/platform/v3/hosts.py b/censys/platform/v3/hosts.py new file mode 100644 index 00000000..ff5a7406 --- /dev/null +++ b/censys/platform/v3/hosts.py @@ -0,0 +1,141 @@ +"""Interact with the Censys Platform Host API.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from .api import CensysPlatformAPIv3 + + +class CensysHosts(CensysPlatformAPIv3): + """Interacts with the Censys Platform Host API. + + Examples: + Inits Censys Platform Hosts. + + >>> from censys.platform import CensysHosts + >>> h = CensysHosts() + + Get host details. + + >>> h.view("1.1.1.1") + { + "host": { + "ip": "1.1.1.1", + ... + }, + ... + } + + Get multiple hosts. + + >>> h.bulk_view(["1.1.1.1", "8.8.8.8"]) + { + "result": [ + { + "host": { + "ip": "1.1.1.1", + ... + } + }, + { + "host": { + "ip": "8.8.8.8", + ... + } + } + ] + } + + Get host events timeline. + + >>> h.timeline("1.1.1.1") + { + "events": [ + { + "timestamp": "2023-01-01T00:00:00.000Z", + ... + }, + ... + ] + } + """ + + INDEX_NAME = "v3/global/asset/host" + ASSET_NAME = "host" + + def __init__( + self, + token: Optional[str] = None, + organization_id: Optional[str] = None, + **kwargs, + ): + """Inits CensysHosts. + + Args: + token (str, optional): Personal Access Token for Censys Platform API. + organization_id (str, optional): Organization ID for Censys Platform API. + **kwargs: Optional kwargs. + """ + CensysPlatformAPIv3.__init__( + self, token=token, organization_id=organization_id, **kwargs + ) + + def view( + self, host_id: str, at_time: Optional[datetime] = None, **kwargs: Any + ) -> Dict[str, Any]: + """Get a host by ID. + + Args: + host_id (str): The host ID to fetch. + at_time (datetime, optional): Point in time to view the host. + **kwargs: Optional keyword args. + + Returns: + Dict[str, Any]: The host details. + """ + params = {} + if at_time: + params["at_time"] = at_time.isoformat() + "Z" + + return self._get(f"{self.INDEX_NAME}/{host_id}", params=params, **kwargs) + + def bulk_view(self, host_ids: List[str], **kwargs: Any) -> Dict[str, Any]: + """Get multiple hosts by ID. + + Args: + host_ids (List[str]): The host IDs to fetch. + **kwargs: Optional keyword args. + + Returns: + Dict[str, Any]: The hosts details. + """ + params = {"host_ids": host_ids} + return self._get(self.INDEX_NAME, params=params, **kwargs) + + def timeline( + self, + host_id: str, + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None, + **kwargs: Any, + ) -> Dict[str, Any]: + """Get the timeline of events for a host. + + Args: + host_id (str): The host ID to fetch events for. + start_time (datetime, optional): Start time for the timeline. + end_time (datetime, optional): End time for the timeline. + **kwargs: Optional keyword args. + + Returns: + Dict[str, Any]: The host events. + """ + params = {} + if start_time: + params["start_time"] = start_time.isoformat() + "Z" + if end_time: + params["end_time"] = end_time.isoformat() + "Z" + + return self._get( + f"{self.INDEX_NAME}/{host_id}/timeline", params=params, **kwargs + ) diff --git a/censys/platform/v3/search.py b/censys/platform/v3/search.py new file mode 100644 index 00000000..53d72043 --- /dev/null +++ b/censys/platform/v3/search.py @@ -0,0 +1,114 @@ +"""Interact with the Censys Platform Search API.""" + +from typing import Any, Dict, List, Optional, Union + +from .api import CensysPlatformAPIv3 + + +class CensysSearch(CensysPlatformAPIv3): + """Interacts with the Censys Platform Search API. + + Examples: + Inits Censys Platform Search. + + >>> from censys.platform import CensysSearch + >>> s = CensysSearch() + + Search for hosts. + + >>> s.query("services.port:443") + { + "result": { + "hits": [ + { + "host": { + "ip": "1.1.1.1", + ... + } + }, + ... + ], + "total": 123456 + } + } + + Aggregate host data. + + >>> s.aggregate("services.port:443", "services.service_name") + { + "result": { + "buckets": [ + { + "key": "HTTP", + "count": 12345 + }, + ... + ] + } + } + """ + + INDEX_NAME = "v3/global/search" + + def __init__(self, token: Optional[str] = None, **kwargs): + """Inits CensysSearch. + + Args: + token (str, optional): Personal Access Token for Censys Platform API. + **kwargs: Optional kwargs. + """ + CensysPlatformAPIv3.__init__(self, token=token, **kwargs) + + def query( + self, + query: str, + per_page: int = 100, + cursor: Optional[str] = None, + fields: Optional[List[str]] = None, + sort: Optional[Union[str, List[str]]] = None, + **kwargs: Any, + ) -> Dict[str, Any]: + """Search the global data. + + Args: + query (str): The query to search for. + per_page (int): Number of results per page. Defaults to 100. + cursor (str, optional): Cursor for pagination. + fields (List[str], optional): Fields to return. + sort (Union[str, List[str]], optional): Field(s) to sort on. + **kwargs: Optional keyword args. + + Returns: + Dict[str, Any]: The search result. + """ + data = {"q": query, "per_page": per_page} + if cursor: + data["cursor"] = cursor + if fields: + data["fields"] = fields + if sort: + data["sort"] = sort + + return self._post(f"{self.INDEX_NAME}/query", data=data, **kwargs) + + def aggregate( + self, + query: str, + field: str, + num_buckets: int = 50, + **kwargs: Any, + ) -> Dict[str, Any]: + """Aggregate the global data. + + Args: + query (str): The query to search for. + field (str): The field to aggregate on. + num_buckets (int): Number of buckets to return. Defaults to 50. + **kwargs: Optional keyword args. + + Returns: + Dict[str, Any]: The aggregation result. + """ + data = {"q": query, "field": field, "num_buckets": num_buckets} + + return self._post(f"{self.INDEX_NAME}/aggregate", data=data, **kwargs) diff --git a/censys/platform/v3/webproperties.py b/censys/platform/v3/webproperties.py new file mode 100644 index 00000000..dcd89c6c --- /dev/null +++ b/censys/platform/v3/webproperties.py @@ -0,0 +1,100 @@ +"""Interact with the Censys Platform WebProperty API.""" + +from datetime import datetime +from typing import Any, Dict, List, Optional + +from .api import CensysPlatformAPIv3 + + +class CensysWebProperties(CensysPlatformAPIv3): + """Interacts with the Censys Platform WebProperty API. + + Examples: + Inits Censys Platform WebProperties. + + >>> from censys.platform import CensysWebProperties + >>> w = CensysWebProperties() + + Get webproperty details. + + >>> w.view("example.com:443") + { + "webproperty": { + "name": "example.com:443", + ... + }, + ... + } + + Get multiple webproperties. + + >>> w.bulk_view(["example.com:443", "example.org:443"]) + { + "result": [ + { + "webproperty": { + "name": "example.com:443", + ... + } + }, + { + "webproperty": { + "name": "example.org:443", + ... + } + } + ] + } + """ + + INDEX_NAME = "v3/global/asset/webproperty" + ASSET_NAME = "webproperty" + + def __init__( + self, + token: Optional[str] = None, + organization_id: Optional[str] = None, + **kwargs, + ): + """Inits CensysWebProperties. + + Args: + token (str, optional): Personal Access Token for Censys Platform API. + organization_id (str, optional): Organization ID to use for API requests. + **kwargs: Optional kwargs. + """ + CensysPlatformAPIv3.__init__( + self, token=token, organization_id=organization_id, **kwargs + ) + + def view( + self, webproperty_id: str, at_time: Optional[datetime] = None, **kwargs: Any + ) -> Dict[str, Any]: + """Get a webproperty by ID. + + Args: + webproperty_id (str): The webproperty ID to fetch. + at_time (datetime, optional): Point in time to view the webproperty. + **kwargs: Optional keyword args. + + Returns: + Dict[str, Any]: The webproperty details. + """ + params = {} + if at_time: + params["at_time"] = at_time.isoformat() + "Z" + + return self._get(f"{self.INDEX_NAME}/{webproperty_id}", params=params, **kwargs) + + def bulk_view(self, webproperty_ids: List[str], **kwargs: Any) -> Dict[str, Any]: + """Get multiple webproperties by ID. + + Args: + webproperty_ids (List[str]): The webproperty IDs to fetch. + **kwargs: Optional keyword args. + + Returns: + Dict[str, Any]: The webproperties details. + """ + params = {"webproperty_ids": webproperty_ids} + return self._get(self.INDEX_NAME, params=params, **kwargs) diff --git a/docs/index.rst b/docs/index.rst index 4f645ff7..a15a3a6e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,6 +29,7 @@ The User Guide quick-start usage-cli + usage-platform usage-v2 usage-v1 usage-asm diff --git a/docs/usage-cli.rst b/docs/usage-cli.rst index b46efbf1..8c165b24 100644 --- a/docs/usage-cli.rst +++ b/docs/usage-cli.rst @@ -4,11 +4,26 @@ CLI Usage .. raw:: html :file: cli.html -Before continuing please ensure you have successfully configured your credentials. +Configuration +------------ -.. prompt:: bash +Censys Python supports two different API endpoints, each with its own dedicated configuration command: + +1. **Legacy Search API Configuration** + + The legacy Search API uses API ID and API Secret for authentication: + + .. prompt:: bash + + censys config + +2. **Platform API Configuration** + + The Platform API uses Platform Token and Organization ID for authentication: + + .. prompt:: bash - censys config + censys platform config The configuration file by default is written to ``~/.config/censys/censys.cfg``, but you can change this by setting the ``CENSYS_CONFIG_PATH`` environment variable. diff --git a/docs/usage-platform.rst b/docs/usage-platform.rst new file mode 100644 index 00000000..c0793f2d --- /dev/null +++ b/docs/usage-platform.rst @@ -0,0 +1,190 @@ +Usage Platform +============== + +The Censys Platform API provides functionality for interacting with all Censys Platform resources through a unified client interface. + +The ``CensysPlatformClient`` class +---------------------------------- + +The Censys Platform API provides a unified client to access all Censys Platform resources through a single interface. + +.. code-block:: python + + from censys.platform import CensysPlatformClient + + # Initialize the client with your credentials + client = CensysPlatformClient( + token="YOUR_API_TOKEN", + organization_id="YOUR_ORGANIZATION_ID" + ) + + # Access different APIs through the client + hosts = client.hosts # CensysHosts instance + certificates = client.certificates # CensysCertificates instance + webproperties = client.webproperties # CensysWebProperties instance + search = client.search # CensysSearch instance + +Configuration +------------ + +The Platform API requires specific configuration that is separate from the legacy Censys API configuration. You'll need both a Platform Token and an Organization ID to use the Platform API. + +You can configure these credentials in one of the following ways: + +1. Using the dedicated platform configuration command: + + .. code-block:: sh + + $ censys platform config + + Censys Platform Token: XXX + Censys Organization ID: XXX + Do you want color output? [y/n]: y + + Note that this is different from the legacy ``censys config`` command, which configures only the Search API credentials. + +2. Setting environment variables: + + .. code-block:: sh + + # Set these environment variables + export CENSYS_PLATFORM_TOKEN="your_platform_token" + export CENSYS_ORGANIZATION_ID="your_organization_id" + +3. Passing credentials directly to the client constructor: + + .. code-block:: python + + client = CensysPlatformClient( + token="your_platform_token", + organization_id="your_organization_id" + ) + +Note that the Platform API configuration is different from the legacy Censys API configuration, which uses API ID and API Secret instead of a Platform Token. + +Search API (Unified Search) +-------------------------- + +The Platform API provides a unified search interface through the `CensysSearch` client. This allows you to search across hosts, certificates, and web properties using the same Censys Query Language (CenQL) syntax. + +.. code-block:: python + + # Search for hosts with specific services + search_query = "host.services: (port = 443 and protocol = 'TCP')" + search_results = client.search.query(search_query, per_page=10) + + # Check results + total_results = search_results.get("result", {}).get("total", 0) + hits = search_results.get("result", {}).get("hits", []) + + # Process results + for hit in hits: + print(f"IP: {hit.get('ip')}") + +Aggregate data across resources: + +.. code-block:: python + + # Aggregate data about services on port 443 + agg_query = "host.services.port: 443" + agg_field = "host.services.service_name" + agg_results = client.search.aggregate(agg_query, agg_field) + + # Process aggregation buckets + buckets = agg_results.get("result", {}).get("buckets", []) + for bucket in buckets[:5]: + print(f"{bucket.get('key')}: {bucket.get('count')} hosts") + +Hosts API +--------- + +Access host-specific operations through the ``hosts`` client: + +.. code-block:: python + + # View details of a specific host + host_ip = "8.8.8.8" + host_details = client.hosts.view(host_ip) + + # Get services running on the host + services = host_details.get("host", {}).get("services", []) + for service in services: + print(f"Port {service.get('port')}: {service.get('service_name')}") + + # Get multiple hosts at once + host_ips = ["8.8.8.8", "1.1.1.1"] + bulk_results = client.hosts.bulk_view(host_ips) + + # Get host timeline + timeline = client.hosts.timeline("8.8.8.8") + +Certificates API +--------------- + +Access certificate-specific operations through the ``certificates`` client: + +.. code-block:: python + + # View details of a specific certificate + cert_id = "fb444eb8e68437bae06232b9f5091bccff62a768ca09e92eb5c9c2cf9d17c426" + cert_details = client.certificates.view(cert_id) + + # Get certificate fields + subject = cert_details.get("certificate", {}).get("subject", {}) + issuer = cert_details.get("certificate", {}).get("issuer", {}) + + print(f"Subject: {subject.get('common_name')}") + print(f"Issuer: {issuer.get('common_name')}") + +Web Properties API +----------------- + +Access web property-specific operations through the ``webproperties`` client: + +.. code-block:: python + + # View details of a specific web property + webprop_id = "example.com:443" + webprop_details = client.webproperties.view(webprop_id) + + # Get web property information + name = webprop_details.get("webproperty", {}).get("name") + protocol = webprop_details.get("webproperty", {}).get("protocol") + + print(f"Name: {name}") + print(f"Protocol: {protocol}") + +Censys Query Language (CenQL) +---------------------------- + +The Platform API uses the Censys Query Language (CenQL) for searches. CenQL provides a unified syntax for searching across all Censys data types. Key features include: + +- Field queries: ``field_name: value`` (case-insensitive substring match) +- Exact equality: ``field_name = value`` (case-sensitive exact match) +- Regex matching: ``field_name =~ pattern`` +- Comparison operators: ``field_name > value``, ``field_name < value`` +- Nested field queries: ``parent_field: (child_field = value and another_field: value)`` + +Examples: + +.. code-block:: python + + # Hosts with SSH running on port 22 + query1 = "host.services: (port = 22 and protocol = 'SSH')" + + # Hosts with a specific software version + query2 = "host.services.software: (product = 'httpd' and version = '2.4.62')" + + # Hosts with a specific HTTP header + query3 = "host.services.endpoints.http.headers: (key = 'Server' and value.headers = 'nginx')" + + # Hosts running nginx with a specific welcome page + query4 = "host.services: (software.product = 'nginx' and endpoints.http.html_title = 'Welcome to nginx!')" + +For more details on CenQL syntax, see the `Censys Query Language documentation `_. + +Complete Example +-------------- + +.. include:: ../examples/platform/platform_client.py + :literal: diff --git a/examples/README.md b/examples/README.md index bb66a680..d219ed42 100644 --- a/examples/README.md +++ b/examples/README.md @@ -25,6 +25,19 @@ from censys.search import CensysCerts from censys.asm import AsmClient ``` +### Platform + +```python +# Access the Platform API +from censys.platform import CensysPlatform + +# Access specific Platform API components +from censys.platform import CensysHosts +from censys.platform import CensysCertificates +from censys.platform import CensysWebProperties +from censys.platform import CensysSearch +``` + ## Available Examples ### Search Examples @@ -54,3 +67,13 @@ from censys.asm import AsmClient - [Get Host Ricks](asm/get_host_risks.py) - [Get Domains and Subdomains](asm/get_subdomains.py) - [Add Seeds in Bulk (from CSV)](asm/add_seeds_bulk.py) + +### Platform Examples + +- [View Host Details](platform/view_host.py) +- [View Host Timeline](platform/host_timeline.py) +- [Bulk View Hosts](platform/bulk_view_hosts.py) +- [View Certificate Details](platform/view_certificates.py) +- [View Web Property Details](platform/view_webproperties.py) +- [Search and Aggregate Data](platform/search.py) +- [Manage Platform Resources](platform/resource_management.py) diff --git a/examples/platform/bulk_view_hosts.py b/examples/platform/bulk_view_hosts.py new file mode 100644 index 00000000..aac8013e --- /dev/null +++ b/examples/platform/bulk_view_hosts.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +"""Example showing how to bulk view hosts using the Censys Platform API.""" + +import json +import os +import sys + +from censys.common.config import get_config +from censys.platform import CensysHosts + +# Load configuration +config = get_config() + +# Get token from environment or config +CENSYS_PLATFORM_TOKEN = os.environ.get("CENSYS_PLATFORM_TOKEN") or config.get( + "DEFAULT", "platform_token" +) + +# Get organization ID from environment or config (required) +CENSYS_ORGANIZATION_ID = os.environ.get("CENSYS_ORGANIZATION_ID") or config.get( + "DEFAULT", "platform_org_id" +) + +if not CENSYS_PLATFORM_TOKEN: + print("Error: Censys Platform Token is required") + print( + "Set the CENSYS_PLATFORM_TOKEN environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +if not CENSYS_ORGANIZATION_ID: + print("Error: Censys Organization ID is required") + print( + "Set the CENSYS_ORGANIZATION_ID environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +# Initialize the Hosts API client with required organization ID +hosts = CensysHosts(token=CENSYS_PLATFORM_TOKEN, organization_id=CENSYS_ORGANIZATION_ID) + +# Define a list of IP addresses to look up +ip_addresses = ["1.1.1.1", "8.8.8.8", "9.9.9.9"] + +# Retrieve host details for multiple IP addresses at once +host_details = hosts.bulk_view(ip_addresses) + +# Print the results +print(f"Host details for {len(ip_addresses)} IPs:") +print(json.dumps(host_details, indent=2)) + +# You can also access individual hosts from the results +if "result" in host_details: + print("\nIndividual host summaries:") + for host_result in host_details["result"]: + if "host" in host_result and "ip" in host_result["host"]: + ip = host_result["host"]["ip"] + print(f"- {ip}: {len(host_result)} fields returned") diff --git a/examples/platform/host_timeline.py b/examples/platform/host_timeline.py new file mode 100644 index 00000000..84ebd71e --- /dev/null +++ b/examples/platform/host_timeline.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +"""Example showing how to retrieve a host's timeline using the Censys Platform API.""" + +import json +import os +import sys +from datetime import datetime, timedelta + +from censys.common.config import get_config +from censys.platform import CensysHosts + +# Load configuration +config = get_config() + +# Get token from environment or config +CENSYS_PLATFORM_TOKEN = os.environ.get("CENSYS_PLATFORM_TOKEN") or config.get( + "DEFAULT", "platform_token" +) + +# Get organization ID from environment or config (required) +CENSYS_ORGANIZATION_ID = os.environ.get("CENSYS_ORGANIZATION_ID") or config.get( + "DEFAULT", "platform_org_id" +) + +if not CENSYS_PLATFORM_TOKEN: + print("Error: Censys Platform Token is required") + print( + "Set the CENSYS_PLATFORM_TOKEN environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +if not CENSYS_ORGANIZATION_ID: + print("Error: Censys Organization ID is required") + print( + "Set the CENSYS_ORGANIZATION_ID environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +# Initialize the Hosts API client with required organization ID +hosts = CensysHosts(token=CENSYS_PLATFORM_TOKEN, organization_id=CENSYS_ORGANIZATION_ID) + +# Specify the IP address to lookup +ip_address = "8.8.8.8" + +# Define the timeframe +end_time = datetime.utcnow() +start_time = end_time - timedelta(days=30) # Last 30 days + +# Retrieve the host's timeline +print(f"Retrieving timeline for {ip_address} from {start_time} to {end_time}") +timeline = hosts.get_host_timeline(ip_address, start_time=start_time, end_time=end_time) + +# Print the results +print(f"Timeline events for {ip_address}:") +print(json.dumps(timeline, indent=2)) diff --git a/examples/platform/platform_client.py b/examples/platform/platform_client.py new file mode 100644 index 00000000..5cb41b25 --- /dev/null +++ b/examples/platform/platform_client.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +"""Example showing how to use the CensysPlatformClient to access all Censys Platform APIs.""" + +import os +import sys + +from censys.common.config import get_config +from censys.platform import CensysPlatformClient + +# Load configuration +config = get_config() + +# Get token from environment or config +CENSYS_PLATFORM_TOKEN = os.environ.get("CENSYS_PLATFORM_TOKEN") or config.get( + "DEFAULT", "platform_token" +) + +# Get organization ID from environment or config (required) +CENSYS_ORGANIZATION_ID = os.environ.get("CENSYS_ORGANIZATION_ID") or config.get( + "DEFAULT", "platform_org_id" +) + +if not CENSYS_PLATFORM_TOKEN: + print("Error: Censys Platform Token is required") + print( + "Set the CENSYS_PLATFORM_TOKEN environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +if not CENSYS_ORGANIZATION_ID: + print("Error: Censys Organization ID is required") + print( + "Set the CENSYS_ORGANIZATION_ID environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +# Initialize the Censys Platform client with required organization ID +client = CensysPlatformClient( + token=CENSYS_PLATFORM_TOKEN, organization_id=CENSYS_ORGANIZATION_ID +) + +print("Censys Platform Client Example") +print("=============================") +print( + "\nThe CensysPlatformClient provides access to all Censys Platform APIs through a single client:" +) + +# Example 1: Search for hosts using the Search API +print("\n1. Search for hosts using the Search API") +print("----------------------------------------") +search_query = "host.services: (port = 443 and protocol = 'TCP')" +print(f"Query: {search_query}") +search_results = client.search.query(search_query, page_size=2) +print(f"Found {search_results.get('result', {}).get('total', 0)} matching hosts") + +# Example 2: View a specific host using the Hosts API +print("\n2. View a specific host using the Hosts API") +print("------------------------------------------") +host_ip = "8.8.8.8" +print(f"Viewing host: {host_ip}") +try: + host_details = client.hosts.view(host_ip) + print(f"Host Name: {host_details.get('host', {}).get('names', ['Unknown'])[0]}") + services = host_details.get("host", {}).get("services", []) + print(f"Services: {len(services)} open ports detected") + for service in services[:3]: # Show first 3 services + print( + f" - Port {service.get('port')}: {service.get('service_name', 'Unknown service')}" + ) +except Exception as e: + print(f"Error viewing host: {e}") + +# Example 3: View a certificate using the Certificates API +print("\n3. View a certificate using the Certificates API") +print("-----------------------------------------------") +print("Searching for a Google certificate...") +cert_query = "certificate.issuer.common_name: *Google*" +cert_results = client.search.query(cert_query, page_size=1) +if cert_results.get("result", {}).get("hits"): + cert_id = cert_results["result"]["hits"][0].get("fingerprint") + print(f"Found certificate: {cert_id}") + try: + cert_details = client.certificates.view(cert_id) + cert_data = cert_details.get("certificate", {}) + print(f"Subject: {cert_data.get('subject', {}).get('common_name', 'Unknown')}") + print(f"Issuer: {cert_data.get('issuer', {}).get('common_name', 'Unknown')}") + print(f"Valid from: {cert_data.get('validity', {}).get('start')}") + print(f"Valid until: {cert_data.get('validity', {}).get('end')}") + except Exception as e: + print(f"Error viewing certificate: {e}") +else: + print("No matching certificates found") + +# Example 4: Aggregate data using the Search API +print("\n4. Aggregate data using the Search API") +print("-------------------------------------") +agg_query = "host.services.port: 443" +agg_field = "host.services.service_name" +print(f"Aggregating {agg_field} where {agg_query}") +try: + agg_results = client.search.aggregate(agg_query, agg_field) + buckets = agg_results.get("result", {}).get("buckets", []) + print("Top service names on port 443:") + for bucket in buckets[:5]: # Show top 5 + print(f" - {bucket.get('key')}: {bucket.get('count')} hosts") +except Exception as e: + print(f"Error aggregating data: {e}") + +print( + "\nThe CensysPlatformClient provides a unified interface to all Censys Platform APIs." +) +print("You can access the following APIs through this client:") +print(" - client.search: For searching across all data types") +print(" - client.hosts: For host-specific operations") +print(" - client.certificates: For certificate-specific operations") +print(" - client.webproperties: For web property-specific operations") diff --git a/examples/platform/search.py b/examples/platform/search.py new file mode 100644 index 00000000..5679ef5f --- /dev/null +++ b/examples/platform/search.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +"""Example showing how to use search with the Censys Platform API.""" + +import json +import os +import sys +import traceback + +from censys.common.config import get_config +from censys.common.exceptions import CensysAPIException +from censys.platform import CensysSearch + +# Load configuration +config = get_config() + +# Get token from environment or config +CENSYS_PLATFORM_TOKEN = os.environ.get("CENSYS_PLATFORM_TOKEN") or config.get( + "DEFAULT", "platform_token" +) + +# Get organization ID from environment or config (required) +CENSYS_ORGANIZATION_ID = os.environ.get("CENSYS_ORGANIZATION_ID") or config.get( + "DEFAULT", "platform_org_id" +) + +if not CENSYS_PLATFORM_TOKEN: + print("Error: Censys Platform Token is required") + print( + "Set the CENSYS_PLATFORM_TOKEN environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +if not CENSYS_ORGANIZATION_ID: + print("Error: Censys Organization ID is required") + print( + "Set the CENSYS_ORGANIZATION_ID environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +# Initialize the Search API client with required organization ID +search = CensysSearch( + token=CENSYS_PLATFORM_TOKEN, organization_id=CENSYS_ORGANIZATION_ID +) + +try: + # Example 1: Simple Search Query + print("Example 1: Simple Search Query") + # Using a very simple query with a different format + simple_query = 'host.ip: "8.8.8.8"' + print(f"Query: {simple_query}") + + simple_results = search.query(simple_query, page_size=10) + + # Print total and sample results + print(f"Found {simple_results.get('result', {}).get('total', 0)} results") + print("Sample hosts:") + for host in simple_results.get("result", {}).get("hits", []): + print(f" - {host.get('ip')} ({host.get('name', 'No name')})") + + # Example 2: Advanced Search Query with multiple filters + print("\nExample 2: Advanced Search Query") + # Using a simpler advanced query with a different format + advanced_query = 'web.endpoints.http.html_title: "Google"' + print(f"Query: {advanced_query}") + advanced_results = search.query(advanced_query, page_size=5) + + # Print the JSON results with formatting + print(f"Found {advanced_results.get('result', {}).get('total', 0)} results") + print("First result:") + if advanced_results.get("result", {}).get("hits", []): + print( + json.dumps(advanced_results.get("result", {}).get("hits", [])[0], indent=2) + ) + else: + print("No results found") + +except CensysAPIException as e: + print(f"API Exception: {e}") + print(f"Status code: {getattr(e, 'status_code', 'unknown')}") + print(f"Error code: {getattr(e, 'error_code', 'unknown')}") + print(f"Error message: {getattr(e, 'message', 'unknown')}") + print(f"Details: {getattr(e, 'details', 'unknown')}") + traceback.print_exc() +except Exception as e: + print(f"Unexpected error: {e}") + traceback.print_exc() diff --git a/examples/platform/unified_search.py b/examples/platform/unified_search.py new file mode 100644 index 00000000..41813da0 --- /dev/null +++ b/examples/platform/unified_search.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +"""Example showing how to use the unified Censys Query Language (CenQL) with the Censys Platform API.""" + +import os +import sys + +from censys.common.config import get_config +from censys.platform import CensysPlatformClient + +# Load configuration +config = get_config() + +# Get token from environment or config +CENSYS_PLATFORM_TOKEN = os.environ.get("CENSYS_PLATFORM_TOKEN") or config.get( + "DEFAULT", "platform_token" +) + +# Get organization ID from environment or config (required) +CENSYS_ORGANIZATION_ID = os.environ.get("CENSYS_ORGANIZATION_ID") or config.get( + "DEFAULT", "platform_org_id" +) + +if not CENSYS_PLATFORM_TOKEN: + print("Error: Censys Platform Token is required") + print( + "Set the CENSYS_PLATFORM_TOKEN environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +if not CENSYS_ORGANIZATION_ID: + print("Error: Censys Organization ID is required") + print( + "Set the CENSYS_ORGANIZATION_ID environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +# Initialize the Censys Platform client with required organization ID +# This provides access to all APIs through a single client +client = CensysPlatformClient( + token=CENSYS_PLATFORM_TOKEN, organization_id=CENSYS_ORGANIZATION_ID +) + +print("Censys Query Language (CenQL) Examples") +print("=====================================") + +# --------------------------------------------- +# Example 1: Simple field queries +# --------------------------------------------- +print("\n1. Simple Field Queries") +print("----------------------") + +# Hosts query with field operators +hosts_query = "host.services.port: 443 AND host.services.service_name: HTTPS" +print(f"Hosts query: {hosts_query}") +hosts_results = client.search.query(hosts_query, per_page=2) +print(f"Found {hosts_results.get('result', {}).get('total', 0)} matching hosts") +if hosts_results.get("result", {}).get("hits"): + print("Sample host IP:", hosts_results["result"]["hits"][0].get("ip")) + +# Web properties query with equality operator +webprops_query = "webproperty.protocol = HTTPS AND webproperty.services.http.response.html_title: Google" +print(f"\nWeb properties query: {webprops_query}") +webprops_results = client.search.query(webprops_query, page_size=2) +print( + f"Found {webprops_results.get('result', {}).get('total', 0)} matching web properties" +) +if webprops_results.get("result", {}).get("hits"): + print("Sample web property:", webprops_results["result"]["hits"][0].get("name")) + +# Certificates query with wildcard +cert_query = "certificate.issuer.common_name: *Google*" +print(f"\nCertificates query: {cert_query}") +cert_results = client.search.query(cert_query, page_size=2) +print(f"Found {cert_results.get('result', {}).get('total', 0)} matching certificates") +if cert_results.get("result", {}).get("hits"): + print( + "Sample certificate fingerprint:", + cert_results["result"]["hits"][0].get("fingerprint"), + ) + +# --------------------------------------------- +# Example 2: Advanced field queries +# --------------------------------------------- +print("\n2. Advanced Field Queries") +print("------------------------") + +# Advanced query with comparison operators +advanced_query = "host.services.port: 22 AND host.services.service_name: SSH" +print(f"Query: {advanced_query}") +advanced_results = client.search.query(advanced_query, per_page=2) +print(f"Found {advanced_results.get('result', {}).get('total', 0)} matching hosts") + +# Advanced query with NOT operator +not_query = "host.services.port: 443 AND NOT host.services.service_name: HTTPS" +print(f"\nNOT query: {not_query}") +not_results = client.search.query(not_query, page_size=2) +print(f"Found {not_results.get('result', {}).get('total', 0)} matching hosts") + +# --------------------------------------------- +# Example 3: Nested field queries +# --------------------------------------------- +print("\n3. Nested Field Queries") +print("----------------------") + +# Nested field query for specific service and port +nested_query = 'host.services.banner: "Apache"' +print(f"Query: {nested_query}") +nested_results = client.search.query(nested_query, per_page=2) +print(f"Found {nested_results.get('result', {}).get('total', 0)} matching hosts") + +# 4. Complex Nested Field Queries +print("\n4. Complex Nested Field Queries") +print("------------------------------") + +complex_nested_query = 'host.services.banner: "Apache" AND host.services.port: 80' +print(f"Query: {complex_nested_query}") +complex_results = client.search.query(complex_nested_query, per_page=2) +print(f"Found {complex_results.get('result', {}).get('total', 0)} matching hosts") + +# --------------------------------------------- +# Example 5: Specialized search techniques +# --------------------------------------------- +print("\n5. Specialized Search Techniques") +print("------------------------------") + +# Regex matching +regex_query = "host.names =~ '.*\\.google\\.com'" +print(f"\nRegex query: {regex_query}") +regex_results = client.search.query(regex_query, per_page=2) +print(f"Found {regex_results.get('result', {}).get('total', 0)} matching hosts") + +# Non-zero value matching (finding hosts with any value for a field) +exists_query = "host.services.http.response.html_title: *" +print(f"\nExists query: {exists_query}") +exists_results = client.search.query(exists_query, per_page=2) +print(f"Found {exists_results.get('result', {}).get('total', 0)} matching hosts") + +# --------------------------------------------- +# Example 6: Aggregations +# --------------------------------------------- +print("\n6. Aggregations") +print("-------------") + +# Aggregate hosts by service name +agg_query = "host.services.port: 443" +agg_field = "host.services.service_name" +print(f"\nAggregating '{agg_field}' where '{agg_query}'") +agg_results = client.search.aggregate(agg_query, agg_field) +buckets = agg_results.get("result", {}).get("buckets", []) +print(f"Found {len(buckets)} buckets") +for i, bucket in enumerate(buckets[:5]): # Show top 5 + print(f" {i+1}. {bucket.get('key')}: {bucket.get('count')} hosts") + +print("\nCenQL provides a unified query syntax across all Censys data types.") +print("Visit https://docs.censys.com/docs/censys-query-language for more details.") diff --git a/examples/platform/view_certificates.py b/examples/platform/view_certificates.py new file mode 100644 index 00000000..d6572609 --- /dev/null +++ b/examples/platform/view_certificates.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +"""Example showing how to view certificate details using the Censys Platform API.""" + +import json +import os +import sys + +from censys.common.config import get_config +from censys.platform import CensysCertificates + +# Load configuration +config = get_config() + +# Get token from environment or config +CENSYS_PLATFORM_TOKEN = os.environ.get("CENSYS_PLATFORM_TOKEN") or config.get( + "DEFAULT", "platform_token" +) + +# Get organization ID from environment or config (required) +CENSYS_ORGANIZATION_ID = os.environ.get("CENSYS_ORGANIZATION_ID") or config.get( + "DEFAULT", "platform_org_id" +) + +if not CENSYS_PLATFORM_TOKEN: + print("Error: Censys Platform Token is required") + print( + "Set the CENSYS_PLATFORM_TOKEN environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +if not CENSYS_ORGANIZATION_ID: + print("Error: Censys Organization ID is required") + print( + "Set the CENSYS_ORGANIZATION_ID environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +# Initialize the Certificates API client with required organization ID +certificates = CensysCertificates( + token=CENSYS_PLATFORM_TOKEN, organization_id=CENSYS_ORGANIZATION_ID +) + +# Specify the certificate hash to lookup +# Replace this with a valid certificate hash +cert_hash = "fb444eb8e68437bae06232b9f5091bccff62a768ca09e92eb5c9c2cf9d17c426" + +# Retrieve certificate details +cert_details = certificates.view(cert_hash) + +# Print the result with nice formatting +print(f"Certificate Details for {cert_hash}:") +print(json.dumps(cert_details, indent=2)) diff --git a/examples/platform/view_host.py b/examples/platform/view_host.py new file mode 100644 index 00000000..f08e8fbe --- /dev/null +++ b/examples/platform/view_host.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +"""Example showing how to view host details using the Censys Platform API.""" + +import json +import os +import sys +from datetime import datetime, timedelta + +from censys.common.config import get_config +from censys.platform import CensysHosts + +# Load configuration +config = get_config() + +# Get token from environment or config +CENSYS_PLATFORM_TOKEN = os.environ.get("CENSYS_PLATFORM_TOKEN") or config.get( + "DEFAULT", "platform_token" +) + +# Get organization ID from environment or config (required) +CENSYS_ORGANIZATION_ID = os.environ.get("CENSYS_ORGANIZATION_ID") or config.get( + "DEFAULT", "platform_org_id" +) + +if not CENSYS_PLATFORM_TOKEN: + print("Error: Censys Platform Token is required") + print( + "Set the CENSYS_PLATFORM_TOKEN environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +if not CENSYS_ORGANIZATION_ID: + print("Error: Censys Organization ID is required") + print( + "Set the CENSYS_ORGANIZATION_ID environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +# Initialize the Hosts API client with required organization ID +hosts = CensysHosts(token=CENSYS_PLATFORM_TOKEN, organization_id=CENSYS_ORGANIZATION_ID) + +# Specify the IP address to lookup +ip_address = "8.8.8.8" + +# Retrieve host details +host_details = hosts.view(ip_address) + +# Print the result with nice formatting +print(f"Host Details for {ip_address}:") +print(json.dumps(host_details, indent=2)) + +# Example: Get host details at a specific time (1 week ago) +one_week_ago = datetime.utcnow() - timedelta(days=7) +print(f"\nHost Details for {ip_address} from one week ago:") +historical_details = hosts.view(ip_address, at_time=one_week_ago) +print(json.dumps(historical_details, indent=2)) diff --git a/examples/platform/view_webproperties.py b/examples/platform/view_webproperties.py new file mode 100644 index 00000000..d7a10052 --- /dev/null +++ b/examples/platform/view_webproperties.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +"""Example showing how to view web properties using the Censys Platform API.""" + +import json +import os +import sys +from datetime import datetime, timedelta + +from censys.common.config import get_config +from censys.platform import CensysWebProperties + +# Load configuration +config = get_config() + +# Get token from environment or config +CENSYS_PLATFORM_TOKEN = os.environ.get("CENSYS_PLATFORM_TOKEN") or config.get( + "DEFAULT", "platform_token" +) + +# Get organization ID from environment or config (required) +CENSYS_ORGANIZATION_ID = os.environ.get("CENSYS_ORGANIZATION_ID") or config.get( + "DEFAULT", "platform_org_id" +) + +if not CENSYS_PLATFORM_TOKEN: + print("Error: Censys Platform Token is required") + print( + "Set the CENSYS_PLATFORM_TOKEN environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +if not CENSYS_ORGANIZATION_ID: + print("Error: Censys Organization ID is required") + print( + "Set the CENSYS_ORGANIZATION_ID environment variable or configure it using 'censys platform config'" + ) + sys.exit(1) + +# Initialize the WebProperties API client with required organization ID +webproperties = CensysWebProperties( + token=CENSYS_PLATFORM_TOKEN, organization_id=CENSYS_ORGANIZATION_ID +) + +# Specify the WebProperty ID to lookup +webproperty_id = "google.com:443" + +# Retrieve webproperty details +webproperty_details = webproperties.view(webproperty_id) + +# Print the result with nice formatting +print(f"WebProperty Details for {webproperty_id}:") +print(json.dumps(webproperty_details, indent=2)) + +# Example: Get webproperty details at a specific time (1 week ago) +one_week_ago = datetime.utcnow() - timedelta(days=7) +print(f"\nWebProperty Details for {webproperty_id} from one week ago:") +historical_details = webproperties.view(webproperty_id, at_time=one_week_ago) +print(json.dumps(historical_details, indent=2)) + +# Example: Bulk view multiple web properties +multi_props = ["example.com:443", "example.org:443", "example.net:443"] +print(f"\nBulk viewing {len(multi_props)} web properties:") +bulk_results = webproperties.bulk_view(multi_props) +print(json.dumps(bulk_results, indent=2)) diff --git a/tests/cli/test_config.py b/tests/cli/test_config.py index 3f22d00f..66370687 100644 --- a/tests/cli/test_config.py +++ b/tests/cli/test_config.py @@ -113,8 +113,8 @@ def test_config_default(self): "censys.common.config.os.path.isfile", return_value=True ) config = get_config() - mock_isfile.return_value = False mock_isfile.assert_called_with(TEST_CONFIG_PATH) + mock_isfile.return_value = False self.mock_open.assert_called_once() for key, value in default_config.items(): assert value == config.get(DEFAULT, key) diff --git a/tests/platform/__init__.py b/tests/platform/__init__.py new file mode 100644 index 00000000..e9433946 --- /dev/null +++ b/tests/platform/__init__.py @@ -0,0 +1 @@ +"""Test Censys Platform API.""" diff --git a/tests/platform/test_client.py b/tests/platform/test_client.py new file mode 100644 index 00000000..22362e9c --- /dev/null +++ b/tests/platform/test_client.py @@ -0,0 +1,67 @@ +"""Tests for the Censys Platform Client.""" + +import unittest +from unittest.mock import patch + +from censys.platform import CensysPlatformClient +from censys.platform.v3 import ( + CensysCertificates, + CensysHosts, + CensysSearch, + CensysWebProperties, +) + + +class CensysPlatformClientTests(unittest.TestCase): + """Tests for the CensysPlatformClient class.""" + + def setUp(self): + """Set up test case.""" + self.client = CensysPlatformClient( + token="test-token", organization_id="test-org-id" + ) + + def test_client_initialization(self): + """Test that client initializes properly with all API instances.""" + # Verify client has all expected API instances + assert isinstance(self.client.hosts, CensysHosts) + assert isinstance(self.client.certificates, CensysCertificates) + assert isinstance(self.client.webproperties, CensysWebProperties) + assert isinstance(self.client.search, CensysSearch) + + def test_credential_propagation(self): + """Test that credentials are properly propagated to all API instances.""" + # Check that organization_id was passed to all instances + assert self.client.hosts.organization_id == "test-org-id" + assert self.client.certificates.organization_id == "test-org-id" + assert self.client.webproperties.organization_id == "test-org-id" + assert self.client.search.organization_id == "test-org-id" + + # Check that token was passed to all instances (indirectly by checking session headers) + assert ( + self.client.hosts._session.headers["Authorization"] == "Bearer test-token" + ) + assert ( + self.client.certificates._session.headers["Authorization"] + == "Bearer test-token" + ) + assert ( + self.client.webproperties._session.headers["Authorization"] + == "Bearer test-token" + ) + assert ( + self.client.search._session.headers["Authorization"] == "Bearer test-token" + ) + + @patch("censys.platform.v3.search.CensysSearch.query") + def test_search_delegation(self, mock_query): + """Test that search query calls are properly delegated to the appropriate API instances.""" + # Setup mock return value + mock_query.return_value = {"result": {"total": 1, "hits": [{"ip": "8.8.8.8"}]}} + + # Call query through client + result = self.client.search.query("host.services: (port = 22)") + + # Verify query was called with correct parameters + mock_query.assert_called_once_with("host.services: (port = 22)") + assert result == {"result": {"total": 1, "hits": [{"ip": "8.8.8.8"}]}} diff --git a/tests/platform/v3/__init__.py b/tests/platform/v3/__init__.py new file mode 100644 index 00000000..8aeeb29e --- /dev/null +++ b/tests/platform/v3/__init__.py @@ -0,0 +1 @@ +"""Test Censys Platform v3 API.""" diff --git a/tests/platform/v3/test_api.py b/tests/platform/v3/test_api.py new file mode 100644 index 00000000..0645e38d --- /dev/null +++ b/tests/platform/v3/test_api.py @@ -0,0 +1,118 @@ +"""Tests for the Censys Platform API base class.""" + +import unittest +from unittest.mock import mock_open, patch + +import pytest +from parameterized import parameterized + +from tests.utils import CensysTestCase + +from censys.common.exceptions import ( + CensysException, + CensysNotFoundException, + CensysRateLimitExceededException, + CensysUnauthorizedException, +) +from censys.platform.v3.api import CensysPlatformAPIv3 + +# Test constants +PLATFORM_URL = "https://platform.censys.io/api" + +# Exception parameters for testing +PlatformExceptionParams = [ + (401, CensysUnauthorizedException), + (403, CensysUnauthorizedException), + (404, CensysNotFoundException), + (429, CensysRateLimitExceededException), +] + + +class CensysPlatformAPITests(CensysTestCase): + """Tests for the base Platform API class.""" + + api: CensysPlatformAPIv3 + + def setUp(self): + """Set up test case.""" + self.api = CensysPlatformAPIv3(token="test-token") + self.api_with_org = CensysPlatformAPIv3( + token="test-token", organization_id="test-org-id" + ) + + @parameterized.expand(PlatformExceptionParams) + def test_get_exception_class(self, status_code, exception): + """Test getting exception class by status code. + + Args: + status_code: HTTP status code to test. + exception: Expected exception class. + """ + res = unittest.mock.Mock() + res.status_code = status_code + assert self.api._get_exception_class(res) == exception + + def test_headers(self): + """Test that proper headers are set.""" + headers = self.api._session.headers + assert "Authorization" in headers + assert headers["Authorization"] == "Bearer test-token" + assert headers["Accept"] == "application/json" + assert headers["Content-Type"] == "application/json" + + def test_headers_with_org_id(self): + """Test that proper headers are set when organization ID is provided.""" + headers = self.api_with_org._session.headers + assert "Authorization" in headers + assert headers["Authorization"] == "Bearer test-token" + assert headers["Accept"] == "application/json" + assert headers["Content-Type"] == "application/json" + assert "X-Organization-ID" in headers + assert headers["X-Organization-ID"] == "test-org-id" + + def test_get_with_organization_id(self): + """Test that _get method includes organization_id in parameters when provided.""" + with patch.object(self.api_with_org, "_session") as mock_session: + mock_response = unittest.mock.Mock() + mock_response.json.return_value = {"status": "ok"} + mock_session.get.return_value = mock_response + + self.api_with_org._get("test/endpoint") + + # Verify organization_id was added to the query parameters + args, kwargs = mock_session.get.call_args + assert "params" in kwargs + assert kwargs["params"].get("organization_id") == "test-org-id" + + def test_asset_specific_accept_header(self): + """Test that asset-specific Accept header is set when ASSET_NAME is provided.""" + + class TestAssetAPI(CensysPlatformAPIv3): + """Test API class with asset name.""" + + ASSET_NAME = "test-asset" + + # Create a test API with asset name + test_api = TestAssetAPI(token="test-token") + headers = test_api._session.headers + + expected_header = "application/vnd.censys.api.v3.test-asset.v1+json" + assert headers["Accept"] == expected_header + + +@patch.dict( + "os.environ", {"CENSYS_API_ID": "", "CENSYS_API_SECRET": "", "CENSYS_API_URL": ""} +) +class CensysPlatformAPIBaseTestsNoEnv(unittest.TestCase): + """Tests for the Platform API base class without environment variables.""" + + @patch("builtins.open", new_callable=mock_open, read_data="[DEFAULT]") + @patch.object(CensysPlatformAPIv3, "DEFAULT_URL", "") + def test_no_env(self, mock_file): + """Test initialization with no environment variables. + + Args: + mock_file: Mock file object. + """ + with pytest.raises(CensysException, match="No API url configured"): + CensysPlatformAPIv3() diff --git a/tests/platform/v3/test_certificates.py b/tests/platform/v3/test_certificates.py new file mode 100644 index 00000000..3b180ab6 --- /dev/null +++ b/tests/platform/v3/test_certificates.py @@ -0,0 +1,81 @@ +"""Tests for the Censys Platform Certificate API.""" + +from unittest.mock import patch + +from tests.utils import CensysTestCase + +from censys.platform.v3.certificates import CensysCertificates + +# Test constants +PLATFORM_URL = "https://api.platform.censys.io" +TEST_CERTIFICATE = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" +TEST_CERTIFICATES = [ + "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", + "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2", +] +TEST_CERTIFICATE_RESPONSE = {"certificate": {"fingerprint_sha256": TEST_CERTIFICATE}} +TEST_BULK_RESPONSE = { + "result": [ + {"certificate": {"fingerprint_sha256": cert}} for cert in TEST_CERTIFICATES + ] +} + + +class TestCertificates(CensysTestCase): + """Tests for the Censys Platform Certificate API.""" + + def setUp(self): + """Set up test case.""" + self.api = CensysCertificates(token="test-token") + self.api_with_org = CensysCertificates( + token="test-token", organization_id="test-org-id" + ) + + def test_init(self): + """Test initialization.""" + assert self.api._token == "test-token" + assert self.api.INDEX_NAME == "v3/global/asset/certificate" + assert self.api.ASSET_NAME == "certificate" + + def test_init_with_org_id(self): + """Test initialization with organization ID.""" + assert self.api_with_org._token == "test-token" + assert self.api_with_org.organization_id == "test-org-id" + assert "X-Organization-ID" in self.api_with_org._session.headers + assert self.api_with_org._session.headers["X-Organization-ID"] == "test-org-id" + + def test_view(self): + """Test view method.""" + with patch.object(self.api, "_get") as mock_get: + self.api.view(TEST_CERTIFICATE) + mock_get.assert_called_with( + f"v3/global/asset/certificate/{TEST_CERTIFICATE}" + ) + + def test_view_with_org_id(self): + """Test view method with organization ID.""" + with patch.object(self.api_with_org, "_get") as mock_get: + self.api_with_org.view(TEST_CERTIFICATE) + mock_get.assert_called_with( + f"v3/global/asset/certificate/{TEST_CERTIFICATE}" + ) + # The _get method in the base class will add organization_id to the params + + def test_bulk_view(self): + """Test bulk_view method.""" + with patch.object(self.api, "_get") as mock_get: + self.api.bulk_view(TEST_CERTIFICATES) + mock_get.assert_called_with( + "v3/global/asset/certificate", + params={"certificate_ids": TEST_CERTIFICATES}, + ) + + def test_bulk_view_with_org_id(self): + """Test bulk_view method with organization ID.""" + with patch.object(self.api_with_org, "_get") as mock_get: + self.api_with_org.bulk_view(TEST_CERTIFICATES) + mock_get.assert_called_with( + "v3/global/asset/certificate", + params={"certificate_ids": TEST_CERTIFICATES}, + ) + # The _get method in the base class will add organization_id to the params diff --git a/tests/platform/v3/test_hosts.py b/tests/platform/v3/test_hosts.py new file mode 100644 index 00000000..67c54443 --- /dev/null +++ b/tests/platform/v3/test_hosts.py @@ -0,0 +1,122 @@ +"""Tests for the Censys Platform Hosts API.""" + +import datetime +from unittest.mock import patch + +import responses + +from tests.utils import CensysTestCase + +from censys.platform.v3.hosts import CensysHosts + +# Test constants +TEST_HOST = "1.1.1.1" +TEST_HOSTS = ["1.1.1.1", "8.8.8.8"] +PLATFORM_URL = "https://api.platform.censys.io" +TEST_HOST_RESPONSE = {"host": {"name": TEST_HOST}} +TEST_BULK_RESPONSE = {"result": [{"host": {"name": host}} for host in TEST_HOSTS]} +TEST_TIMELINE_RESPONSE = {"result": [{"timestamp": "2023-01-01T00:00:00Z"}]} + + +class TestHosts(CensysTestCase): + """Tests for the Platform Hosts API.""" + + def setUp(self): + """Set up test case.""" + self.api = CensysHosts(token="test-token") + self.api_with_org = CensysHosts( + token="test-token", organization_id="test-org-id" + ) + + def test_init(self): + """Test initialization.""" + assert self.api._token == "test-token" + assert self.api.INDEX_NAME == "v3/global/asset/host" + assert self.api.ASSET_NAME == "host" + + def test_init_with_org_id(self): + """Test initialization with organization ID.""" + assert self.api_with_org._token == "test-token" + assert self.api_with_org.organization_id == "test-org-id" + assert "X-Organization-ID" in self.api_with_org._session.headers + assert self.api_with_org._session.headers["X-Organization-ID"] == "test-org-id" + + def test_view(self): + """Test view method.""" + with patch.object(self.api, "_get") as mock_get: + self.api.view("1.1.1.1") + mock_get.assert_called_with("v3/global/asset/host/1.1.1.1", params={}) + + def test_view_with_org_id(self): + """Test view method with organization ID.""" + with patch.object(self.api_with_org, "_get") as mock_get: + self.api_with_org.view("1.1.1.1") + mock_get.assert_called_with("v3/global/asset/host/1.1.1.1", params={}) + # The _get method in the base class will add organization_id to the params + + def test_bulk_view(self): + """Test bulk_view method.""" + with patch.object(self.api, "_get") as mock_get: + self.api.bulk_view(["1.1.1.1", "8.8.8.8"]) + mock_get.assert_called_with( + "v3/global/asset/host", params={"host_ids": ["1.1.1.1", "8.8.8.8"]} + ) + + def test_bulk_view_with_org_id(self): + """Test bulk_view method with organization ID.""" + with patch.object(self.api_with_org, "_get") as mock_get: + self.api_with_org.bulk_view(["1.1.1.1", "8.8.8.8"]) + mock_get.assert_called_with( + "v3/global/asset/host", params={"host_ids": ["1.1.1.1", "8.8.8.8"]} + ) + # The _get method in the base class will add organization_id to the params + + def test_timeline(self): + """Test timeline method.""" + with patch.object(self.api, "_get") as mock_get: + self.api.timeline("1.1.1.1") + mock_get.assert_called_with( + "v3/global/asset/host/1.1.1.1/timeline", params={} + ) + + def test_timeline_with_org_id(self): + """Test timeline method with organization ID.""" + with patch.object(self.api_with_org, "_get") as mock_get: + self.api_with_org.timeline("1.1.1.1") + mock_get.assert_called_with( + "v3/global/asset/host/1.1.1.1/timeline", params={} + ) + # The _get method in the base class will add organization_id to the params + + @responses.activate + def test_view_at_time(self): + """Test viewing host details at a specific time.""" + test_date = datetime.datetime(2023, 1, 1) + responses.add( + responses.GET, + f"{PLATFORM_URL}/v3/global/asset/host/{TEST_HOST}", + json=TEST_HOST_RESPONSE, + status=200, + ) + res = self.api.view(TEST_HOST, at_time=test_date) + assert res == TEST_HOST_RESPONSE + assert "at_time" in responses.calls[0].request.url + assert "2023-01-01" in responses.calls[0].request.url + + @responses.activate + def test_timeline_with_dates(self): + """Test viewing host timeline with date parameters.""" + start_date = datetime.datetime(2023, 1, 1) + end_date = datetime.datetime(2023, 1, 2) + responses.add( + responses.GET, + f"{PLATFORM_URL}/v3/global/asset/host/{TEST_HOST}/timeline", + json=TEST_TIMELINE_RESPONSE, + status=200, + ) + res = self.api.timeline(TEST_HOST, start_time=start_date, end_time=end_date) + assert res == TEST_TIMELINE_RESPONSE + assert "start_time" in responses.calls[0].request.url + assert "end_time" in responses.calls[0].request.url + assert "2023-01-01" in responses.calls[0].request.url + assert "2023-01-02" in responses.calls[0].request.url diff --git a/tests/platform/v3/test_search.py b/tests/platform/v3/test_search.py new file mode 100644 index 00000000..da184c03 --- /dev/null +++ b/tests/platform/v3/test_search.py @@ -0,0 +1,123 @@ +"""Tests for the Censys Platform Search API.""" + +from unittest.mock import patch + +from tests.utils import CensysTestCase + +from censys.platform.v3.search import CensysSearch + +# Test constants +PLATFORM_URL = "https://api.platform.censys.io" +TEST_SEARCH_QUERY = "services.service_name: HTTPS" +TEST_SEARCH_RESPONSE = {"result": {"hits": [{"ip": "1.1.1.1"}]}} +TEST_AGGREGATE_RESPONSE = {"result": {"buckets": [{"key": "HTTPS", "count": 1}]}} + + +class TestSearch(CensysTestCase): + """Tests for the Censys Platform Search API.""" + + def setUp(self): + """Set up test case.""" + self.api = CensysSearch(token="test-token") + self.api_with_org = CensysSearch( + token="test-token", organization_id="test-org-id" + ) + + def test_init(self): + """Test initialization.""" + assert self.api._token == "test-token" + assert self.api.INDEX_NAME == "v3/global/search" + + def test_init_with_org_id(self): + """Test initialization with organization ID.""" + assert self.api_with_org._token == "test-token" + assert self.api_with_org.organization_id == "test-org-id" + assert "X-Organization-ID" in self.api_with_org._session.headers + assert self.api_with_org._session.headers["X-Organization-ID"] == "test-org-id" + + def test_query(self): + """Test search query.""" + with patch.object(self.api, "_post") as mock_post: + self.api.query(TEST_SEARCH_QUERY) + mock_post.assert_called_with( + "v3/global/search/query", data={"q": TEST_SEARCH_QUERY, "per_page": 100} + ) + + def test_query_with_org_id(self): + """Test search query with organization ID.""" + with patch.object(self.api_with_org, "_post") as mock_post: + self.api_with_org.query(TEST_SEARCH_QUERY) + mock_post.assert_called_with( + "v3/global/search/query", data={"q": TEST_SEARCH_QUERY, "per_page": 100} + ) + # The _post method in the base class will add organization_id to the params + + def test_query_with_params(self): + """Test search query with parameters.""" + with patch.object(self.api, "_post") as mock_post: + self.api.query( + TEST_SEARCH_QUERY, + per_page=50, + cursor="nextCursor", + fields=["ip", "services.port"], + sort="ip", + ) + mock_post.assert_called_with( + "v3/global/search/query", + data={ + "q": TEST_SEARCH_QUERY, + "per_page": 50, + "cursor": "nextCursor", + "fields": ["ip", "services.port"], + "sort": "ip", + }, + ) + + def test_query_with_params_and_org_id(self): + """Test search query with parameters and organization ID.""" + with patch.object(self.api_with_org, "_post") as mock_post: + self.api_with_org.query( + TEST_SEARCH_QUERY, + per_page=50, + cursor="nextCursor", + fields=["ip", "services.port"], + sort="ip", + ) + mock_post.assert_called_with( + "v3/global/search/query", + data={ + "q": TEST_SEARCH_QUERY, + "per_page": 50, + "cursor": "nextCursor", + "fields": ["ip", "services.port"], + "sort": "ip", + }, + ) + # The _post method in the base class will add organization_id to the params + + def test_aggregate(self): + """Test search aggregate.""" + with patch.object(self.api, "_post") as mock_post: + self.api.aggregate(TEST_SEARCH_QUERY, "services.service_name") + mock_post.assert_called_with( + "v3/global/search/aggregate", + data={ + "q": TEST_SEARCH_QUERY, + "field": "services.service_name", + "num_buckets": 50, + }, + ) + + def test_aggregate_with_org_id(self): + """Test search aggregate with organization ID.""" + with patch.object(self.api_with_org, "_post") as mock_post: + self.api_with_org.aggregate(TEST_SEARCH_QUERY, "services.service_name") + mock_post.assert_called_with( + "v3/global/search/aggregate", + data={ + "q": TEST_SEARCH_QUERY, + "field": "services.service_name", + "num_buckets": 50, + }, + ) + # The _post method in the base class will add organization_id to the params diff --git a/tests/platform/v3/test_webproperties.py b/tests/platform/v3/test_webproperties.py new file mode 100644 index 00000000..81661434 --- /dev/null +++ b/tests/platform/v3/test_webproperties.py @@ -0,0 +1,98 @@ +"""Tests for the Censys Platform WebProperty API.""" + +import datetime +from unittest.mock import patch + +from tests.utils import CensysTestCase + +from censys.platform.v3.webproperties import CensysWebProperties + +# Test constants +PLATFORM_URL = "https://api.platform.censys.io" +TEST_WEBPROPERTY = "example.com:443" +TEST_WEBPROPERTIES = ["example.com:443", "example.org:443"] +TEST_WEBPROPERTY_RESPONSE = {"webproperty": {"name": TEST_WEBPROPERTY}} +TEST_BULK_RESPONSE = { + "result": [{"webproperty": {"name": wp}} for wp in TEST_WEBPROPERTIES] +} + + +class TestWebProperties(CensysTestCase): + """Tests for the Censys Platform WebProperty API.""" + + def setUp(self): + """Set up test case.""" + self.api = CensysWebProperties(token="test-token") + self.api_with_org = CensysWebProperties( + token="test-token", organization_id="test-org-id" + ) + + def test_init(self): + """Test initialization.""" + assert self.api._token == "test-token" + assert self.api.INDEX_NAME == "v3/global/asset/webproperty" + assert self.api.ASSET_NAME == "webproperty" + + def test_init_with_org_id(self): + """Test initialization with organization ID.""" + assert self.api_with_org._token == "test-token" + assert self.api_with_org.organization_id == "test-org-id" + assert "X-Organization-ID" in self.api_with_org._session.headers + assert self.api_with_org._session.headers["X-Organization-ID"] == "test-org-id" + + def test_view(self): + """Test view method.""" + with patch.object(self.api, "_get") as mock_get: + self.api.view(TEST_WEBPROPERTY) + mock_get.assert_called_with( + f"v3/global/asset/webproperty/{TEST_WEBPROPERTY}", params={} + ) + + def test_view_with_org_id(self): + """Test view method with organization ID.""" + with patch.object(self.api_with_org, "_get") as mock_get: + self.api_with_org.view(TEST_WEBPROPERTY) + mock_get.assert_called_with( + f"v3/global/asset/webproperty/{TEST_WEBPROPERTY}", params={} + ) + # The _get method in the base class will add organization_id to the params + + def test_view_at_time(self): + """Test viewing webproperty details at a specific time.""" + test_date = datetime.datetime(2023, 1, 1) + with patch.object(self.api, "_get") as mock_get: + self.api.view(TEST_WEBPROPERTY, at_time=test_date) + mock_get.assert_called_with( + f"v3/global/asset/webproperty/{TEST_WEBPROPERTY}", + params={"at_time": "2023-01-01T00:00:00Z"}, + ) + + def test_view_at_time_with_org_id(self): + """Test viewing webproperty details at a specific time with organization ID.""" + test_date = datetime.datetime(2023, 1, 1) + with patch.object(self.api_with_org, "_get") as mock_get: + self.api_with_org.view(TEST_WEBPROPERTY, at_time=test_date) + mock_get.assert_called_with( + f"v3/global/asset/webproperty/{TEST_WEBPROPERTY}", + params={"at_time": "2023-01-01T00:00:00Z"}, + ) + # The _get method in the base class will add organization_id to the params + + def test_bulk_view(self): + """Test bulk_view method.""" + with patch.object(self.api, "_get") as mock_get: + self.api.bulk_view(TEST_WEBPROPERTIES) + mock_get.assert_called_with( + "v3/global/asset/webproperty", + params={"webproperty_ids": TEST_WEBPROPERTIES}, + ) + + def test_bulk_view_with_org_id(self): + """Test bulk_view method with organization ID.""" + with patch.object(self.api_with_org, "_get") as mock_get: + self.api_with_org.bulk_view(TEST_WEBPROPERTIES) + mock_get.assert_called_with( + "v3/global/asset/webproperty", + params={"webproperty_ids": TEST_WEBPROPERTIES}, + ) + # The _get method in the base class will add organization_id to the params diff --git a/tests/search/__init__.py b/tests/search/__init__.py new file mode 100644 index 00000000..e69de29b From da4979cd755b655cbe1381fb3509a0f3fbfb135b Mon Sep 17 00:00:00 2001 From: Aidan Holland Date: Tue, 11 Mar 2025 16:34:10 -0400 Subject: [PATCH 2/3] feat: Enhance Platform API support with CLI, pagination, and end-to-end testing This commit adds comprehensive support for the Censys Platform API, including: - CLI commands for platform search, view, and aggregate operations - A new ResultPaginator class for efficient search result pagination - End-to-end tests for Platform API functionality - Updated exception handling and type hints - Example script demonstrating Platform API usage - Configuration for end-to-end testing with pytest --- Taskfile.yml | 64 ++++ censys/asm/api.py | 26 +- censys/cli/commands/platform.py | 242 +++++++++++++- censys/common/exceptions.py | 60 ++-- censys/platform/v3/api.py | 2 +- censys/platform/v3/certificates.py | 2 +- censys/platform/v3/hosts.py | 2 +- censys/platform/v3/search.py | 344 ++++++++++++++++--- censys/platform/v3/webproperties.py | 5 +- censys/search/v1/api.py | 8 +- censys/search/v2/api.py | 8 +- conftest.py | 14 + docs/usage-cli.rst | 97 ++++++ examples/platform_search_demo.py | 370 +++++++++++++++++++++ pyproject.toml | 2 +- tests/asm/test_api.py | 29 ++ tests/cli/test_platform.py | 141 ++++++++ tests/common/test_exceptions_complete.py | 28 ++ tests/platform/test_client.py | 6 +- tests/platform/v3/conftest.py | 32 ++ tests/platform/v3/test_api_error_cases.py | 101 ++++++ tests/platform/v3/test_end_to_end.py | 100 ++++++ tests/platform/v3/test_result_paginator.py | 218 ++++++++++++ tests/platform/v3/test_search.py | 26 +- tests/platform/v3/test_search_complete.py | 41 +++ tests/platform/v3/test_webproperties.py | 4 +- 26 files changed, 1858 insertions(+), 114 deletions(-) create mode 100644 Taskfile.yml create mode 100644 examples/platform_search_demo.py create mode 100644 tests/cli/test_platform.py create mode 100644 tests/common/test_exceptions_complete.py create mode 100644 tests/platform/v3/conftest.py create mode 100644 tests/platform/v3/test_api_error_cases.py create mode 100644 tests/platform/v3/test_end_to_end.py create mode 100644 tests/platform/v3/test_result_paginator.py create mode 100644 tests/platform/v3/test_search_complete.py diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 00000000..478b485d --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,64 @@ +# https://taskfile.dev + +version: "3" + +vars: + PYTHON: python + TEST_ARGS: "" + +tasks: + default: + deps: + - install + silent: true + + install: + desc: Install project dependencies + cmds: + - poetry install + silent: true + + test: + desc: Run all tests + cmds: + - poetry run pytest {{.TEST_ARGS}} + silent: true + + test:e2e: + desc: Run end-to-end tests only + cmds: + - poetry run pytest -v -m e2e --run-e2e {{.TEST_ARGS}} + silent: true + + test:unit: + desc: Run unit tests only (exclude e2e tests) + cmds: + - poetry run pytest -v -m 'not e2e' {{.TEST_ARGS}} + silent: true + + lint: + desc: Run linters (black, flake8, etc) + cmds: + - poetry run black . + - poetry run flake8 + silent: true + + format: + desc: Format code with black + cmds: + - poetry run black . + silent: true + + examples: + desc: Run examples with platform credentials + cmds: + - poetry run python examples/platform_search_demo.py + silent: true + + clean: + desc: Clean build artifacts + cmds: + - rm -rf build/ dist/ *.egg-info + - find . -type d -name __pycache__ -exec rm -rf {} + + - find . -type f -name "*.pyc" -delete + silent: true diff --git a/censys/asm/api.py b/censys/asm/api.py index a47cce67..fc5ddbef 100644 --- a/censys/asm/api.py +++ b/censys/asm/api.py @@ -9,9 +9,10 @@ from censys.common.base import CensysAPIBase from censys.common.config import DEFAULT, get_config from censys.common.exceptions import ( - CensysAsmException, + CensysAPIException, CensysException, CensysExceptionMapper, + CensysInternalServerErrorException, ) @@ -53,10 +54,25 @@ def __init__(self, api_key: Optional[str] = None, **kwargs): def _get_exception_class( # type: ignore self, res: Response - ) -> Type[CensysAsmException]: - return CensysExceptionMapper.ASM_EXCEPTIONS.get( - res.json().get("errorCode"), CensysAsmException - ) + ) -> Type[CensysAPIException]: + try: + # First try to get error code from JSON response + json_data = res.json() + error_code = json_data.get("errorCode") + if error_code: + # ASM has its own error codes, use the ASM mapper + return CensysExceptionMapper._get_exception_class(error_code, "asm") + + # Handle specific HTTP status codes for ASM + if res.status_code == 500: + return CensysInternalServerErrorException + except (ValueError, KeyError): + # If JSON parsing fails, check status code first + if res.status_code == 500: + return CensysInternalServerErrorException + else: # pragma: no cover + # Otherwise use HTTP status code with ASM mapper + return CensysExceptionMapper._get_exception_class(res.status_code, "asm") def _get_page( self, diff --git a/censys/cli/commands/platform.py b/censys/cli/commands/platform.py index 8901883b..9e88c5fa 100644 --- a/censys/cli/commands/platform.py +++ b/censys/cli/commands/platform.py @@ -1,13 +1,16 @@ """Censys Platform CLI commands.""" import argparse +import json import os import sys +from typing import Any, Dict from rich.prompt import Confirm, Prompt -from censys.cli.utils import console +from censys.cli.utils import console, write_file from censys.common.config import DEFAULT, get_config, write_config +from censys.platform import CensysPlatformClient def cli_platform_config(_: argparse.Namespace): # pragma: no cover @@ -105,6 +108,147 @@ def cli_platform_config(_: argparse.Namespace): # pragma: no cover sys.exit(1) +def get_platform_client() -> CensysPlatformClient: + """Get a Censys Platform client. + + Returns: + CensysPlatformClient: The platform client. + """ + config = get_config() + token = os.environ.get("CENSYS_PLATFORM_TOKEN") or config.get( + DEFAULT, "platform_token", fallback=None + ) + org_id = os.environ.get("CENSYS_ORGANIZATION_ID") or config.get( + DEFAULT, "platform_org_id", fallback=None + ) + + if not token: + console.print( + "[red]Error: Censys Platform Token is required.[/red]\n" + "Set the CENSYS_PLATFORM_TOKEN environment variable " + "or configure it using 'censys platform config'" + ) + sys.exit(1) + + if not org_id: + console.print( + "[red]Error: Censys Organization ID is required.[/red]\n" + "Set the CENSYS_ORGANIZATION_ID environment variable " + "or configure it using 'censys platform config'" + ) + sys.exit(1) + + return CensysPlatformClient(token=token, organization_id=org_id) + + +def cli_platform_search(args: argparse.Namespace): # pragma: no cover + """Platform search subcommand. + + Args: + args: Argparse Namespace. + """ + client = get_platform_client() + search = client.search + + # Prepare fields parameter + fields = None + if args.fields: + fields = [field.strip() for field in args.fields.split(",")] + + # Prepare sort parameter + sort = None + if args.sort: + sort = [sort_field.strip() for sort_field in args.sort.split(",")] + + # Use the ResultPaginator + paginator = search.search( + args.query, + page_size=args.page_size, + pages=args.pages, + fields=fields, + sort=sort, + ) + + # Structure to hold the results + all_results: Dict[str, Any] = {"result": {}} + + # If only requesting one page, use get_page for efficiency + if args.pages == 1: + # Just get the first page without pagination machinery + hits = paginator.get_page() + # Query once to get the total + paginator.get_page() + all_results["result"] = { + "hits": hits, + "total": paginator.total, + } + else: + # Get all requested pages + all_hits = paginator.get_all_results() + all_results["result"] = { + "hits": all_hits, + "total": paginator.total, + } + + # Output results + if args.output: + write_file(all_results, args.output) + console.print(f"[green]Search results written to {args.output}[/green]") + else: + console.print(json.dumps(all_results, indent=2)) + + +def cli_platform_view(args: argparse.Namespace): # pragma: no cover + """Platform view subcommand. + + Args: + args: Argparse Namespace. + """ + client = get_platform_client() + + # Determine the resource type and call the appropriate API + if args.resource_type == "host": + result = client.hosts.view(args.resource_id) + elif args.resource_type == "certificate": + result = client.certificates.view(args.resource_id) + elif args.resource_type == "webproperty": + result = client.webproperties.view(args.resource_id) + else: + console.print(f"[red]Unknown resource type: {args.resource_type}[/red]") + sys.exit(1) + + # Output results + if args.output: + write_file(result, args.output) + console.print(f"[green]View results written to {args.output}[/green]") + else: + console.print(json.dumps(result, indent=2)) + + +def cli_platform_aggregate(args: argparse.Namespace): # pragma: no cover + """Platform aggregate subcommand. + + Args: + args: Argparse Namespace. + """ + client = get_platform_client() + search = client.search + + # Execute aggregate query + results = search.aggregate( + args.query, + args.field, + number_of_buckets=args.num_buckets, + ) + + # Output results + if args.output: + write_file(results, args.output) + console.print(f"[green]Aggregate results written to {args.output}[/green]") + else: + console.print(json.dumps(results, indent=2)) + + def include(parent_parser: argparse._SubParsersAction, parents: dict): """Include this subcommand into the parent parser. @@ -126,3 +270,99 @@ def include(parent_parser: argparse._SubParsersAction, parents: dict): help="configure Censys Platform API settings", ) config_parser.set_defaults(func=cli_platform_config) + + # Platform search command + search_parser = platform_subparsers.add_parser( + "search", + description="Search the Censys Platform using a query string", + help="search the Censys Platform", + ) + search_parser.add_argument( + "query", + type=str, + help="a string written in Censys Search syntax", + ) + search_parser.add_argument( + "--page-size", + type=int, + default=100, + help="number of results per page (default: 100)", + ) + search_parser.add_argument( + "--pages", + type=int, + default=-1, + help="number of pages to fetch (-1 for all pages)", + ) + search_parser.add_argument( + "--fields", + type=str, + help="comma-separated list of fields to return", + ) + search_parser.add_argument( + "--sort", + type=str, + help="comma-separated list of fields to sort on", + ) + search_parser.add_argument( + "-o", + "--output", + type=str, + help="json output file path", + ) + search_parser.set_defaults(func=cli_platform_search) + + # Platform view command + view_parser = platform_subparsers.add_parser( + "view", + description="View details of a specific resource in the Censys Platform", + help="view resource details", + ) + view_parser.add_argument( + "resource_type", + type=str, + choices=["host", "certificate", "webproperty"], + help="type of resource to view", + ) + view_parser.add_argument( + "resource_id", + type=str, + help="ID of the resource to view (e.g. IP address, certificate hash, domain name)", + ) + view_parser.add_argument( + "-o", + "--output", + type=str, + help="json output file path", + ) + view_parser.set_defaults(func=cli_platform_view) + + # Platform aggregate command + aggregate_parser = platform_subparsers.add_parser( + "aggregate", + description="Aggregate data from the Censys Platform", + help="aggregate platform data", + ) + aggregate_parser.add_argument( + "query", + type=str, + help="a string written in Censys Search syntax", + ) + aggregate_parser.add_argument( + "field", + type=str, + help="field to aggregate on", + ) + aggregate_parser.add_argument( + "--num-buckets", + type=int, + default=50, + help="number of buckets to return (default: 50)", + ) + aggregate_parser.add_argument( + "-o", + "--output", + type=str, + help="json output file path", + ) + aggregate_parser.set_defaults(func=cli_platform_aggregate) diff --git a/censys/common/exceptions.py b/censys/common/exceptions.py index 6cabb3bf..ccf28e80 100644 --- a/censys/common/exceptions.py +++ b/censys/common/exceptions.py @@ -41,12 +41,8 @@ def __init__( self.details = details super().__init__(self.message) - -class CensysSearchException(CensysAPIException): - """Base Exception for the Censys search API.""" - def __repr__(self) -> str: - """Representation of CensysSearchException. + """Representation of CensysAPIException. Returns: str: Printable representation. @@ -56,18 +52,12 @@ def __repr__(self) -> str: __str__ = __repr__ -class CensysPlatformException(CensysAPIException): - """Base Exception for the Censys Platform API.""" - - def __repr__(self) -> str: - """Representation of CensysPlatformException. +class CensysSearchException(CensysAPIException): + """Base Exception for the Censys search API.""" - Returns: - str: Printable representation. - """ - return f"{self.status_code} ({self.const}): {self.message or self.body}" - __str__ = __repr__ +class CensysPlatformException(CensysAPIException): + """Base Exception for the Censys Platform API.""" class CensysAsmException(CensysAPIException): @@ -84,30 +74,28 @@ def __repr__(self) -> str: f"{self.message}. {self.details}" ) - __str__ = __repr__ - class CensysMissingApiKeyException(CensysAsmException): """Exception raised when there is no provided ASM API key.""" -class CensysRateLimitExceededException(CensysSearchException): +class CensysRateLimitExceededException(CensysAPIException): """Exception raised when your Censys rate limit has been exceeded.""" -class CensysNotFoundException(CensysSearchException): +class CensysNotFoundException(CensysAPIException): """Exception raised when the resource requested is not found.""" -class CensysUnauthorizedException(CensysSearchException): +class CensysUnauthorizedException(CensysAPIException): """Exception raised when you doesn't have access to the requested resource.""" -class CensysJSONDecodeException(CensysSearchException): +class CensysJSONDecodeException(CensysAPIException): """Exception raised when the resource requested is not valid JSON.""" -class CensysInternalServerException(CensysSearchException): +class CensysInternalServerException(CensysAPIException): """Exception raised when the server encountered an internal error.""" @@ -369,7 +357,7 @@ class CensysExceptionMapper: } """Map of status code to ASM Exception.""" - SEARCH_EXCEPTIONS: Dict[int, Type[CensysSearchException]] = { + SEARCH_EXCEPTIONS: Dict[int, Type[CensysAPIException]] = { 401: CensysUnauthorizedException, 403: CensysUnauthorizedException, 404: CensysNotFoundException, @@ -378,7 +366,7 @@ class CensysExceptionMapper: } """Map of status code to Search Exception.""" - PLATFORM_EXCEPTIONS: Dict[int, Type[CensysPlatformException]] = { + PLATFORM_EXCEPTIONS: Dict[int, Type[CensysAPIException]] = { 401: CensysUnauthorizedException, 403: CensysUnauthorizedException, 404: CensysNotFoundException, @@ -388,17 +376,31 @@ class CensysExceptionMapper: """Map of status code to Platform Exception.""" @staticmethod - def exception_for_status_code(status_code: int) -> Type[CensysAPIException]: - """Return the appropriate exception class for the given status code. + def _get_exception_class( + status_code: int, api_type: str + ) -> Type[CensysAPIException]: + """Return the appropriate exception class for the given status code and API type. Args: - status_code (int): HTTP status code. + status_code (int): HTTP status code or error code. + api_type (str): The API type ('platform', 'search', or 'asm'). Returns: Type[CensysAPIException]: The exception class to raise. """ - if status_code in CensysExceptionMapper.PLATFORM_EXCEPTIONS: + if ( + api_type.lower() == "platform" + and status_code in CensysExceptionMapper.PLATFORM_EXCEPTIONS + ): return CensysExceptionMapper.PLATFORM_EXCEPTIONS[status_code] - if status_code in CensysExceptionMapper.SEARCH_EXCEPTIONS: + elif ( + api_type.lower() == "search" + and status_code in CensysExceptionMapper.SEARCH_EXCEPTIONS + ): return CensysExceptionMapper.SEARCH_EXCEPTIONS[status_code] + elif ( + api_type.lower() == "asm" + and status_code in CensysExceptionMapper.ASM_EXCEPTIONS + ): + return CensysExceptionMapper.ASM_EXCEPTIONS[status_code] return CensysAPIException diff --git a/censys/platform/v3/api.py b/censys/platform/v3/api.py index 35964919..b935b57e 100644 --- a/censys/platform/v3/api.py +++ b/censys/platform/v3/api.py @@ -104,7 +104,7 @@ def _get_exception_class( # type: ignore Returns: Type[CensysPlatformException]: The exception class to raise. """ - return CensysExceptionMapper.exception_for_status_code(res.status_code) # type: ignore + return CensysExceptionMapper._get_exception_class(res.status_code, "platform") # type: ignore def _get(self, endpoint: str, args: Optional[dict] = None, **kwargs: Any) -> dict: """Get data from a REST API endpoint. diff --git a/censys/platform/v3/certificates.py b/censys/platform/v3/certificates.py index fc0a26d0..1b5be7b4 100644 --- a/censys/platform/v3/certificates.py +++ b/censys/platform/v3/certificates.py @@ -66,7 +66,7 @@ def __init__( self, token=token, organization_id=organization_id, **kwargs ) - def view(self, certificate_id: str, **kwargs: Any) -> Dict[str, Any]: + def view(self, certificate_id: str, **kwargs: Any) -> Dict[str, Any]: # type: ignore[override] """Get a certificate by ID. Args: diff --git a/censys/platform/v3/hosts.py b/censys/platform/v3/hosts.py index ff5a7406..db9d092e 100644 --- a/censys/platform/v3/hosts.py +++ b/censys/platform/v3/hosts.py @@ -80,7 +80,7 @@ def __init__( self, token=token, organization_id=organization_id, **kwargs ) - def view( + def view( # type: ignore[override] self, host_id: str, at_time: Optional[datetime] = None, **kwargs: Any ) -> Dict[str, Any]: """Get a host by ID. diff --git a/censys/platform/v3/search.py b/censys/platform/v3/search.py index 53d72043..c158f10b 100644 --- a/censys/platform/v3/search.py +++ b/censys/platform/v3/search.py @@ -1,51 +1,262 @@ """Interact with the Censys Platform Search API.""" -from typing import Any, Dict, List, Optional, Union +import warnings +from typing import Any, Dict, Iterable, Iterator, List, Optional, Union from .api import CensysPlatformAPIv3 +class ResultPaginator(Iterable): + """Iterator for search results that handles pagination automatically. + + This class provides a convenient way to iterate through all pages of search results + without manually handling cursors and pagination. + + Examples: + Basic usage - iterate through pages: + + >>> from censys.platform import CensysSearch + >>> search = CensysSearch() + >>> paginator = search.search("services.port:443") + >>> for page in paginator: + ... for hit in page: + ... print(hit.get("ip")) + + Get a single page without pagination: + + >>> paginator = search.search("services.port:443") + >>> first_page = paginator.get_page() + >>> for hit in first_page: + ... print(hit.get("ip")) + + Get a specific page by number: + + >>> third_page = paginator.get_page(page_number=3) + + Get all results in a single list: + + >>> all_results = paginator.get_all_results() + >>> print(f"Found {len(all_results)} results") + >>> print(f"Total available: {paginator.total}") + + Limit pages and specify fields: + + >>> paginator = search.search( + ... "services.port:443", + ... pages=2, + ... fields=["ip", "services.port"] + ... ) + """ + + # Total number of results (Set after first query) + total: Optional[int] = None + + def __init__( + self, + api: "CensysSearch", + query: str, + page_size: int = 100, + cursor: Optional[str] = None, + pages: int = -1, + fields: Optional[List[str]] = None, + sort: Optional[Union[str, List[str]]] = None, + **kwargs: Any, + ): + """Initialize ResultPaginator. + + Args: + api: Parent Search API object. + query: The query to be executed. + page_size: Number of results per page. Defaults to 100. + cursor: Optional cursor for pagination. + pages: Number of pages to return. Defaults to -1 (all pages). + fields: Optional fields to return. + sort: Optional field(s) to sort on. + **kwargs: Optional keyword args. + """ + self.api = api + self.query = query + self.page_size = page_size + self.cursor = cursor + self.next_cursor: Optional[str] = None + self.page = 0 + + # If pages is <= 0, get all pages + if pages <= 0: + self.pages = float("inf") + else: + self.pages = pages + + self.fields = fields + self.sort = sort + self.extra_args = kwargs + + def __call__(self, page_size: Optional[int] = None) -> List[dict]: + """Execute search and return the next page of results. + + Args: + page_size: Override the number of results per page. + + Raises: + StopIteration: When all requested pages have been received. + + Returns: + List[dict]: One page of result hits. + """ + if self.page > self.pages: + raise StopIteration + + # Use the page_size parameter from the method call or the instance variable + page_size = page_size if page_size is not None else self.page_size + + results = self.api.query( + query=self.query, + page_size=page_size, # Pass page_size to the query method + cursor=self.next_cursor or self.cursor, + fields=self.fields, + sort=self.sort, + **self.extra_args, + ) + + self.page += 1 + result = results.get("result", {}) + self.total = result.get("total", 0) + self.next_cursor = result.get("cursor") + + # If there are no more results or no cursor for next page + if self.total == 0 or not self.next_cursor: + self.pages = 0 + + return result.get("hits", []) + + def __next__(self) -> List[dict]: + """Get the next page of search results. + + Returns: + List[dict]: One page of result hits. + """ + return self.__call__() + + def __iter__(self) -> Iterator[List[dict]]: + """Get iterator. + + Returns: + Iterator: Returns self. + """ + return self + + def get_page( + self, page_number: int = 1, cursor: Optional[str] = None + ) -> List[dict]: + """Get a specific page of results without iterating. + + This method fetches a single page of results either by page number or cursor, + without engaging the full pagination process. + + Args: + page_number: The page number to retrieve (1-indexed). Defaults to 1. + cursor: Optional cursor string to fetch a specific page directly. + If provided, page_number is ignored. + + Returns: + List[dict]: The requested page of results. + """ + # Save current state to restore later + original_page = self.page + original_cursor = self.next_cursor + + try: + if cursor: + # Use the provided cursor directly + self.next_cursor = cursor + self.page = 1 # Reset page counter + return self.__call__() + else: + # Start from the beginning + self.next_cursor = None + self.page = 1 + + # Skip pages until we reach the desired page + for _ in range(page_number - 1): + # Warn once if skipping pages (since this uses quota) + if page_number > 1 and _ == 0: + warnings.warn( + "Warning: Skipping to a specific page still uses API quota for each page skipped. " + "Consider using get_all_results() if you need data from multiple pages." + ) + + self.__call__() + + # Return the requested page + return self.__call__() + finally: + # Restore original state + self.page = original_page + self.next_cursor = original_cursor + + def get_all_results(self) -> List[dict]: + """Collect all results into a single list. + + This method will iterate through all pages and combine the results. + + Returns: + List[dict]: All results combined into a single list. + """ + all_results = [] + + # Save the current state to restore it after + original_page = self.page + original_cursor = self.next_cursor + + # Reset the paginator + self.page = 1 + self.next_cursor = None + + try: + for page in self: + all_results.extend(page) + finally: + # Restore the original state if needed (for reuse) + if original_page != 1: + self.page = original_page + self.next_cursor = original_cursor + + return all_results + + class CensysSearch(CensysPlatformAPIv3): """Interacts with the Censys Platform Search API. Examples: - Inits Censys Platform Search. + Initialize the search client: >>> from censys.platform import CensysSearch >>> s = CensysSearch() - Search for hosts. - - >>> s.query("services.port:443") - { - "result": { - "hits": [ - { - "host": { - "ip": "1.1.1.1", - ... - } - }, - ... - ], - "total": 123456 - } - } - - Aggregate host data. - - >>> s.aggregate("services.port:443", "services.service_name") - { - "result": { - "buckets": [ - { - "key": "HTTP", - "count": 12345 - }, - ... - ] - } - } + Simple query - get raw API response: + + >>> results = s.query("services.port:443") + >>> print(f"Found {results['result']['total']} results") + >>> for hit in results['result']['hits']: + ... print(hit.get('ip')) + + Using the ResultPaginator for easier iteration: + + >>> paginator = s.search("services.port:443") + >>> # Get just the first page + >>> first_page = paginator.get_page() + >>> # Iterate through all pages + >>> for page in paginator: + ... for hit in page: + ... print(hit.get('ip')) + >>> # Get all results at once + >>> all_results = paginator.get_all_results() + + Aggregation example: + + >>> agg_results = s.aggregate("services.port:443", "services.service_name") + >>> for bucket in agg_results['result']['buckets']: + ... print(f"{bucket['key']}: {bucket['count']}") """ INDEX_NAME = "v3/global/search" @@ -54,7 +265,7 @@ def __init__(self, token: Optional[str] = None, **kwargs): """Inits CensysSearch. Args: - token (str, optional): Personal Access Token for Censys Platform API. + token: Personal Access Token for Censys Platform API. **kwargs: Optional kwargs. """ CensysPlatformAPIv3.__init__(self, token=token, **kwargs) @@ -62,7 +273,7 @@ def __init__(self, token: Optional[str] = None, **kwargs): def query( self, query: str, - per_page: int = 100, + page_size: int = 100, cursor: Optional[str] = None, fields: Optional[List[str]] = None, sort: Optional[Union[str, List[str]]] = None, @@ -71,17 +282,17 @@ def query( """Search the global data. Args: - query (str): The query to search for. - per_page (int): Number of results per page. Defaults to 100. - cursor (str, optional): Cursor for pagination. - fields (List[str], optional): Fields to return. - sort (Union[str, List[str]], optional): Field(s) to sort on. + query: The query to search for. + page_size: Number of results per page. Defaults to 100. + cursor: Optional cursor for pagination. + fields: Optional fields to return. + sort: Optional field(s) to sort on. **kwargs: Optional keyword args. Returns: Dict[str, Any]: The search result. """ - data = {"q": query, "per_page": per_page} + data = {"query": query, "page_size": page_size} if cursor: data["cursor"] = cursor if fields: @@ -91,24 +302,61 @@ def query( return self._post(f"{self.INDEX_NAME}/query", data=data, **kwargs) + def search( + self, + query: str, + page_size: int = 100, + cursor: Optional[str] = None, + pages: int = -1, + fields: Optional[List[str]] = None, + sort: Optional[Union[str, List[str]]] = None, + **kwargs: Any, + ) -> ResultPaginator: + """Search the global data and return a paginator for easy iteration. + + This method returns a ResultPaginator that handles pagination automatically. + + Args: + query: The query to search for. + page_size: Number of results per page. Defaults to 100. + cursor: Optional cursor for pagination. + pages: Number of pages to return. Defaults to -1 (all pages). + fields: Optional fields to return. + sort: Optional field(s) to sort on. + **kwargs: Optional keyword args. + + Returns: + ResultPaginator: Paginator for iterating through search results. + """ + return ResultPaginator( + self, + query, + page_size=page_size, + cursor=cursor, + pages=pages, + fields=fields, + sort=sort, + **kwargs, + ) + def aggregate( self, query: str, field: str, - num_buckets: int = 50, + number_of_buckets: int = 50, **kwargs: Any, ) -> Dict[str, Any]: - """Aggregate the global data. + """Aggregate search results by a field. Args: - query (str): The query to search for. - field (str): The field to aggregate on. - num_buckets (int): Number of buckets to return. Defaults to 50. + query: The query string. + field: The field to aggregate on. + number_of_buckets: Number of buckets to return. Defaults to 50. **kwargs: Optional keyword args. Returns: Dict[str, Any]: The aggregation result. """ - data = {"q": query, "field": field, "num_buckets": num_buckets} + data = {"query": query, "field": field, "number_of_buckets": number_of_buckets} return self._post(f"{self.INDEX_NAME}/aggregate", data=data, **kwargs) diff --git a/censys/platform/v3/webproperties.py b/censys/platform/v3/webproperties.py index dcd89c6c..14cd8be4 100644 --- a/censys/platform/v3/webproperties.py +++ b/censys/platform/v3/webproperties.py @@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional from .api import CensysPlatformAPIv3 +from censys.common.utils import format_rfc3339 class CensysWebProperties(CensysPlatformAPIv3): @@ -67,7 +68,7 @@ def __init__( self, token=token, organization_id=organization_id, **kwargs ) - def view( + def view( # type: ignore[override] self, webproperty_id: str, at_time: Optional[datetime] = None, **kwargs: Any ) -> Dict[str, Any]: """Get a webproperty by ID. @@ -82,7 +83,7 @@ def view( """ params = {} if at_time: - params["at_time"] = at_time.isoformat() + "Z" + params["at_time"] = format_rfc3339(at_time) return self._get(f"{self.INDEX_NAME}/{webproperty_id}", params=params, **kwargs) diff --git a/censys/search/v1/api.py b/censys/search/v1/api.py index 5e433ed5..7c41144a 100644 --- a/censys/search/v1/api.py +++ b/censys/search/v1/api.py @@ -8,9 +8,9 @@ from censys.common.base import CensysAPIBase from censys.common.config import DEFAULT, get_config from censys.common.exceptions import ( + CensysAPIException, CensysException, CensysExceptionMapper, - CensysSearchException, ) Fields = Optional[List[str]] @@ -68,10 +68,8 @@ def __init__( def _get_exception_class( # type: ignore self, res: Response - ) -> Type[CensysSearchException]: - return CensysExceptionMapper.SEARCH_EXCEPTIONS.get( - res.status_code, CensysSearchException - ) + ) -> Type[CensysAPIException]: + return CensysExceptionMapper._get_exception_class(res.status_code, "search") def account(self) -> dict: """Gets the current account information. diff --git a/censys/search/v2/api.py b/censys/search/v2/api.py index 1edc2ddb..0648c3b4 100644 --- a/censys/search/v2/api.py +++ b/censys/search/v2/api.py @@ -9,9 +9,9 @@ from censys.common.base import CensysAPIBase from censys.common.config import DEFAULT, get_config from censys.common.exceptions import ( + CensysAPIException, CensysException, CensysExceptionMapper, - CensysSearchException, ) INDEX_TO_KEY = {"hosts": "ip", "certificates": "fingerprint_sha256"} @@ -72,10 +72,8 @@ def __init__( def _get_exception_class( # type: ignore self, res: Response - ) -> Type[CensysSearchException]: - return CensysExceptionMapper.SEARCH_EXCEPTIONS.get( - res.status_code, CensysSearchException - ) + ) -> Type[CensysAPIException]: + return CensysExceptionMapper._get_exception_class(res.status_code, "search") def account(self) -> dict: """Gets the current account's query quota. diff --git a/conftest.py b/conftest.py index beaf56bc..b16f1619 100644 --- a/conftest.py +++ b/conftest.py @@ -12,3 +12,17 @@ def _mock_settings_env_vars(): with patch.dict("os.environ", {"CENSYS_ASM_API_KEY": "testing"}): yield + + +def pytest_addoption(parser): + """Add command-line options to pytest. + + Args: + parser: The pytest command-line parser + """ + parser.addoption( + "--run-e2e", + action="store_true", + default=False, + help="Run end-to-end tests that make real API calls", + ) diff --git a/docs/usage-cli.rst b/docs/usage-cli.rst index 8c165b24..fa049025 100644 --- a/docs/usage-cli.rst +++ b/docs/usage-cli.rst @@ -41,6 +41,103 @@ Optionally, you can enable tab completion for the CLI by adding this line to you Please note that autocomplete is supported for field names in the `search` command. +``platform`` +------------ + +The Platform API provides a unified interface to access all Censys data sources. The following commands allow you to interact with the Censys Platform API directly from the CLI. + +``platform search`` +^^^^^^^^^^^^^^^^^ + +The ``platform search`` command allows you to search across all Censys data sources. + +Basic search query: + +.. prompt:: bash + + censys platform search 'host.services.port:443' + +Limiting fields and results: + +.. prompt:: bash + + censys platform search 'host.services.port:443' --fields ip,name,services.port --page-size 10 + +Retrieving a specific number of pages of results: + +.. prompt:: bash + + censys platform search 'host.services.port:443' --pages 2 + +Getting all pages of results (default behavior): + +.. prompt:: bash + + censys platform search 'host.services.port:443' --pages -1 + +Sorting results: + +.. prompt:: bash + + censys platform search 'host.services.port:443' --sort ip + +Saving results to a file: + +.. prompt:: bash + + censys platform search 'host.services.port:443' --output results.json + +``platform view`` +^^^^^^^^^^^^^^^^ + +The ``platform view`` command allows you to view details for a specific resource. + +Viewing a host: + +.. prompt:: bash + + censys platform view host 8.8.8.8 + +Viewing a certificate: + +.. prompt:: bash + + censys platform view certificate fb444eb8e68437bae06232b9f5091bccff62a768ca09e92eb5c9c2cf9d17c426 + +Viewing a web property: + +.. prompt:: bash + + censys platform view webproperty example.com + +Saving results to a file: + +.. prompt:: bash + + censys platform view host 8.8.8.8 --output host.json + +``platform aggregate`` +^^^^^^^^^^^^^^^^^^^^ + +The ``platform aggregate`` command allows you to perform data aggregations. + +Basic aggregation: + +.. prompt:: bash + + censys platform aggregate 'host.services.port:443' 'host.services.protocol' + +Changing the number of buckets: + +.. prompt:: bash + + censys platform aggregate 'host.services.port:443' 'host.services.protocol' --num-buckets 20 + +Saving results to a file: + +.. prompt:: bash + + censys platform aggregate 'host.services.port:443' 'host.services.protocol' --output aggregate.json ``search`` ---------- diff --git a/examples/platform_search_demo.py b/examples/platform_search_demo.py new file mode 100644 index 00000000..73089cd1 --- /dev/null +++ b/examples/platform_search_demo.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python +"""Demo script showing how to use the Censys Platform Search API with ResultPaginator. + +This script demonstrates various ways to use the ResultPaginator class +to efficiently work with paginated search results. + +Usage: + python platform_search_demo.py + +Environment variables: + CENSYS_PLATFORM_TOKEN: Your Censys Platform API token + CENSYS_ORGANIZATION_ID: Your Censys Organization ID + +To set up these variables, you can: +1. Export them in your shell: + export CENSYS_PLATFORM_TOKEN=your_token + export CENSYS_ORGANIZATION_ID=your_org_id + +2. Or use the CLI config command: + censys platform config +""" + +import json +import os +import sys +from typing import Dict, List, Optional, Tuple + +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +from censys.common.config import DEFAULT, get_config +from censys.platform import CensysSearch + + +def get_credentials() -> Tuple[Optional[str], Optional[str]]: + """Get credentials from environment variables or config file. + + Returns: + Tuple[Optional[str], Optional[str]]: API token and organization ID + """ + # Try environment variables first + token = os.environ.get("CENSYS_PLATFORM_TOKEN") + org_id = os.environ.get("CENSYS_ORGANIZATION_ID") + + # If both are set, return them + if token and org_id: + return token, org_id + + # Try config file + try: + config = get_config() + if not token: + token = config.get(DEFAULT, "platform_token", fallback=None) + if not org_id: + org_id = config.get(DEFAULT, "platform_org_id", fallback=None) + except Exception: + # If there's an error reading config, just continue with None values + pass + + return token, org_id + + +def print_result_summary(results: List[Dict], total: int, query: str): + """Print a summary of the search results. + + Args: + results: List of search results + total: Total number of matching results + query: The search query + """ + console = Console() + console.print(f"[bold]Query:[/bold] {query}") + console.print( + f"[bold]Results:[/bold] Showing {len(results)} of {total} total matches" + ) + + +def display_results_table(results: List[Dict], fields: List[str] = None): + """Display results in a table format. + + Args: + results: List of search results + fields: Optional list of fields to display. If None, will try to detect common fields. + """ + if not results: + Console().print("[yellow]No results to display.[/yellow]") + return + + # If no fields specified, try to detect common fields + if not fields: + # Look at the first result to determine available fields + first_result = results[0] + if "ip" in first_result: + fields = ["ip", "autonomous_system.name"] + if "services" in first_result: + fields.append("services.port") + elif "domain" in first_result: + fields = ["domain", "alexa_rank"] + else: + # Use all top-level keys + fields = list(first_result.keys()) + + # Create table + table = Table(title="Search Results") + + # Add columns + for field in fields: + # Use the field name as column title, prettified + column_name = field.split(".")[-1].replace("_", " ").title() + table.add_column(column_name, style="cyan") + + # Add rows + for result in results: + row = [] + for field in fields: + # Handle nested fields (e.g., "autonomous_system.name") + parts = field.split(".") + value = result + for part in parts: + if isinstance(value, dict) and part in value: + value = value[part] + elif ( + isinstance(value, list) + and part == "port" + and parts[0] == "services" + ): + # Handle services.port special case - show all ports as comma-separated list + ports = [ + str(service.get("port", "")) + for service in result.get("services", []) + if "port" in service + ] + value = ", ".join(ports) + break + else: + value = "" + break + + # Format the value + if isinstance(value, list): + value = ", ".join(str(v) for v in value) + elif isinstance(value, dict): + value = json.dumps(value, indent=2) + + row.append(str(value)) + + table.add_row(*row) + + # Print table + console = Console() + console.print(table) + + +def basic_search_demo(): + """Basic search demo using the Censys Platform API.""" + console = Console() + console.print("\n[bold blue]===== Basic Search Demo =====[/bold blue]") + + # Get credentials + token, org_id = get_credentials() + + if not token or not org_id: + console.print( + "[red]Error: Missing credentials.[/red]\n" + "Please set CENSYS_PLATFORM_TOKEN and CENSYS_ORGANIZATION_ID environment variables.\n" + "You can also configure them using: censys platform config" + ) + return + + # Create client + search = CensysSearch(token=token, organization_id=org_id) + + # Simple search query + query = "services.port:443 and services.service_name:HTTPS" + console.print(f"Executing search: [cyan]{query}[/cyan]") + + # Method 1: Direct API query + console.print("\n[bold]Method 1:[/bold] Direct API Query") + results = search.query(query, per_page=5) + hits = results["result"]["hits"] + total = results["result"]["total"] + + print_result_summary(hits, total, query) + display_results_table(hits, ["ip", "autonomous_system.name", "services.port"]) + + # Method 2: Using get_page() for a single page + console.print("\n[bold]Method 2:[/bold] Using get_page() for a single page") + paginator = search.search(query, per_page=5) + page = paginator.get_page() + + print_result_summary(page, paginator.total, query) + display_results_table(page, ["ip", "autonomous_system.name", "services.port"]) + + # Method 3: Using get_all_results() for all pages (limited to 2 for demo) + console.print( + "\n[bold]Method 3:[/bold] Using get_all_results() (limited to 2 pages)" + ) + paginator = search.search(query, per_page=5, pages=2) + all_results = paginator.get_all_results() + + print_result_summary(all_results, paginator.total, query) + display_results_table( + all_results, ["ip", "autonomous_system.name", "services.port"] + ) + + # Method 4: Manual iteration through pages + console.print( + "\n[bold]Method 4:[/bold] Manual iteration through pages (limited to 2 pages)" + ) + paginator = search.search(query, per_page=5, pages=2) + + console.print(f"[bold]Query:[/bold] {query}") + console.print(f"[bold]Total results:[/bold] {paginator.total}") + + for i, page in enumerate(paginator, 1): + console.print(f"\n[bold]Page {i}[/bold] ({len(page)} results)") + display_results_table(page, ["ip", "autonomous_system.name", "services.port"]) + + +def search_with_fields_demo(): + """Demo showing search with specific fields.""" + console = Console() + console.print("\n[bold blue]===== Search with Fields Demo =====[/bold blue]") + + # Get credentials + token, org_id = get_credentials() + + if not token or not org_id: + console.print("[red]Error: Missing credentials.[/red]") + return + + # Create client + search = CensysSearch(token=token, organization_id=org_id) + + # Search for HTTPS servers, but only return specific fields + query = "services.port:443 and services.service_name:HTTPS" + fields = ["ip", "autonomous_system.name", "autonomous_system.asn", "services.port"] + + console.print(f"Executing search with specific fields: [cyan]{query}[/cyan]") + console.print(f"Fields: {', '.join(fields)}") + + paginator = search.search(query, per_page=5, fields=fields, pages=1) + page = paginator.get_page() + + print_result_summary(page, paginator.total, query) + display_results_table(page, fields) + + +def search_with_sorting_demo(): + """Demo showing search with sorting.""" + console = Console() + console.print("\n[bold blue]===== Search with Sorting Demo =====[/bold blue]") + + # Get credentials + token, org_id = get_credentials() + + if not token or not org_id: + console.print("[red]Error: Missing credentials.[/red]") + return + + # Create client + search = CensysSearch(token=token, organization_id=org_id) + + # Search for HTTPS servers, sort by ASN + query = "services.port:443 and services.service_name:HTTPS" + fields = ["ip", "autonomous_system.name", "autonomous_system.asn", "services.port"] + sort = "autonomous_system.asn" + + console.print(f"Executing search with sorting: [cyan]{query}[/cyan]") + console.print(f"Sort by: {sort}") + + paginator = search.search(query, per_page=5, fields=fields, sort=sort, pages=1) + page = paginator.get_page() + + print_result_summary(page, paginator.total, query) + display_results_table(page, fields) + + +def aggregate_demo(): + """Demo showing aggregation functionality.""" + console = Console() + console.print("\n[bold blue]===== Aggregation Demo =====[/bold blue]") + + # Get credentials + token, org_id = get_credentials() + + if not token or not org_id: + console.print("[red]Error: Missing credentials.[/red]") + return + + # Create client + search = CensysSearch(token=token, organization_id=org_id) + + # Aggregate by service name + query = "services.port:443" + field = "services.service_name" + + console.print(f"Executing aggregation: [cyan]{query}[/cyan]") + console.print(f"Group by: {field}") + + result = search.aggregate(query, field, num_buckets=10) + buckets = result["result"]["buckets"] + + # Create table + table = Table( + title=f"Top services on port 443 (Total: {result['result']['total']})" + ) + table.add_column("Service Name", style="cyan") + table.add_column("Count", style="green") + table.add_column("Percentage", style="yellow") + + for bucket in buckets: + percentage = (bucket["count"] / result["result"]["total"]) * 100 + table.add_row(bucket["key"], str(bucket["count"]), f"{percentage:.2f}%") + + console.print(table) + + +def check_credentials(): + """Check if credentials are configured and provide guidance if not. + + Returns: + bool: True if credentials are configured, False otherwise + """ + console = Console() + token, org_id = get_credentials() + + if token and org_id: + return True + + console.print( + Panel( + "[bold red]Error: Censys Platform credentials not found![/bold red]\n\n" + "To use this demo, you need to configure your Censys Platform credentials.\n\n" + "[bold]Option 1:[/bold] Set environment variables:\n" + " export CENSYS_PLATFORM_TOKEN=your_token\n" + " export CENSYS_ORGANIZATION_ID=your_organization_id\n\n" + "[bold]Option 2:[/bold] Use the Censys CLI to configure:\n" + " censys platform config\n\n" + "You can get your credentials from the Censys Platform web interface at:\n" + "https://app.censys.io/account/api", + title="Configuration Required", + border_style="red", + ) + ) + + return False + + +if __name__ == "__main__": + console = Console() + console.print("[bold green]Censys Platform Search API Demo[/bold green]") + console.print( + "This demo shows different ways to use the ResultPaginator class " + "for efficient pagination of search results." + ) + + # Check credentials first + if not check_credentials(): + sys.exit(1) + + try: + basic_search_demo() + search_with_fields_demo() + search_with_sorting_demo() + aggregate_demo() + except Exception as e: + console.print(f"[red]Error:[/red] {str(e)}") + sys.exit(1) diff --git a/pyproject.toml b/pyproject.toml index e702a674..2c505974 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "censys" -version = "2.2.16" +version = "2.3.0b1" description = "An easy-to-use and lightweight API wrapper for Censys APIs (censys.io)." authors = ["Censys, Inc. "] license = "Apache-2.0" diff --git a/tests/asm/test_api.py b/tests/asm/test_api.py index 1459e20b..5d3f26ff 100644 --- a/tests/asm/test_api.py +++ b/tests/asm/test_api.py @@ -13,6 +13,7 @@ CensysAsmException, CensysException, CensysExceptionMapper, + CensysInternalServerErrorException, ) @@ -66,6 +67,34 @@ def test_get_exception_class(self, status_code, exception): # Actual call/assertion assert CensysAsmAPI()._get_exception_class(response) == exception + def test_get_exception_class_http_500(self): + """Test that HTTP 500 errors are mapped to CensysInternalServerErrorException.""" + # Create a response with a 500 status code + response = Response() + response.status_code = 500 + + # Mock the json method to raise a ValueError + # This simulates a non-JSON response or parsing error + mock_response = self.mocker.patch.object( + response, "json", side_effect=ValueError + ) + + # Actual call/assertion + assert ( + CensysAsmAPI()._get_exception_class(response) + == CensysInternalServerErrorException + ) + + # Test with a valid JSON response but no error code + mock_response.side_effect = None + mock_response.return_value = {"message": "Internal Server Error"} + + # Actual call/assertion + assert ( + CensysAsmAPI()._get_exception_class(response) + == CensysInternalServerErrorException + ) + def test_exception_repr(self): # Actual call exception = CensysAsmException( diff --git a/tests/cli/test_platform.py b/tests/cli/test_platform.py new file mode 100644 index 00000000..eed7d7e7 --- /dev/null +++ b/tests/cli/test_platform.py @@ -0,0 +1,141 @@ +"""Tests for CLI platform commands.""" + +import os +from unittest.mock import MagicMock, patch + +from censys.cli.commands.platform import get_platform_client +from censys.common.config import DEFAULT + + +@patch("censys.cli.commands.platform.get_config") +@patch("censys.cli.commands.platform.console") +@patch("censys.cli.commands.platform.CensysPlatformClient") +def test_get_platform_client_with_env_vars(mock_client, mock_console, mock_get_config): + """Test get_platform_client with environment variables. + + Args: + mock_client: Mock of the CensysPlatformClient. + mock_console: Mock of the console object. + mock_get_config: Mock of the get_config function. + """ + # Mock environment variables + with patch.dict( + os.environ, + { + "CENSYS_PLATFORM_TOKEN": "test-token", + "CENSYS_ORGANIZATION_ID": "test-org-id", + }, + ): + # Mock config + mock_config = MagicMock() + mock_get_config.return_value = mock_config + + # Call the function + client = get_platform_client() + + # Assertions + mock_client.assert_called_once_with( + token="test-token", organization_id="test-org-id" + ) + assert client == mock_client.return_value + + +@patch("censys.cli.commands.platform.get_config") +@patch("censys.cli.commands.platform.console") +@patch("censys.cli.commands.platform.CensysPlatformClient") +def test_get_platform_client_with_config(mock_client, mock_console, mock_get_config): + """Test get_platform_client with config values. + + Args: + mock_client: Mock of the CensysPlatformClient. + mock_console: Mock of the console object. + mock_get_config: Mock of the get_config function. + """ + # Clear environment variables + with patch.dict( + os.environ, + {"CENSYS_PLATFORM_TOKEN": "", "CENSYS_ORGANIZATION_ID": ""}, + clear=True, + ): + # Mock config + mock_config = MagicMock() + mock_config.get.side_effect = lambda section, key, fallback=None: { + (DEFAULT, "platform_token"): "config-token", + (DEFAULT, "platform_org_id"): "config-org-id", + }.get((section, key), fallback) + mock_get_config.return_value = mock_config + + # Call the function + client = get_platform_client() + + # Assertions + mock_client.assert_called_once_with( + token="config-token", organization_id="config-org-id" + ) + assert client == mock_client.return_value + + +@patch("censys.cli.commands.platform.get_config") +@patch("censys.cli.commands.platform.console") +@patch("censys.cli.commands.platform.sys.exit") +def test_get_platform_client_missing_token(mock_exit, mock_console, mock_get_config): + """Test get_platform_client with missing token. + + Args: + mock_exit: Mock of sys.exit function. + mock_console: Mock of the console object. + mock_get_config: Mock of the get_config function. + """ + # Clear environment variables + with patch.dict( + os.environ, + {"CENSYS_PLATFORM_TOKEN": "", "CENSYS_ORGANIZATION_ID": "test-org-id"}, + clear=True, + ): + # Mock config + mock_config = MagicMock() + mock_config.get.side_effect = lambda section, key, fallback=None: { + (DEFAULT, "platform_token"): None, + (DEFAULT, "platform_org_id"): "config-org-id", + }.get((section, key), fallback) + mock_get_config.return_value = mock_config + + # Call the function + get_platform_client() + + # Assertions + mock_console.print.assert_called_once() + mock_exit.assert_called_once_with(1) + + +@patch("censys.cli.commands.platform.get_config") +@patch("censys.cli.commands.platform.console") +@patch("censys.cli.commands.platform.sys.exit") +def test_get_platform_client_missing_org_id(mock_exit, mock_console, mock_get_config): + """Test get_platform_client with missing organization ID. + + Args: + mock_exit: Mock of sys.exit function. + mock_console: Mock of the console object. + mock_get_config: Mock of the get_config function. + """ + # Clear environment variables + with patch.dict( + os.environ, + {"CENSYS_PLATFORM_TOKEN": "test-token", "CENSYS_ORGANIZATION_ID": ""}, + clear=True, + ): + # Mock config + mock_config = MagicMock() + mock_config.get.side_effect = lambda section, key, fallback=None: { + (DEFAULT, "platform_token"): "config-token", + (DEFAULT, "platform_org_id"): None, + }.get((section, key), fallback) + mock_get_config.return_value = mock_config + + # Call the function + get_platform_client() + + # Assertions + mock_console.print.assert_called_once() + mock_exit.assert_called_once_with(1) diff --git a/tests/common/test_exceptions_complete.py b/tests/common/test_exceptions_complete.py new file mode 100644 index 00000000..4a701b01 --- /dev/null +++ b/tests/common/test_exceptions_complete.py @@ -0,0 +1,28 @@ +"""Tests for the Censys Exceptions module with focus on complete coverage.""" + +from censys.common.exceptions import ( + CensysAPIException, + CensysExceptionMapper, + CensysUnauthorizedException, +) + + +def test_get_exception_class_platform_exception(): + """Test that get_exception_class returns a platform exception when status code matches.""" + # Use 401 which is a known platform exception + exception_class = CensysExceptionMapper._get_exception_class(401, "platform") + assert exception_class == CensysUnauthorizedException + + +def test_get_exception_class_search_exception(): + """Test that get_exception_class returns a search exception when status code matches.""" + # Use 401 which is a known search exception + exception_class = CensysExceptionMapper._get_exception_class(401, "search") + assert exception_class == CensysUnauthorizedException + + +def test_get_exception_class_fallback(): + """Test that get_exception_class falls back to CensysAPIException for unknown status codes.""" + # Use 599 which is not a known exception code + exception_class = CensysExceptionMapper._get_exception_class(599, "platform") + assert exception_class == CensysAPIException diff --git a/tests/platform/test_client.py b/tests/platform/test_client.py index 22362e9c..b91e98f8 100644 --- a/tests/platform/test_client.py +++ b/tests/platform/test_client.py @@ -55,7 +55,11 @@ def test_credential_propagation(self): @patch("censys.platform.v3.search.CensysSearch.query") def test_search_delegation(self, mock_query): - """Test that search query calls are properly delegated to the appropriate API instances.""" + """Test that search query calls are properly delegated to the appropriate API instances. + + Args: + mock_query: Mock object for the CensysSearch.query method. + """ # Setup mock return value mock_query.return_value = {"result": {"total": 1, "hits": [{"ip": "8.8.8.8"}]}} diff --git a/tests/platform/v3/conftest.py b/tests/platform/v3/conftest.py new file mode 100644 index 00000000..a6e23bca --- /dev/null +++ b/tests/platform/v3/conftest.py @@ -0,0 +1,32 @@ +"""Configuration for platform v3 tests.""" +import pytest + + +def pytest_configure(config): + """Register custom markers for pytest. + + Args: + config: The pytest config object + """ + config.addinivalue_line( + "markers", + "e2e: marks end-to-end tests that make real API calls and should be run manually, not in CI", + ) + + +def pytest_collection_modifyitems(config, items): + """Modify collected test items before running tests. + + This hook skips all e2e tests by default unless the --run-e2e flag is provided. + + Args: + config: The pytest config object + items: List of collected test items + """ + if hasattr(config.option, "run_e2e") and not config.option.run_e2e: + skip_e2e = pytest.mark.skip( + reason="Need --run-e2e option to run end-to-end tests" + ) + for item in items: + if "e2e" in item.keywords: + item.add_marker(skip_e2e) diff --git a/tests/platform/v3/test_api_error_cases.py b/tests/platform/v3/test_api_error_cases.py new file mode 100644 index 00000000..bcffe303 --- /dev/null +++ b/tests/platform/v3/test_api_error_cases.py @@ -0,0 +1,101 @@ +"""Tests for error handling in the CensysPlatformAPIv3 class.""" + +import configparser +from unittest.mock import patch + +import pytest + +from censys.platform.v3.api import CensysPlatformAPIv3 + + +def test_init_with_api_id_warning(): + """Test initialization with deprecated api_id and api_secret parameters.""" + with pytest.warns(DeprecationWarning): + CensysPlatformAPIv3( + token="test-token", api_id="test-id", api_secret="test-secret" + ) + + +@patch("censys.platform.v3.api.get_config") +def test_init_missing_token(mock_get_config): + """Test initialization with missing token. + + Args: + mock_get_config: Mock of the get_config function. + """ + # Mock the config to return empty values + config = configparser.ConfigParser() + config["DEFAULT"]["platform_token"] = "" + config["DEFAULT"]["platform_org_id"] = "test-org" + mock_get_config.return_value = config + + # Test the ValueError is raised + with pytest.raises( + ValueError, match="Personal Access Token is required for Platform API." + ): + CensysPlatformAPIv3(token=None, organization_id=None) + + +@patch("censys.platform.v3.api.get_config") +def test_init_missing_organization_id(mock_get_config): + """Test initialization with missing organization_id when authenticating. + + Args: + mock_get_config: Mock of the get_config function. + """ + # Mock the config to return empty values + config = configparser.ConfigParser() + config["DEFAULT"]["platform_token"] = "test-token" + config["DEFAULT"]["platform_org_id"] = "" + mock_get_config.return_value = config + + # Test the ValueError is raised + with pytest.raises( + ValueError, match="Organization ID is required for Platform API." + ): + CensysPlatformAPIv3(token=None, organization_id=None) + + +@patch("censys.common.base.CensysAPIBase._get") +def test_get_with_organization_id_and_args(mock_base_get): + """Test _get with organization_id and existing args. + + Args: + mock_base_get: Mock of the base _get method. + """ + api = CensysPlatformAPIv3(token="test-token", organization_id="test-org") + api._get("endpoint", {"arg1": "value1"}) + mock_base_get.assert_called_with( + "endpoint", {"arg1": "value1", "organization_id": "test-org"} + ) + + +@patch("censys.common.base.CensysAPIBase._get") +def test_get_with_organization_id_no_args(mock_base_get): + """Test _get with organization_id and no args. + + Args: + mock_base_get: Mock of the base _get method. + """ + api = CensysPlatformAPIv3(token="test-token", organization_id="test-org") + api._get("endpoint") + mock_base_get.assert_called_with("endpoint", {"organization_id": "test-org"}) + + +@patch("censys.common.base.CensysAPIBase._get") +def test_view_with_params(mock_base_get): + """Test view with params. + + Args: + mock_base_get: Mock of the base _get method. + """ + api = CensysPlatformAPIv3(token="test-token", organization_id="test-org") + api.INDEX_NAME = "test-index" + api.view("resource-id", params={"param1": "value1"}) + + # The actual implementation passes params to _get + mock_base_get.assert_called_with( + f"{api.INDEX_NAME}/resource-id", + {"organization_id": "test-org"}, + params={"param1": "value1"}, + ) diff --git a/tests/platform/v3/test_end_to_end.py b/tests/platform/v3/test_end_to_end.py new file mode 100644 index 00000000..fa657b8e --- /dev/null +++ b/tests/platform/v3/test_end_to_end.py @@ -0,0 +1,100 @@ +"""End-to-end tests for the Censys Platform API. + +These tests require valid Platform credentials and make actual API calls. +They should only be run manually and not in CI environments. +""" +import configparser +import os + +import pytest + +from tests.utils import CensysTestCase + +from censys.common.config import DEFAULT, get_config +from censys.platform.v3.search import CensysSearch, ResultPaginator + +# Mark the entire module as "e2e" tests +pytestmark = pytest.mark.e2e + + +def is_config_valid() -> bool: + """Check if Censys Platform credentials are properly configured. + + Returns: + bool: True if both CENSYS_PLATFORM_TOKEN and CENSYS_ORGANIZATION_ID are found, + either as environment variables or in the configuration file. + """ + # Check environment variables first + token = os.environ.get("CENSYS_PLATFORM_TOKEN") + org_id = os.environ.get("CENSYS_ORGANIZATION_ID") + + if token and org_id: + return True + + # If not in environment variables, check config file + try: + config = get_config() + token = config.get(DEFAULT, "platform_token", fallback=None) + org_id = config.get(DEFAULT, "platform_org_id", fallback=None) + return bool(token and org_id) + except (configparser.Error, FileNotFoundError): + return False + + +class TestPlatformEndToEnd(CensysTestCase): + """End-to-end tests for the Platform Search API.""" + + def test_live_search(self, request): + """Test live search against the Censys Platform API. + + This test makes real API calls to the Censys Platform API and will + count against your quota when run with the --run-e2e flag. + + Args: + request: Pytest request fixture to access config options + """ + # Skip this test unless --run-e2e flag is provided + run_e2e = request.config.getoption("--run-e2e", default=False) + if not run_e2e: + pytest.skip("Test requires --run-e2e flag to run") + + # Also check if we have valid credentials + if not is_config_valid(): + pytest.skip( + "No Censys Platform credentials found. Run 'censys platform config' to configure." + ) + + # Get credentials from environment variables or config file + token = os.environ.get("CENSYS_PLATFORM_TOKEN") + org_id = os.environ.get("CENSYS_ORGANIZATION_ID") + + if not token or not org_id: + # If environment variables are not set, try config file + config = get_config() + token = config.get(DEFAULT, "platform_token", fallback=None) + org_id = config.get(DEFAULT, "platform_org_id", fallback=None) + + # Initialize client with real credentials + client = CensysSearch(token=token, organization_id=org_id) + + # Test simple query + query_result = client.query("services.port:443", per_page=5) + assert "result" in query_result + assert "hits" in query_result["result"] + + # Test pagination + paginator = ResultPaginator(client, "services.port:443", per_page=2) + + # Get first page + page1 = paginator.get_page() + assert len(page1) > 0 + + # Get second page if it exists + if paginator.has_next_page(): + page2 = paginator.get_page() + assert len(page2) > 0 + + # Test get_all_results + all_results = paginator.get_all_results(max_results=5) + assert len(all_results) > 0 + assert len(all_results) <= 5 # We limited to 5 results diff --git a/tests/platform/v3/test_result_paginator.py b/tests/platform/v3/test_result_paginator.py new file mode 100644 index 00000000..ea6e0f86 --- /dev/null +++ b/tests/platform/v3/test_result_paginator.py @@ -0,0 +1,218 @@ +"""Tests for the ResultPaginator class.""" + +import warnings +from unittest.mock import MagicMock + +from tests.utils import CensysTestCase + +from censys.platform.v3.search import CensysSearch, ResultPaginator + + +class TestResultPaginator(CensysTestCase): + """Tests for the ResultPaginator class.""" + + def setUp(self): + """Set up test case.""" + self.api = MagicMock(spec=CensysSearch) + + # Mock API responses + self.first_page_response = { + "result": { + "hits": [{"id": "1"}, {"id": "2"}], + "total": 5, + "cursor": "next-cursor", + } + } + + self.second_page_response = { + "result": { + "hits": [{"id": "3"}, {"id": "4"}], + "total": 5, + "cursor": "last-cursor", + } + } + + self.last_page_response = { + "result": { + "hits": [{"id": "5"}], + "total": 5, + "cursor": None, + } + } + + def test_initialization(self): + """Test initialization of ResultPaginator.""" + paginator = ResultPaginator( + self.api, + "test query", + page_size=10, + pages=2, + fields=["id", "name"], + sort="id", + ) + + assert paginator.query == "test query" + assert paginator.page_size == 10 + assert paginator.pages == 2 + assert paginator.fields == ["id", "name"] + assert paginator.sort == "id" + assert paginator.page == 0 + assert paginator.total is None + + def test_get_page(self): + """Test getting a specific page.""" + # Set up API mock for first page test + self.api.query.reset_mock() + self.api.query.side_effect = [self.first_page_response] + + paginator = ResultPaginator(self.api, "test query") + + # Get first page + results = paginator.get_page() + + # Check that API was called with correct parameters + self.api.query.assert_called_once_with( + query="test query", + page_size=100, + cursor=None, + fields=None, + sort=None, + ) + + # Check results + assert results == [{"id": "1"}, {"id": "2"}] + assert paginator.total == 5 + # The get_page method restores the original cursor state, + # so instead we verify that the API returned the expected cursor + assert self.first_page_response["result"]["cursor"] == "next-cursor" + + # Test getting page by number + # Reset and set up API mock for page number test + # For page 2, we need to return first page, then second page + self.api.query.reset_mock() + self.api.query.side_effect = [ + self.first_page_response, + self.second_page_response, + ] + + # Use context manager to ignore the warning about quota + with warnings.catch_warnings(record=True) as w: + # Get page 2 + results = paginator.get_page(page_number=2) + + # Verify a warning was issued + assert len(w) == 1 + assert "API quota" in str(w[0].message) + + # Check API call count (should be called twice) + assert self.api.query.call_count == 2 + + # Check results are from the second page + assert results == [{"id": "3"}, {"id": "4"}] + + # Test getting page by cursor + self.api.query.reset_mock() + self.api.query.side_effect = [self.second_page_response] + + # Get page with specific cursor + results = paginator.get_page(cursor="custom-cursor") + + # Check that API was called with the custom cursor + self.api.query.assert_called_once_with( + query="test query", + page_size=100, + cursor="custom-cursor", + fields=None, + sort=None, + ) + + # Check results + assert results == [{"id": "3"}, {"id": "4"}] + + def test_get_all_results(self): + """Test getting all results.""" + # Set up API mock for all pages + self.api.query.reset_mock() + self.api.query.side_effect = [ + self.first_page_response, + self.second_page_response, + self.last_page_response, + ] + + paginator = ResultPaginator(self.api, "test query") + + # Get all results + results = paginator.get_all_results() + + # Check that API was called for all three pages + assert self.api.query.call_count == 3 + + # Check that all results were collected + assert results == [ + {"id": "1"}, + {"id": "2"}, + {"id": "3"}, + {"id": "4"}, + {"id": "5"}, + ] + + # Check pagination state + assert paginator.total == 5 + assert paginator.next_cursor is None + + def test_iteration(self): + """Test iteration through pages.""" + # Set up API mock for all pages + self.api.query.reset_mock() + self.api.query.side_effect = [ + self.first_page_response, + self.second_page_response, + self.last_page_response, + ] + + paginator = ResultPaginator(self.api, "test query") + + # Collect pages through iteration + collected_pages = [] + for page in paginator: + collected_pages.append(page) + + # Check that API was called for all three pages + assert self.api.query.call_count == 3 + + # Check that all pages were collected + assert collected_pages == [ + [{"id": "1"}, {"id": "2"}], + [{"id": "3"}, {"id": "4"}], + [{"id": "5"}], + ] + + # Check pagination state + assert paginator.total == 5 + assert paginator.next_cursor is None + + def test_page_limit(self): + """Test page limit functionality.""" + # Reset API mock + self.api.query.reset_mock() + self.api.query.side_effect = [ + self.first_page_response, + self.second_page_response, + ] + + # Create paginator with page limit of 2 + paginator = ResultPaginator(self.api, "test query", pages=2) + + # Get all results + results = paginator.get_all_results() + + # Check that API was called only twice (respecting page limit) + assert self.api.query.call_count == 2 + + # Check that only results from first two pages were collected + assert results == [ + {"id": "1"}, + {"id": "2"}, + {"id": "3"}, + {"id": "4"}, + ] diff --git a/tests/platform/v3/test_search.py b/tests/platform/v3/test_search.py index da184c03..f6615109 100644 --- a/tests/platform/v3/test_search.py +++ b/tests/platform/v3/test_search.py @@ -40,7 +40,8 @@ def test_query(self): with patch.object(self.api, "_post") as mock_post: self.api.query(TEST_SEARCH_QUERY) mock_post.assert_called_with( - "v3/global/search/query", data={"q": TEST_SEARCH_QUERY, "per_page": 100} + "v3/global/search/query", + data={"query": TEST_SEARCH_QUERY, "page_size": 100}, ) def test_query_with_org_id(self): @@ -48,7 +49,8 @@ def test_query_with_org_id(self): with patch.object(self.api_with_org, "_post") as mock_post: self.api_with_org.query(TEST_SEARCH_QUERY) mock_post.assert_called_with( - "v3/global/search/query", data={"q": TEST_SEARCH_QUERY, "per_page": 100} + "v3/global/search/query", + data={"query": TEST_SEARCH_QUERY, "page_size": 100}, ) # The _post method in the base class will add organization_id to the params @@ -57,7 +59,7 @@ def test_query_with_params(self): with patch.object(self.api, "_post") as mock_post: self.api.query( TEST_SEARCH_QUERY, - per_page=50, + page_size=50, cursor="nextCursor", fields=["ip", "services.port"], sort="ip", @@ -65,8 +67,8 @@ def test_query_with_params(self): mock_post.assert_called_with( "v3/global/search/query", data={ - "q": TEST_SEARCH_QUERY, - "per_page": 50, + "query": TEST_SEARCH_QUERY, + "page_size": 50, "cursor": "nextCursor", "fields": ["ip", "services.port"], "sort": "ip", @@ -78,7 +80,7 @@ def test_query_with_params_and_org_id(self): with patch.object(self.api_with_org, "_post") as mock_post: self.api_with_org.query( TEST_SEARCH_QUERY, - per_page=50, + page_size=50, cursor="nextCursor", fields=["ip", "services.port"], sort="ip", @@ -86,8 +88,8 @@ def test_query_with_params_and_org_id(self): mock_post.assert_called_with( "v3/global/search/query", data={ - "q": TEST_SEARCH_QUERY, - "per_page": 50, + "query": TEST_SEARCH_QUERY, + "page_size": 50, "cursor": "nextCursor", "fields": ["ip", "services.port"], "sort": "ip", @@ -102,9 +104,9 @@ def test_aggregate(self): mock_post.assert_called_with( "v3/global/search/aggregate", data={ - "q": TEST_SEARCH_QUERY, + "query": TEST_SEARCH_QUERY, "field": "services.service_name", - "num_buckets": 50, + "number_of_buckets": 50, }, ) @@ -115,9 +117,9 @@ def test_aggregate_with_org_id(self): mock_post.assert_called_with( "v3/global/search/aggregate", data={ - "q": TEST_SEARCH_QUERY, + "query": TEST_SEARCH_QUERY, "field": "services.service_name", - "num_buckets": 50, + "number_of_buckets": 50, }, ) # The _post method in the base class will add organization_id to the params diff --git a/tests/platform/v3/test_search_complete.py b/tests/platform/v3/test_search_complete.py new file mode 100644 index 00000000..178b39ae --- /dev/null +++ b/tests/platform/v3/test_search_complete.py @@ -0,0 +1,41 @@ +"""Tests for the Censys Platform Search API with focus on complete coverage.""" + +from unittest.mock import patch + +from censys.platform.v3.search import CensysSearch, ResultPaginator + + +def test_search_method_complete(): + """Test that the search method returns a ResultPaginator with all parameters.""" + with patch.object(CensysSearch, "__init__", return_value=None), patch( + "censys.platform.v3.search.ResultPaginator" + ) as mock_paginator: + # Initialize the search class + search = CensysSearch() + search.INDEX_NAME = "test/index" + + # Call the search method with all parameters + result: ResultPaginator = search.search( + query="test query", + page_size=50, + cursor="test-cursor", + pages=2, + fields=["field1", "field2"], + sort=["field1", "field2"], + extra_param="extra_value", + ) + + # Verify ResultPaginator was created with all parameters + mock_paginator.assert_called_once_with( + search, + "test query", + page_size=50, + cursor="test-cursor", + pages=2, + fields=["field1", "field2"], + sort=["field1", "field2"], + extra_param="extra_value", + ) + + # Ensure the result is what we got from ResultPaginator + assert result == mock_paginator.return_value diff --git a/tests/platform/v3/test_webproperties.py b/tests/platform/v3/test_webproperties.py index 81661434..8d254000 100644 --- a/tests/platform/v3/test_webproperties.py +++ b/tests/platform/v3/test_webproperties.py @@ -64,7 +64,7 @@ def test_view_at_time(self): self.api.view(TEST_WEBPROPERTY, at_time=test_date) mock_get.assert_called_with( f"v3/global/asset/webproperty/{TEST_WEBPROPERTY}", - params={"at_time": "2023-01-01T00:00:00Z"}, + params={"at_time": "2023-01-01T00:00:00.000000Z"}, ) def test_view_at_time_with_org_id(self): @@ -74,7 +74,7 @@ def test_view_at_time_with_org_id(self): self.api_with_org.view(TEST_WEBPROPERTY, at_time=test_date) mock_get.assert_called_with( f"v3/global/asset/webproperty/{TEST_WEBPROPERTY}", - params={"at_time": "2023-01-01T00:00:00Z"}, + params={"at_time": "2023-01-01T00:00:00.000000Z"}, ) # The _get method in the base class will add organization_id to the params From 95b4763eef41d7ce7a2923fed765e6bc464e0f49 Mon Sep 17 00:00:00 2001 From: Aidan Holland Date: Tue, 11 Mar 2025 18:12:09 -0400 Subject: [PATCH 3/3] chore(deps): Update dependencies to latest versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumped package versions: - argcomplete: 3.5.3 → 3.6.0 - certifi: 2024.12.14 → 2025.1.31 - pytest: 8.3.4 → 8.3.5 - responses: 0.25.5 → 0.25.7 - black: <25.0 → <26.0 Updated Poetry lock file to reflect these dependency upgrades. --- poetry.lock | 28 ++++++++++++++-------------- pyproject.toml | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7bd15332..dff42c53 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,15 +1,15 @@ -# This file is automatically @generated by Poetry 2.0.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. [[package]] name = "argcomplete" -version = "3.5.3" +version = "3.6.0" description = "Bash tab completion for argparse" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "argcomplete-3.5.3-py3-none-any.whl", hash = "sha256:2ab2c4a215c59fd6caaff41a869480a23e8f6a5f910b266c1808037f4e375b61"}, - {file = "argcomplete-3.5.3.tar.gz", hash = "sha256:c12bf50eded8aebb298c7b7da7a5ff3ee24dffd9f5281867dfe1424b58c55392"}, + {file = "argcomplete-3.6.0-py3-none-any.whl", hash = "sha256:4e3e4e10beb20e06444dbac0ac8dda650cb6349caeefe980208d3c548708bedd"}, + {file = "argcomplete-3.6.0.tar.gz", hash = "sha256:2e4e42ec0ba2fff54b0d244d0b1623e86057673e57bafe72dda59c64bd5dee8b"}, ] [package.extras] @@ -103,14 +103,14 @@ black = ">=22.1" [[package]] name = "certifi" -version = "2024.12.14" +version = "2025.1.31" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" groups = ["main", "dev"] files = [ - {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"}, - {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"}, + {file = "certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"}, + {file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"}, ] [[package]] @@ -798,14 +798,14 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pytest" -version = "8.3.4" +version = "8.3.5" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"}, - {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"}, + {file = "pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820"}, + {file = "pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845"}, ] [package.dependencies] @@ -958,14 +958,14 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "responses" -version = "0.25.5" +version = "0.25.7" description = "A utility library for mocking out the `requests` Python library." optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "responses-0.25.5-py3-none-any.whl", hash = "sha256:b3e1ae252f69301b84146ff615a869a4182fbe17e8b606f1ac54142515dad5eb"}, - {file = "responses-0.25.5.tar.gz", hash = "sha256:e53991613f76d17ba293c1e3cce8af107c5c7a6a513edf25195aafd89a870dd3"}, + {file = "responses-0.25.7-py3-none-any.whl", hash = "sha256:92ca17416c90fe6b35921f52179bff29332076bb32694c0df02dcac2c6bc043c"}, + {file = "responses-0.25.7.tar.gz", hash = "sha256:8ebae11405d7a5df79ab6fd54277f6f2bc29b2d002d0dd2d5c632594d1ddcedb"}, ] [package.dependencies] @@ -1112,4 +1112,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.1" python-versions = ">=3.8,<4.0" -content-hash = "0636bc5c37dffe33dfad83a11d4729bb86e2edfb1628d8801d5ce06fec8a52ed" +content-hash = "d37f4525deea6d5ea451c4a27f1a00edf0baafde7d9aa22ea79ad05a2b120e21" diff --git a/pyproject.toml b/pyproject.toml index 2c505974..8a6612d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ argcomplete = ">=2.0.0,<4.0.0" [tool.poetry.group.dev.dependencies] # Lint -black = ">=23.3,<25.0" +black = ">=23.3,<26.0" blacken-docs = "^1.13.0" darglint = "^1.8.1" flake8 = "^5.0.4"