diff --git a/ghostwriter/commandcenter/admin.py b/ghostwriter/commandcenter/admin.py index 3e1a9d40c..2787eb8f6 100644 --- a/ghostwriter/commandcenter/admin.py +++ b/ghostwriter/commandcenter/admin.py @@ -13,6 +13,7 @@ ExtraFieldSpec, GeneralConfiguration, NamecheapConfiguration, + CloudflareConfiguration, ReportConfiguration, SlackConfiguration, VirusTotalConfiguration, @@ -23,6 +24,7 @@ admin.site.register(CloudServicesConfiguration, SingletonModelAdmin) admin.site.register(CompanyInformation, SingletonModelAdmin) admin.site.register(NamecheapConfiguration, SingletonModelAdmin) +admin.site.register(CloudflareConfiguration, SingletonModelAdmin) admin.site.register(SlackConfiguration, SingletonModelAdmin) admin.site.register(VirusTotalConfiguration, SingletonModelAdmin) admin.site.register(GeneralConfiguration, SingletonModelAdmin) diff --git a/ghostwriter/commandcenter/migrations/0001_initial.py b/ghostwriter/commandcenter/migrations/0001_initial.py index 0d59ec854..855317bdc 100644 --- a/ghostwriter/commandcenter/migrations/0001_initial.py +++ b/ghostwriter/commandcenter/migrations/0001_initial.py @@ -90,6 +90,21 @@ class Migration(migrations.Migration): "verbose_name": "Namecheap Configuration", }, ), + migrations.CreateModel( + name="CloudflareConfiguration", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("enable", models.BooleanField(default=False)), + ("api_key", models.CharField(default="Cloudflare Global API Key", max_length=255)), + ("username", models.CharField(default="Account email", max_length=255)), + ("account_id", models.CharField(default="Account ID", max_length=255)), + ("client_ip", models.CharField(default="Whitelisted IP Address", max_length=255)), + ("page_size", models.IntegerField(default=100)), + ], + options={ + "verbose_name": "Cloudflare Configuration", + }, + ), migrations.CreateModel( name="ReportConfiguration", fields=[ diff --git a/ghostwriter/commandcenter/migrations/0034_add_gcp_fields.py b/ghostwriter/commandcenter/migrations/0034_add_gcp_fields.py new file mode 100644 index 000000000..086aa72c6 --- /dev/null +++ b/ghostwriter/commandcenter/migrations/0034_add_gcp_fields.py @@ -0,0 +1,92 @@ +# Generated manually to add GCP credential fields + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("commandcenter", "0033_auto_20241204_1810"), # Replace with your actual previous migration + ] + + operations = [ + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_project_id", + field=models.CharField( + max_length=255, + default="your-project-id", + verbose_name="GCP Project ID", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_private_key_id", + field=models.CharField( + max_length=255, + default="your-private-key-id", + verbose_name="GCP Private Key ID", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_private_key", + field=models.TextField( + default="your-private-key", + verbose_name="GCP Private Key", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_client_email", + field=models.CharField( + max_length=255, + default="your-service-account@project.iam.gserviceaccount.com", + verbose_name="GCP Client Email", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_client_id", + field=models.CharField( + max_length=255, + default="your-client-id", + verbose_name="GCP Client ID", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_auth_uri", + field=models.CharField( + max_length=255, + default="https://accounts.google.com/o/oauth2/auth", + verbose_name="GCP Auth URI", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_token_uri", + field=models.CharField( + max_length=255, + default="https://oauth2.googleapis.com/token", + verbose_name="GCP Token URI", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_auth_cert_url", + field=models.CharField( + max_length=255, + default="https://www.googleapis.com/oauth2/v1/certs", + verbose_name="GCP Auth Provider Cert URL", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_client_cert_url", + field=models.TextField( + default="https://www.googleapis.com/robot/v1/metadata/x509/your-service-account@project.iam.gserviceaccount.com", + verbose_name="GCP Client Cert URL", + ), + ), + ] diff --git a/ghostwriter/commandcenter/models.py b/ghostwriter/commandcenter/models.py index 472572efa..97a062078 100644 --- a/ghostwriter/commandcenter/models.py +++ b/ghostwriter/commandcenter/models.py @@ -74,6 +74,48 @@ class Meta: def sanitized_api_key(self): return sanitize(self.api_key) +class CloudflareConfiguration(SingletonModel): + enable = models.BooleanField(default=False) + api_key = models.CharField(max_length=255, default="Cloudflare API key", help_text="Your Cloudflare API key") + username = models.CharField(max_length=255, default="Account Email", help_text="Your Cloudflare email") + account_id = models.CharField( + "Account ID", + max_length=255, + default="Account ID", + help_text="Your Cloudflare Account ID", + ) + client_ip = models.CharField( + "Whitelisted IP Address", + max_length=255, + default="Whitelisted IP Address", + help_text="Your external IP address registered with Cloudflare", + ) + page_size = models.IntegerField( + "Page Size", + default=100, + help_text="Maximum number of domains to return (100 is the max allowed)", + ) + + def __str__(self): + return "Cloudflare Configuration" + + class Meta: + verbose_name = "Cloudflare Configuration" + + @property + def sanitized_api_key(self): + return sanitize(self.api_key) + + def __str__(self): + return "Cloudflare Configuration" + + class Meta: + verbose_name = "Cloudflare Configuration" + + @property + def sanitized_api_key(self): + return sanitize(self.api_key) + class ReportConfiguration(SingletonModel): enable_borders = models.BooleanField(default=False, help_text="Enable borders around images in Word documents") @@ -281,6 +323,16 @@ class CloudServicesConfiguration(SingletonModel): default=7, help_text="Number of days to delay cloud monitoring notifications for teardown", ) + # GCP-Specific Configuration Fields + gcp_project_id = models.CharField("GCP Project ID", max_length=255, default="your-project-id") + gcp_private_key_id = models.CharField("GCP Private Key ID", max_length=255, default="your-private-key-id") + gcp_private_key = models.TextField("GCP Private Key", default="your-private-key") + gcp_client_email = models.CharField("GCP Client Email", max_length=255, default="your-service-account@example.iam.gserviceaccount.com") + gcp_client_id = models.CharField("GCP Client ID", max_length=255, default="your-client-id") + gcp_auth_uri = models.CharField("GCP Auth URI", max_length=255, default="https://accounts.google.com/o/oauth2/auth") + gcp_token_uri = models.CharField("GCP Token URI", max_length=255, default="https://oauth2.googleapis.com/token") + gcp_auth_cert_url = models.CharField("GCP Auth Provider Cert URL", max_length=255, default="https://www.googleapis.com/oauth2/v1/certs") + gcp_client_cert_url = models.TextField("GCP Client Cert URL", default="https://www.googleapis.com/robot/v1/metadata/x509/your-service-account@example.iam.gserviceaccount.com") def __str__(self): return "Cloud Services Configuration" diff --git a/ghostwriter/factories.py b/ghostwriter/factories.py index c755f4efd..b3dceab6a 100644 --- a/ghostwriter/factories.py +++ b/ghostwriter/factories.py @@ -803,6 +803,17 @@ class Meta: page_size = 100 +class CloudflareConfigurationFactory(factory.django.DjangoModelFactory): + class Meta: + model = "commandcenter.CloudflareConfiguration" + + enable = Faker("boolean") + api_key = Faker("credit_card_number") + username = Faker("user_name") + api_username = Faker("user_name") + client_ip = Faker("ipv4_private") + page_size = 100 + class ReportConfigurationFactory(factory.django.DjangoModelFactory): class Meta: model = "commandcenter.ReportConfiguration" diff --git a/ghostwriter/home/templates/home/management.html b/ghostwriter/home/templates/home/management.html index 1ebe72e3a..aeb9ba2cc 100644 --- a/ghostwriter/home/templates/home/management.html +++ b/ghostwriter/home/templates/home/management.html @@ -18,6 +18,7 @@ {% get_solo "commandcenter.CompanyInformation" as company_config %} {% get_solo "commandcenter.CloudServicesConfiguration" as cloud_config %} {% get_solo "commandcenter.NamecheapConfiguration" as namecheap_config %} + {% get_solo "commandcenter.CloudflareConfiguration" as cloudflare_config %} {% get_solo "commandcenter.ReportConfiguration" as report_config %} {% get_solo "commandcenter.SlackConfiguration" as slack_config %} {% get_solo "commandcenter.VirusTotalConfiguration" as vt_config %} @@ -136,6 +137,37 @@

API & Notification Configurations

Disabled {% endif %} + {% if cloudflare_config.enable %} + + Cloudflare API Enabled + {{ cloudflare_config.enable }} + + + Cloudflare Whitelisted IP + {{ cloudflare_config.client_ip }} + + + Cloudflare API Key + {{ cloudflare_config.sanitized_api_key }} + + + Cloudflare Email + {{ cloudflare_config.username }} + + + Cloudflare Account ID + {{ cloudflare_config.account_id }} + + + Cloudflare Page Size + {{ cloudflare_config.page_size }} + + {% else %} + + Cloudflare API Enabled + Disabled + + {% endif %} @@ -165,6 +197,14 @@

API & Notification Configurations

Digital Ocean API Key {{ cloud_config.sanitized_do_api_key }} + + GCP Project ID + {{ cloud_config.gcp_project_id }} + + + GCP Private Key ID + {{ cloud_config.gcp_private_key_id }} + {% else %} Cloud Monitoring Enabled @@ -309,12 +349,18 @@
Test Configurations
+
+ diff --git a/ghostwriter/home/urls.py b/ghostwriter/home/urls.py index 8f68f6ce9..fa19821ff 100644 --- a/ghostwriter/home/urls.py +++ b/ghostwriter/home/urls.py @@ -27,11 +27,21 @@ views.TestDOConnection.as_view(), name="ajax_test_do", ), + path( + "ajax/management/test/gcp", + views.TestGCPConnection.as_view(), + name="ajax_test_gcp", + ), path( "ajax/management/test/namecheap", views.TestNamecheapConnection.as_view(), name="ajax_test_namecheap", ), + path( + "ajax/management/test/cloudflare", + views.TestCloudflareConnection.as_view(), + name="ajax_test_cloudflare", + ), path( "ajax/management/test/slack", views.TestSlackConnection.as_view(), diff --git a/ghostwriter/home/views.py b/ghostwriter/home/views.py index c6ae22076..cd3ffc826 100644 --- a/ghostwriter/home/views.py +++ b/ghostwriter/home/views.py @@ -47,12 +47,6 @@ def update_session(request): else: request.session["sidebar"] = {} request.session["sidebar"]["sticky"] = True - if req_data == "filter": - if "filter" in request.session.keys(): - request.session["filter"]["sticky"] ^= True - else: - request.session["filter"] = {} - request.session["filter"]["sticky"] = True request.session.save() data = { "result": "success", @@ -228,6 +222,40 @@ def post(self, request, *args, **kwargs): } return JsonResponse(data) +class TestGCPConnection(RoleBasedAccessControlMixin, View): + """ + Create an individual :model:`django_q.Task` under group ``GCP Test`` with + :task:`shepherd.tasks.test_gcp` to test the GCP info stored in + :model:`commandcenter.CloudServicesConfiguration`. + """ + + def test_func(self): + return verify_user_is_privileged(self.request.user) + + def handle_no_permission(self): + messages.error(self.request, "You do not have permission to access that.") + return redirect("home:dashboard") + + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``GCP Test`` + result = "success" + try: + async_task( + "ghostwriter.shepherd.tasks.test_gcp", + self.request.user, + group="GCP Test", + ) + message = "GCP test has been successfully queued." + except Exception: # pragma: no cover + result = "error" + message = "GCP test could not be queued." + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) + class TestNamecheapConnection(RoleBasedAccessControlMixin, View): """ @@ -263,6 +291,40 @@ def post(self, request, *args, **kwargs): } return JsonResponse(data) +class TestCloudflareConnection(RoleBasedAccessControlMixin, View): + """ + Create an individual :model:`django_q.Task` under group ``Cloudflare Test`` with + :task:`shepherd.tasks.test_cloudflare` to test the Cloudflare API configuration stored + in :model:`commandcenter.CloudflareConfiguration`. + """ + + def test_func(self): + return verify_user_is_privileged(self.request.user) + + def handle_no_permission(self): + messages.error(self.request, "You do not have permission to access that.") + return redirect("home:dashboard") + + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``Cloudflare Test`` + result = "success" + try: + async_task( + "ghostwriter.shepherd.tasks.test_cloudflare", + self.request.user, + group="Cloudflare Test", + ) + message = "Cloudflare API test has been successfully queued." + except Exception: # pragma: no cover + result = "error" + message = "Cloudflare API test could not be queued." + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) + class TestSlackConnection(RoleBasedAccessControlMixin, View): """ diff --git a/ghostwriter/modules/cloud_monitors.py b/ghostwriter/modules/cloud_monitors.py index 79d23e4f7..7fdcb83dd 100644 --- a/ghostwriter/modules/cloud_monitors.py +++ b/ghostwriter/modules/cloud_monitors.py @@ -11,6 +11,8 @@ import requests from botocore.config import Config from botocore.exceptions import ClientError, ConnectTimeoutError +from googleapiclient.discovery import build +from google.oauth2 import service_account # Using __name__ resolves to ghostwriter.modules.cloud_monitors logger = logging.getLogger(__name__) @@ -484,3 +486,116 @@ def fetch_digital_ocean(api_key, ignore_tags=None, do_only_running=False): ) return {"capable": capable, "message": message, "instances": instances} + +def format_gcp_private_key(raw_key: str) -> str: + """ + Normalize a GCP private key string to ensure it's in proper PEM format. + Handles keys stored with literal '\\n' as well as already-formatted PEM blocks. + """ + if not raw_key: + return "" + + if "\\n" in raw_key: + raw_key = raw_key.replace("\\n", "\n") + + raw_key = raw_key.strip() + + if not raw_key.startswith("-----BEGIN PRIVATE KEY-----"): + raw_key = f"-----BEGIN PRIVATE KEY-----\n{raw_key}" + if not raw_key.endswith("-----END PRIVATE KEY-----"): + raw_key = f"{raw_key}\n-----END PRIVATE KEY-----" + + return raw_key + +def fetch_gcp_instances(cloud_config): + """ + Fetches all virtual machine instances from GCP using values from the + CloudServicesConfiguration model. + """ + instances = {} + + try: + private_key = format_gcp_private_key(cloud_config.gcp_private_key) + + service_account_info = { + "type": "service_account", + "project_id": cloud_config.gcp_project_id, + "private_key_id": cloud_config.gcp_private_key_id, + "private_key": private_key, + "client_email": cloud_config.gcp_client_email, + "client_id": cloud_config.gcp_client_id, + "auth_uri": cloud_config.gcp_auth_uri, + "token_uri": cloud_config.gcp_token_uri, + "auth_provider_x509_cert_url": cloud_config.gcp_auth_cert_url, + "client_x509_cert_url": cloud_config.gcp_client_cert_url + } + + credentials = service_account.Credentials.from_service_account_info(service_account_info) + compute = build('compute', 'v1', credentials=credentials, cache_discovery=False) + + # Get all zones + zones_request = compute.zones().list(project=cloud_config.gcp_project_id) + zones_response = zones_request.execute() + zones = zones_response.get('items', []) + + for zone in zones: + zone_name = zone['name'] + instance_request = compute.instances().list(project=cloud_config.gcp_project_id, zone=zone_name) + response = instance_request.execute() + + if 'items' in response: + for instance in response['items']: + instance_id = instance['id'] + name = instance['name'] + status = instance['status'] + tags = instance.get('tags', {}).get('items', []) + + internal_ip = None + external_ip = None + network_interfaces = instance.get('networkInterfaces', []) + if network_interfaces: + internal_ip = network_interfaces[0].get('networkIP') + access_configs = network_interfaces[0].get('accessConfigs', []) + if access_configs: + external_ip = access_configs[0].get('natIP') + + # Parse launch time + raw_launch_time = instance.get("creationTimestamp") + launch_time = None + time_up = "Unknown" + if raw_launch_time: + try: + launch_time = datetime.strptime(raw_launch_time, "%Y-%m-%dT%H:%M:%S.%f%z") + now = datetime.now(timezone.utc) + delta = now - launch_time + time_up = f"{delta.days // 30} months" + except Exception: + logger.warning(f"Could not parse launch time: {raw_launch_time}") + + # Determine machine type (extract from URL) + machine_type_full = instance.get("machineType", "") + machine_type = machine_type_full.split("/")[-1] if "/" in machine_type_full else machine_type_full + + instances[instance_id] = { + "id": instance_id, + "name": name, + "provider": "Google Cloud Platform", + "service": "Compute Engine", + "type": machine_type, + "monthly_cost": 0.0, # Can be updated later via billing API + "cost_to_date": 0.0, # Optional estimate if billing integrated + "zone": zone_name, + "state": status, + "tags": ", ".join(tags), + "public_ip": [external_ip] if external_ip else [], + "private_ip": [internal_ip] if internal_ip else [], + "launch_time": launch_time, + "time_up": time_up, + "ignore": False + } + + return {"message": "", "capable": True, "instances": instances} + + except Exception as e: + logger.exception("GCP instance fetch failed") + return {"message": str(e), "capable": False, "instances": {}} \ No newline at end of file diff --git a/ghostwriter/shepherd/forms.py b/ghostwriter/shepherd/forms.py index 41b9689eb..7984061e7 100644 --- a/ghostwriter/shepherd/forms.py +++ b/ghostwriter/shepherd/forms.py @@ -184,6 +184,7 @@ def __init__(self, *args, **kwargs): self.fields[field].widget.attrs["autocomplete"] = "off" self.fields["name"].widget.attrs["placeholder"] = "ghostwriter.wiki" self.fields["registrar"].widget.attrs["placeholder"] = "Namecheap" + self.fields["registrar"].widget.attrs["placeholder"] = "Cloudflare" self.fields["domain_status"].empty_label = "-- Select Status --" self.fields["whois_status"].empty_label = "-- Select Status --" self.fields["health_status"].empty_label = "-- Select Status --" diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index 922683486..3e8224cb1 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -8,6 +8,9 @@ from collections import defaultdict from datetime import date, datetime, timedelta from math import ceil +from cloudflare import Cloudflare +from googleapiclient.discovery import build +from google.oauth2 import service_account # Django Imports from django.db.models import Q @@ -23,6 +26,7 @@ from ghostwriter.commandcenter.models import ( CloudServicesConfiguration, NamecheapConfiguration, + CloudflareConfiguration, VirusTotalConfiguration, ) from ghostwriter.modules.cloud_monitors import ( @@ -30,6 +34,7 @@ fetch_aws_lightsail, fetch_aws_s3, fetch_digital_ocean, + fetch_gcp_instances, test_aws, ) from ghostwriter.modules.dns_toolkit import DNSCollector @@ -68,6 +73,22 @@ def __call__(self, r): r.headers["Authorization"] = "Bearer " + self.token return r +def parse_iso_date(date_str): + """ + Convert ISO 8601 datetime string to a Python datetime.date object. + If the string is None or invalid, returns None. + """ + if not date_str: + return None + try: + # This handles the 'Z' suffix for UTC + return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ").date() + except ValueError: + try: + # Fallback if microseconds are missing + return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%SZ").date() + except Exception: + return None def namecheap_reset_dns(namecheap_config, domain): """ @@ -903,6 +924,132 @@ def fetch_namecheap_domains(): return domain_changes +def fetch_cloudflare_domains(): + """ + Fetch a list of registered domains for the specified Cloudflare account using the Cloudflare API. + Requires an API key and email for authentication. Returns a dictionary containing errors + and each domain name paired with change status. + + Result status: created, updated, burned, updated & burned + """ + domains_list = [] + domain_changes = {"errors": {}, "updates": {}} + + logger.info("Starting Cloudflare synchronization task at %s", datetime.now()) + + cloudflare_config = CloudflareConfiguration.get_solo() + + try: + client = Cloudflare( + api_email=cloudflare_config.username, + api_key=cloudflare_config.api_key + ) + + page = 1 + per_page = 20 # Adjust based on API limits + total_pages = 1 + + while page <= total_pages: + logger.info("Requesting page %s of %s", page, total_pages) + response = client.zones.list(page=page, per_page=per_page) + + if response.success: + total_pages = ceil(response.result_info.total_count / response.result_info.per_page) + + for domain in response.result: + domains_list.append({ + "id": domain.id, + "name": domain.name, + "type": domain.type, + "status": "active" if not domain.meta.phishing_detected else "burned", + }) + else: + error_messages = " | ".join([error.message for error in response.errors]) + logger.error("Cloudflare API returned errors: %s", error_messages) + return {"errors": {"cloudflare": f"Cloudflare API returned errors: {error_messages}"}} + + page += 1 + except Exception: + trace = traceback.format_exc() + logger.exception("Cloudflare API request failed") + domain_changes["errors"]["cloudflare"] = f"The Cloudflare API request failed: {trace}" + return domain_changes + + # No domains returned if the provided account doesn't have any + if domains_list: + domain_queryset = Domain.objects.filter(registrar="Cloudflare") + expired_status = DomainStatus.objects.get(domain_status="Expired") + burned_status = DomainStatus.objects.get(domain_status="Burned") + available_status = DomainStatus.objects.get(domain_status="Available") + health_burned_status = HealthStatus.objects.get(health_status="Burned") + + for domain in domain_queryset: + if not any(d["name"] == domain.name for d in domains_list): + logger.info("Domain %s is not in the Cloudflare data", domain.name) + if not domain.expired: + domain_changes["updates"][domain.id] = {"domain": domain.name, "change": "expired"} + domain.expired = True + domain.auto_renew = False + domain.domain_status = expired_status + domain.save() + _ = DomainNote.objects.create( + domain=domain, + note="Automatically set to Expired because the domain did not appear in Cloudflare during a sync.", + ) + else: + if domain.expired: + domain.expired = False + domain.domain_status = available_status + domain.save() + domain_changes["updates"][domain.id] = {"domain": domain.name, "change": "renewed"} + + for domain in domains_list: + # Use your Cloudflare account ID from config + account_id = cloudflare_config.account_id # Make sure you have this field in your model + + # Make the registrar lookup call + registrar_info = client.registrar.domains.get( + domain_name=domain["name"], + account_id=account_id + ) + + # Enrich the entry with registrar metadata + entry = { + "name": domain["name"], + "registrar": "Cloudflare", + "expiration": parse_iso_date(registrar_info.get("expires_at")), + "creation": parse_iso_date(registrar_info.get("created_at")), + } + + newly_burned = domain["status"] == "burned" + + if newly_burned: + entry["health_status"] = health_burned_status + entry["domain_status"] = burned_status + entry["burned_explanation"] = "

Cloudflare detected potential phishing activity on this domain.

" + + try: + instance, created = Domain.objects.update_or_create(name=domain["name"], defaults=entry) + instance.save() + change_status = ( + "created & burned" if created and newly_burned else + "burned" if newly_burned else + "created" if created else + "updated" + ) + domain_changes["updates"][instance.id] = {"domain": domain["name"], "change": change_status} + except Exception: + trace = traceback.format_exc() + logger.exception("Failed to update domain %s", domain["name"]) + domain_changes["errors"][domain["name"]] = {"error": trace} + + + logger.info("Cloudflare synchronization completed at %s with these changes:\n%s", datetime.now(), domain_changes) + else: + logger.warning("No domains were returned for the provided Cloudflare account!") + + return domain_changes + def months_between(date1, date2): """ @@ -944,7 +1091,6 @@ def json_datetime_converter(dt): return str(dt) return None - def review_cloud_infrastructure(aws_only_running=False, do_only_running=False): """ Fetch active virtual machines/instances in Digital Ocean, Azure, and AWS and @@ -1027,6 +1173,21 @@ def review_cloud_infrastructure(aws_only_running=False, do_only_running=False): for instance in do_results["instances"]: vps_info["instances"][instance["id"]] = instance + ############### + # GCP Section # + ############### + + # Fetch GCP results using full cloud_config object + gcp_results = fetch_gcp_instances(cloud_config) + + if gcp_results["message"]: + vps_info["errors"]["gcp"] = gcp_results["message"] + else: + if gcp_results["capable"]: + for instance_id, instance in gcp_results["instances"].items(): + vps_info["instances"][instance_id] = instance + + ################## # Notifications # ################## @@ -1427,3 +1588,109 @@ def test_virustotal(user): logger.info("Test of the VirusTotal completed at %s", datetime.now()) return {"result": level, "message": message} + +def test_cloudflare(user): + """ + Test the Cloudflare API configuration stored in environment variables stored in :model:`commandcenter.CloudflareConfiguration`. + """ + level = "error" + logger.info("Starting Cloudflare API test at %s", datetime.now()) + cloudflare_config = CloudflareConfiguration.get_solo() + try: + client = Cloudflare( + api_email=cloudflare_config.username, + api_key=cloudflare_config.api_key + ) + response = client.zones.list() + if response.success: + level = "success" + message = "Successfully authenticated to Cloudflare" + else: + message = "Cloudflare API authentication failed: " + " | ".join([error["message"] for error in response.errors]) + except Exception: + trace = traceback.format_exc() + logger.exception("Cloudflare API request failed") + message = f"The Cloudflare API request failed: {trace}" + return {"result": level, "message": message} + +def format_gcp_private_key(raw_key: str) -> str: + """ + Normalize a GCP private key string to ensure it's in proper PEM format. + Handles keys stored with literal '\\n' as well as already-formatted PEM blocks. + """ + if not raw_key: + return "" + + if "\\n" in raw_key: + raw_key = raw_key.replace("\\n", "\n") + + raw_key = raw_key.strip() + + if not raw_key.startswith("-----BEGIN PRIVATE KEY-----"): + raw_key = f"-----BEGIN PRIVATE KEY-----\n{raw_key}" + if not raw_key.endswith("-----END PRIVATE KEY-----"): + raw_key = f"{raw_key}\n-----END PRIVATE KEY-----" + + return raw_key + +def test_gcp(user): + """ + Test the GCP service account credentials configured in + :model:`commandcenter.CloudServicesConfiguration`. + """ + cloud_config = CloudServicesConfiguration.get_solo() + level = "error" + logger.info("Starting a test of the GCP credentials at %s", datetime.now()) + + private_key = format_gcp_private_key(cloud_config.gcp_private_key) + + try: + # Build credentials from individual variables (not file) + service_account_info = { + "type": "service_account", + "project_id": cloud_config.gcp_project_id, + "private_key_id": cloud_config.gcp_private_key_id, + "private_key": private_key, + "client_email": cloud_config.gcp_client_email, + "client_id": cloud_config.gcp_client_id, + "auth_uri": cloud_config.gcp_auth_uri, + "token_uri": cloud_config.gcp_token_uri, + "auth_provider_x509_cert_url": cloud_config.gcp_auth_cert_url, + "client_x509_cert_url": cloud_config.gcp_client_cert_url + } + + credentials = service_account.Credentials.from_service_account_info(service_account_info) + compute = build('compute', 'v1', credentials=credentials, cache_discovery=False) + + # Attempt a simple API call — list zones in the project + request = compute.zones().list(project=cloud_config.gcp_project_id) + response = request.execute() + + if "items" in response: + logger.info("GCP credentials are functional, beginning infrastructure review") + logger.info("Successfully verified the GCP service account credentials") + message = "Successfully verified the GCP service account credentials" + level = "success" + else: + logger.warning("GCP credentials did not return zone list as expected") + message = "GCP credentials did not return zone list as expected — please double-check access scopes and permissions" + + except Exception as e: + logger.exception("Testing authentication to GCP failed") + message = f"Testing authentication to GCP failed: {str(e)}" + + # Send a message to the requesting user + async_to_sync(channel_layer.group_send)( + f"notify_{user}", + { + "type": "message", + "message": { + "message": message, + "level": level, + "title": "GCP Test Complete", + }, + }, + ) + + logger.info("Test of the GCP credentials completed at %s", datetime.now()) + return {"result": level, "message": message} \ No newline at end of file diff --git a/ghostwriter/shepherd/templates/shepherd/update.html b/ghostwriter/shepherd/templates/shepherd/update.html index 17ab35a6d..2562c0b59 100644 --- a/ghostwriter/shepherd/templates/shepherd/update.html +++ b/ghostwriter/shepherd/templates/shepherd/update.html @@ -95,6 +95,42 @@

Pull Domains from Namecheap

{% endif %} + {% if enable_cloudflare %} +
+

Pull Domains from Cloudflare

+
+ +

The domain library sync with Cloudflare was last requested on:

+

+ {{ cloudflare_last_update_requested }} +

+ + {% if cloudflare_last_update_completed %} + {% if cloudflare_last_update_completed == 'Failed' %} +

Request Status: {{ cloudflare_last_update_completed }}

+ {% if cloudflare_last_result %} +
+ Error: + + {{ cloudflare_last_result.errors }} + +
+ {% endif %} + {% else %} + {% if cloudflare_last_update_completed %} +

Request Status: Completed on {{ cloudflare_last_update_completed }} in {{ cloudflare_last_update_time }} minutes

+ {% endif %} + {% endif %} + {% endif %} + +
+ {% csrf_token %} + + +
+
+ {% endif %} + {% if enable_vt %}
diff --git a/ghostwriter/shepherd/urls.py b/ghostwriter/shepherd/urls.py index 81f3a6d72..fb0b8eb66 100644 --- a/ghostwriter/shepherd/urls.py +++ b/ghostwriter/shepherd/urls.py @@ -74,6 +74,11 @@ views.RegistrarSyncNamecheap.as_view(), name="ajax_update_namecheap", ), + path( + "ajax/update/cloudflare", + views.RegistrarSyncCloudflare.as_view(), + name="ajax_update_cloudflare", + ), path( "ajax/update/cloud", views.MonitorCloudInfrastructure.as_view(), diff --git a/ghostwriter/shepherd/views.py b/ghostwriter/shepherd/views.py index 9b382a089..7280ae3d3 100644 --- a/ghostwriter/shepherd/views.py +++ b/ghostwriter/shepherd/views.py @@ -37,6 +37,7 @@ CloudServicesConfiguration, ExtraFieldSpec, NamecheapConfiguration, + CloudflareConfiguration, VirusTotalConfiguration, ) from ghostwriter.modules.shared import add_content_disposition_header @@ -263,6 +264,18 @@ def post(self, *args, **kwargs): group="Individual Domain Update", hook="ghostwriter.modules.notifications_slack.send_slack_complete_msg", ) + + # Cloudflare + if domain_instance.registrar.lower() == "cloudflare": + cloudflare_config = CloudflareConfiguration.get_solo() + if cloudflare_config.enable: + async_task( + "ghostwriter.shepherd.tasks.cloudflare_reset_dns", + cloudflare_config=cloudflare_config, + domain=domain_instance, + group="Individual Domain Update", + hook="ghostwriter.modules.notifications_slack.send_slack_complete_msg", + ) return JsonResponse(data) @@ -423,7 +436,33 @@ def post(self, request, *args, **kwargs): "message": message, } return JsonResponse(data) + +class RegistrarSyncCloudflare(RoleBasedAccessControlMixin, View): + """ + Create an individual :model:`django_q.Task` under group ``Cloudflare Update`` with + :task:`shepherd.tasks.fetch_cloudflare_domains` to create or update one or more + :model:`shepherd.Domain`. + """ + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``Cloudflare Update`` + result = "success" + try: + task_id = async_task( + "ghostwriter.shepherd.tasks.fetch_cloudflare_domains", + group="Cloudflare Update", + hook="ghostwriter.modules.notifications_slack.send_slack_complete_msg", + ) + message = "Successfully queued Cloudflare update task (Task ID {task}).".format(task=task_id) + except Exception: + result = "error" + message = "Cloudflare update task could not be queued!" + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) class MonitorCloudInfrastructure(RoleBasedAccessControlMixin, View): """ @@ -702,6 +741,16 @@ def update(request): End time of latest :model:`django_q.Task` for group "Namecheap Update" ``namecheap_last_result`` Result of latest :model:`django_q.Task` for group "Namecheap Update" + ``enable_cloudflare`` + The associated value from :model:`commandcenter.CloudflareConfiguration` + ``cloudflare_last_update_requested`` + Start time of latest :model:`django_q.Task` for group "Cloudflare Update" + ``cloudflare_last_update_completed`` + End time of latest :model:`django_q.Task` for group "Cloudflare Update" + ``cloudflare_last_update_time`` + End time of latest :model:`django_q.Task` for group "Cloudflare Update" + ``cloudflare_last_result`` + Result of latest :model:`django_q.Task` for group "Cloudflare Update" ``enable_cloud_monitor`` The associated value from :model:`commandcenter.CloudServicesConfiguration` ``cloud_last_update_requested`` @@ -727,6 +776,8 @@ def update(request): enable_cloud_monitor = cloud_config.enable namecheap_config = NamecheapConfiguration.get_solo() enable_namecheap = namecheap_config.enable + cloudflare_config = CloudflareConfiguration.get_solo() + enable_cloudflare = cloudflare_config.enable # Collect data for category updates cat_last_update_completed = "" @@ -794,6 +845,27 @@ def update(request): else: namecheap_last_update_requested = "Namecheap Syncing is Disabled" + # Collect data for Cloudflare updates + cloudflare_last_update_completed = "" + cloudflare_last_update_time = "" + cloudflare_last_result = "" + if enable_cloudflare: + try: + queryset = Task.objects.filter(group="Cloudflare Update")[0] + cloudflare_last_update_requested = queryset.started + cloudflare_last_result = queryset.result + if queryset.success: + cloudflare_last_update_completed = queryset.stopped + cloudflare_last_update_time = round(queryset.time_taken() / 60, 2) + if cloudflare_last_result["errors"]: + cloudflare_last_update_completed = "Failed" + else: + cloudflare_last_update_completed = "Failed" + except IndexError: + cloudflare_last_update_requested = "Cloudflare Sync Has Not Been Run Yet" + else: + cloudflare_last_update_requested = "Cloudflare Syncing is Disabled" + # Collect data for cloud monitoring cloud_last_update_completed = "" cloud_last_update_time = "" @@ -831,6 +903,11 @@ def update(request): "namecheap_last_update_completed": namecheap_last_update_completed, "namecheap_last_update_time": namecheap_last_update_time, "namecheap_last_result": namecheap_last_result, + "enable_cloudflare": enable_cloudflare, + "cloudflare_last_update_requested": cloudflare_last_update_requested, + "cloudflare_last_update_completed": cloudflare_last_update_completed, + "cloudflare_last_update_time": cloudflare_last_update_time, + "cloudflare_last_result": cloudflare_last_result, "enable_cloud_monitor": enable_cloud_monitor, "cloud_last_update_requested": cloud_last_update_requested, "cloud_last_update_completed": cloud_last_update_completed, diff --git a/requirements/base.txt b/requirements/base.txt index ae87b76bf..1eee7e89e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,6 +6,11 @@ argon2-cffi==21.3.0 # https://github.com/hynek/argon2_cffi redis==3.5.3 # https://github.com/antirez/redis uvicorn[standard]==0.22.0 # https://github.com/encode/uvicorn Twisted==20.3.0 # Temporary constraint for channels: https://github.com/django/daphne/pull/359 +cloudflare==4.0.0 +google-api-python-client>=2.115.0 +google-auth>=2.25.0 +google-auth-httplib2>=0.2.0 +google-auth-oauthlib>=1.1.0 # Django # ------------------------------------------------------------------------------