From c29ea7db79429d62de220af5b0bd45f5f8bdbdf9 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:25:43 -0500 Subject: [PATCH 01/30] Update tasks.py --- ghostwriter/shepherd/tasks.py | 136 ++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index 922683486..08f178c30 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -903,6 +903,119 @@ def fetch_namecheap_domains(): return domain_changes +import os +import traceback +import logging +from datetime import datetime +from math import ceil +from cloudflare import Cloudflare + +logger = logging.getLogger(__name__) + +def fetch_cloudflare_domains(): + """ + Fetch a list of registered domains for the specified Cloudflare account using the Cloudflare API. + Requires an API key and email for authentication. Returns a dictionary containing errors + and each domain name paired with change status. + + Result status: created, updated, burned, updated & burned + """ + domains_list = [] + domain_changes = {"errors": {}, "updates": {}} + + logger.info("Starting Cloudflare synchronization task at %s", datetime.now()) + + try: + client = Cloudflare( + api_email=os.environ.get("CLOUDFLARE_EMAIL"), + api_key=os.environ.get("CLOUDFLARE_API_KEY"), + ) + + page = 1 + per_page = 50 # Adjust based on API limits + total_pages = 1 + + while page <= total_pages: + logger.info("Requesting page %s of %s", page, total_pages) + response = client.zones.list(page=page, per_page=per_page) + + if response.success: + result_info = response.result_info + total_pages = ceil(result_info["total_count"] / result_info["per_page"]) + + for domain in response.result: + domains_list.append({ + "id": domain["id"], + "name": domain["name"], + "type": domain.get("type", "unknown"), + "status": "active" if not domain["meta"].get("phishing_detected", False) else "burned", + }) + else: + error_messages = " | ".join([error["message"] for error in response.errors]) + logger.error("Cloudflare API returned errors: %s", error_messages) + domain_changes["errors"]["cloudflare"] = f"Cloudflare API returned errors: {error_messages}" + return domain_changes + + page += 1 + except Exception: + trace = traceback.format_exc() + logger.exception("Cloudflare API request failed") + domain_changes["errors"]["cloudflare"] = f"The Cloudflare API request failed: {trace}" + return domain_changes + + # No domains returned if the provided account doesn't have any + if domains_list: + domain_queryset = Domain.objects.filter(registrar="Cloudflare") + expired_status = DomainStatus.objects.get(domain_status="Expired") + burned_status = DomainStatus.objects.get(domain_status="Burned") + available_status = DomainStatus.objects.get(domain_status="Available") + health_burned_status = HealthStatus.objects.get(health_status="Burned") + + for domain in domain_queryset: + if not any(d["name"] == domain.name for d in domains_list): + logger.info("Domain %s is not in the Cloudflare data", domain.name) + if not domain.expired: + domain_changes["updates"][domain.id] = {"domain": domain.name, "change": "expired"} + domain.expired = True + domain.auto_renew = False + domain.domain_status = expired_status + domain.save() + _ = DomainNote.objects.create( + domain=domain, + note="Automatically set to Expired because the domain did not appear in Cloudflare during a sync.", + ) + else: + if domain.expired: + domain.expired = False + domain.domain_status = available_status + domain.save() + domain_changes["updates"][domain.id] = {"domain": domain.name, "change": "renewed"} + + for domain in domains_list: + entry = {"name": domain["name"], "registrar": "Cloudflare"} + newly_burned = domain["status"] == "burned" + + if newly_burned: + entry["health_status"] = health_burned_status + entry["domain_status"] = burned_status + entry["burned_explanation"] = "

Cloudflare detected potential phishing activity on this domain.

" + + try: + instance, created = Domain.objects.update_or_create(name=domain["name"], defaults=entry) + instance.save() + change_status = "created & burned" if created and newly_burned else "burned" if newly_burned else "created" if created else "updated" + domain_changes["updates"][instance.id] = {"domain": domain["name"], "change": change_status} + except Exception: + trace = traceback.format_exc() + logger.exception("Failed to update domain %s", domain["name"]) + domain_changes["errors"][domain["name"]] = {"error": trace} + + logger.info("Cloudflare synchronization completed at %s with these changes:\n%s", datetime.now(), domain_changes) + else: + logger.warning("No domains were returned for the provided Cloudflare account!") + + return domain_changes + def months_between(date1, date2): """ @@ -1427,3 +1540,26 @@ def test_virustotal(user): logger.info("Test of the VirusTotal completed at %s", datetime.now()) return {"result": level, "message": message} + +def test_cloudflare_api(user): + """ + Test the Cloudflare API configuration stored in environment variables stored in :model:`commandcenter.CloudflareConfiguration`. + """ + level = "error" + logger.info("Starting Cloudflare API test at %s", datetime.now()) + try: + client = Cloudflare( + api_email=os.environ.get("CLOUDFLARE_EMAIL"), + api_key=os.environ.get("CLOUDFLARE_API_KEY"), + ) + response = client.accounts.tokens.verify() + if response.success: + level = "success" + message = "Successfully authenticated to Cloudflare" + else: + message = "Cloudflare API authentication failed: " + " | ".join([error["message"] for error in response.errors]) + except Exception: + trace = traceback.format_exc() + logger.exception("Cloudflare API request failed") + message = f"The Cloudflare API request failed: {trace}" + return {"result": level, "message": message} From b79a32b8f2203243cda976950533aeb8994ef324 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:27:45 -0500 Subject: [PATCH 02/30] Update tasks.py --- ghostwriter/shepherd/tasks.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index 08f178c30..5d7c783dd 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -903,15 +903,6 @@ def fetch_namecheap_domains(): return domain_changes -import os -import traceback -import logging -from datetime import datetime -from math import ceil -from cloudflare import Cloudflare - -logger = logging.getLogger(__name__) - def fetch_cloudflare_domains(): """ Fetch a list of registered domains for the specified Cloudflare account using the Cloudflare API. From 571b18bebdd2f53ad67a465d3de09d01c0fa075a Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:30:40 -0500 Subject: [PATCH 03/30] Update models.py --- ghostwriter/commandcenter/models.py | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/ghostwriter/commandcenter/models.py b/ghostwriter/commandcenter/models.py index 472572efa..b821d30a4 100644 --- a/ghostwriter/commandcenter/models.py +++ b/ghostwriter/commandcenter/models.py @@ -74,6 +74,38 @@ class Meta: def sanitized_api_key(self): return sanitize(self.api_key) +class CloudflareConfiguration(SingletonModel): + enable = models.BooleanField(default=False) + api_key = models.CharField(max_length=255, default="Cloudflare API Key", help_text="Your Cloudflare API key") + username = models.CharField(max_length=255, default="Account Email", help_text="Your Cloudflare email") + api_username = models.CharField( + "Account ID", + max_length=255, + default="Account ID", + help_text="Your Cloudflare Account ID", + ) + client_ip = models.CharField( + "Whitelisted IP Address", + max_length=255, + default="Whitelisted IP Address", + help_text="Your external IP address registered with Cloudflare", + ) + page_size = models.IntegerField( + "Page Size", + default=100, + help_text="Maximum number of domains to return (100 is the max allowed)", + ) + + def __str__(self): + return "Cloudflare Configuration" + + class Meta: + verbose_name = "Cloudflare Configuration" + + @property + def sanitized_api_key(self): + return sanitize(self.api_key) + class ReportConfiguration(SingletonModel): enable_borders = models.BooleanField(default=False, help_text="Enable borders around images in Word documents") From 7c63e63aa6dc21706990ef750740ff9e534ba847 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:32:09 -0500 Subject: [PATCH 04/30] Update factories.py --- ghostwriter/factories.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ghostwriter/factories.py b/ghostwriter/factories.py index c755f4efd..b3dceab6a 100644 --- a/ghostwriter/factories.py +++ b/ghostwriter/factories.py @@ -803,6 +803,17 @@ class Meta: page_size = 100 +class CloudflareConfigurationFactory(factory.django.DjangoModelFactory): + class Meta: + model = "commandcenter.CloudflareConfiguration" + + enable = Faker("boolean") + api_key = Faker("credit_card_number") + username = Faker("user_name") + api_username = Faker("user_name") + client_ip = Faker("ipv4_private") + page_size = 100 + class ReportConfigurationFactory(factory.django.DjangoModelFactory): class Meta: model = "commandcenter.ReportConfiguration" From 01ffaec174bf97fe11f1e252e4cff30d29e30e8e Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:32:57 -0500 Subject: [PATCH 05/30] Update urls.py --- ghostwriter/home/urls.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ghostwriter/home/urls.py b/ghostwriter/home/urls.py index 8f68f6ce9..b95e8f630 100644 --- a/ghostwriter/home/urls.py +++ b/ghostwriter/home/urls.py @@ -32,6 +32,11 @@ views.TestNamecheapConnection.as_view(), name="ajax_test_namecheap", ), + path( + "ajax/management/test/cloudflare", + views.TestCloudflareConnection.as_view(), + name="ajax_test_cloudflare", + ), path( "ajax/management/test/slack", views.TestSlackConnection.as_view(), From 39c410fd9ee1b6fb4f07d242bbfcce3b55bab5bb Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:34:18 -0500 Subject: [PATCH 06/30] Update views.py --- ghostwriter/home/views.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/ghostwriter/home/views.py b/ghostwriter/home/views.py index 761dc4915..595089443 100644 --- a/ghostwriter/home/views.py +++ b/ghostwriter/home/views.py @@ -257,6 +257,40 @@ def post(self, request, *args, **kwargs): } return JsonResponse(data) +class TestCloudflareConnection(RoleBasedAccessControlMixin, View): + """ + Create an individual :model:`django_q.Task` under group ``Cloudflare Test`` with + :task:`shepherd.tasks.test_Cloudflare` to test the Cloudflare API configuration stored + in :model:`commandcenter.CloudflareConfiguration`. + """ + + def test_func(self): + return verify_user_is_privileged(self.request.user) + + def handle_no_permission(self): + messages.error(self.request, "You do not have permission to access that.") + return redirect("home:dashboard") + + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``Cloudflare Test`` + result = "success" + try: + async_task( + "ghostwriter.shepherd.tasks.test_Cloudflare", + self.request.user, + group="Cloudflare Test", + ) + message = "Cloudflare API test has been successfully queued." + except Exception: # pragma: no cover + result = "error" + message = "Cloudflare API test could not be queued." + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) + class TestSlackConnection(RoleBasedAccessControlMixin, View): """ From 5328ca38d642120f7ec6c1798f783fd3fea6ae71 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:36:01 -0500 Subject: [PATCH 07/30] Update management.html --- ghostwriter/home/templates/home/management.html | 1 + 1 file changed, 1 insertion(+) diff --git a/ghostwriter/home/templates/home/management.html b/ghostwriter/home/templates/home/management.html index 1ebe72e3a..666e3dcd5 100644 --- a/ghostwriter/home/templates/home/management.html +++ b/ghostwriter/home/templates/home/management.html @@ -18,6 +18,7 @@ {% get_solo "commandcenter.CompanyInformation" as company_config %} {% get_solo "commandcenter.CloudServicesConfiguration" as cloud_config %} {% get_solo "commandcenter.NamecheapConfiguration" as namecheap_config %} + {% get_solo "commandcenter.CloudflareConfiguration" as cloudflare_config %} {% get_solo "commandcenter.ReportConfiguration" as report_config %} {% get_solo "commandcenter.SlackConfiguration" as slack_config %} {% get_solo "commandcenter.VirusTotalConfiguration" as vt_config %} From 15475dd5d3383af25cd11125183c1c3ebc12ee05 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:40:38 -0500 Subject: [PATCH 08/30] Update update.html --- .../shepherd/templates/shepherd/update.html | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ghostwriter/shepherd/templates/shepherd/update.html b/ghostwriter/shepherd/templates/shepherd/update.html index 17ab35a6d..2562c0b59 100644 --- a/ghostwriter/shepherd/templates/shepherd/update.html +++ b/ghostwriter/shepherd/templates/shepherd/update.html @@ -95,6 +95,42 @@

Pull Domains from Namecheap

{% endif %} + {% if enable_cloudflare %} +
+

Pull Domains from Cloudflare

+
+ +

The domain library sync with Cloudflare was last requested on:

+

+ {{ cloudflare_last_update_requested }} +

+ + {% if cloudflare_last_update_completed %} + {% if cloudflare_last_update_completed == 'Failed' %} +

Request Status: {{ cloudflare_last_update_completed }}

+ {% if cloudflare_last_result %} +
+ Error: + + {{ cloudflare_last_result.errors }} + +
+ {% endif %} + {% else %} + {% if cloudflare_last_update_completed %} +

Request Status: Completed on {{ cloudflare_last_update_completed }} in {{ cloudflare_last_update_time }} minutes

+ {% endif %} + {% endif %} + {% endif %} + +
+ {% csrf_token %} + + +
+
+ {% endif %} + {% if enable_vt %}
From f7ebd8c845ab3a21912ac858f367fa3f78e1d9ca Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:42:38 -0500 Subject: [PATCH 09/30] Update 0001_initial.py --- .../commandcenter/migrations/0001_initial.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ghostwriter/commandcenter/migrations/0001_initial.py b/ghostwriter/commandcenter/migrations/0001_initial.py index 0d59ec854..245f986c0 100644 --- a/ghostwriter/commandcenter/migrations/0001_initial.py +++ b/ghostwriter/commandcenter/migrations/0001_initial.py @@ -90,6 +90,21 @@ class Migration(migrations.Migration): "verbose_name": "Namecheap Configuration", }, ), + migrations.CreateModel( + name="CloudflareConfiguration", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("enable", models.BooleanField(default=False)), + ("api_key", models.CharField(default="Cloudflare API Key", max_length=255)), + ("username", models.CharField(default="Account email", max_length=255)), + ("api_username", models.CharField(default="Account ID", max_length=255)), + ("client_ip", models.CharField(default="Whitelisted IP Address", max_length=255)), + ("page_size", models.IntegerField(default=100)), + ], + options={ + "verbose_name": "Cloudflare Configuration", + }, + ), migrations.CreateModel( name="ReportConfiguration", fields=[ From dcbfff1724058cbab7ccafdf7076376989f27e05 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:43:21 -0500 Subject: [PATCH 10/30] Update forms.py --- ghostwriter/shepherd/forms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ghostwriter/shepherd/forms.py b/ghostwriter/shepherd/forms.py index 41b9689eb..7984061e7 100644 --- a/ghostwriter/shepherd/forms.py +++ b/ghostwriter/shepherd/forms.py @@ -184,6 +184,7 @@ def __init__(self, *args, **kwargs): self.fields[field].widget.attrs["autocomplete"] = "off" self.fields["name"].widget.attrs["placeholder"] = "ghostwriter.wiki" self.fields["registrar"].widget.attrs["placeholder"] = "Namecheap" + self.fields["registrar"].widget.attrs["placeholder"] = "Cloudflare" self.fields["domain_status"].empty_label = "-- Select Status --" self.fields["whois_status"].empty_label = "-- Select Status --" self.fields["health_status"].empty_label = "-- Select Status --" From be14b390c2d6c07d2564a748dd9f1c0b6ea2eb31 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:43:58 -0500 Subject: [PATCH 11/30] Update urls.py --- ghostwriter/shepherd/urls.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ghostwriter/shepherd/urls.py b/ghostwriter/shepherd/urls.py index 81f3a6d72..98fdbd189 100644 --- a/ghostwriter/shepherd/urls.py +++ b/ghostwriter/shepherd/urls.py @@ -74,6 +74,11 @@ views.RegistrarSyncNamecheap.as_view(), name="ajax_update_namecheap", ), + path( + "ajax/update/cloudflare", + views.RegistrarSynccloudflare.as_view(), + name="ajax_update_cloudflare", + ), path( "ajax/update/cloud", views.MonitorCloudInfrastructure.as_view(), From c4ae3f4db9e9ca6b300b7f42e87f8ad88f6a97ad Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:47:17 -0500 Subject: [PATCH 12/30] Update admin.py --- ghostwriter/commandcenter/admin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ghostwriter/commandcenter/admin.py b/ghostwriter/commandcenter/admin.py index 3e1a9d40c..2787eb8f6 100644 --- a/ghostwriter/commandcenter/admin.py +++ b/ghostwriter/commandcenter/admin.py @@ -13,6 +13,7 @@ ExtraFieldSpec, GeneralConfiguration, NamecheapConfiguration, + CloudflareConfiguration, ReportConfiguration, SlackConfiguration, VirusTotalConfiguration, @@ -23,6 +24,7 @@ admin.site.register(CloudServicesConfiguration, SingletonModelAdmin) admin.site.register(CompanyInformation, SingletonModelAdmin) admin.site.register(NamecheapConfiguration, SingletonModelAdmin) +admin.site.register(CloudflareConfiguration, SingletonModelAdmin) admin.site.register(SlackConfiguration, SingletonModelAdmin) admin.site.register(VirusTotalConfiguration, SingletonModelAdmin) admin.site.register(GeneralConfiguration, SingletonModelAdmin) From b6f895c72194300dc4623d968a19ddad9aa156e5 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 12:53:42 -0500 Subject: [PATCH 13/30] Update views.py --- ghostwriter/shepherd/views.py | 77 +++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/ghostwriter/shepherd/views.py b/ghostwriter/shepherd/views.py index a85238819..bac632bed 100644 --- a/ghostwriter/shepherd/views.py +++ b/ghostwriter/shepherd/views.py @@ -37,6 +37,7 @@ CloudServicesConfiguration, ExtraFieldSpec, NamecheapConfiguration, + CloudflareConfiguration, VirusTotalConfiguration, ) from ghostwriter.modules.shared import add_content_disposition_header @@ -263,6 +264,18 @@ def post(self, *args, **kwargs): group="Individual Domain Update", hook="ghostwriter.modules.notifications_slack.send_slack_complete_msg", ) + + # Cloudflare + if domain_instance.registrar.lower() == "cloudflare": + cloudflare_config = CloudflareConfiguration.get_solo() + if cloudflare_config.enable: + async_task( + "ghostwriter.shepherd.tasks.cloudflare_reset_dns", + cloudflare_config=cloudflare_config, + domain=domain_instance, + group="Individual Domain Update", + hook="ghostwriter.modules.notifications_slack.send_slack_complete_msg", + ) return JsonResponse(data) @@ -423,7 +436,33 @@ def post(self, request, *args, **kwargs): "message": message, } return JsonResponse(data) + +class RegistrarSyncCloudflare(RoleBasedAccessControlMixin, View): + """ + Create an individual :model:`django_q.Task` under group ``Cloudflare Update`` with + :task:`shepherd.tasks.fetch_cloudflare_domains` to create or update one or more + :model:`shepherd.Domain`. + """ + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``Cloudflare Update`` + result = "success" + try: + task_id = async_task( + "ghostwriter.shepherd.tasks.fetch_cloudflare_domains", + group="Cloudflare Update", + hook="ghostwriter.modules.notifications_slack.send_slack_complete_msg", + ) + message = "Successfully queued Cloudflare update task (Task ID {task}).".format(task=task_id) + except Exception: + result = "error" + message = "Cloudflare update task could not be queued!" + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) class MonitorCloudInfrastructure(RoleBasedAccessControlMixin, View): """ @@ -702,6 +741,16 @@ def update(request): End time of latest :model:`django_q.Task` for group "Namecheap Update" ``namecheap_last_result`` Result of latest :model:`django_q.Task` for group "Namecheap Update" + ``enable_cloudflare`` + The associated value from :model:`commandcenter.CloudflareConfiguration` + ``cloudflare_last_update_requested`` + Start time of latest :model:`django_q.Task` for group "Cloudflare Update" + ``cloudflare_last_update_completed`` + End time of latest :model:`django_q.Task` for group "Cloudflare Update" + ``cloudflare_last_update_time`` + End time of latest :model:`django_q.Task` for group "Cloudflare Update" + ``cloudflare_last_result`` + Result of latest :model:`django_q.Task` for group "Cloudflare Update" ``enable_cloud_monitor`` The associated value from :model:`commandcenter.CloudServicesConfiguration` ``cloud_last_update_requested`` @@ -727,6 +776,8 @@ def update(request): enable_cloud_monitor = cloud_config.enable namecheap_config = NamecheapConfiguration.get_solo() enable_namecheap = namecheap_config.enable + cloudflare_config = CloudflareConfiguration.get_solo() + enable_cloudflare = cloudflare_config.enable # Collect data for category updates cat_last_update_completed = "" @@ -794,6 +845,27 @@ def update(request): else: namecheap_last_update_requested = "Namecheap Syncing is Disabled" + # Collect data for Cloudflare updates + cloudflare_last_update_completed = "" + cloudflare_last_update_time = "" + cloudflare_last_result = "" + if enable_cloudflare: + try: + queryset = Task.objects.filter(group="Cloudflare Update")[0] + cloudflare_last_update_requested = queryset.started + cloudflare_last_result = queryset.result + if queryset.success: + cloudflare_last_update_completed = queryset.stopped + cloudflare_last_update_time = round(queryset.time_taken() / 60, 2) + if cloudflare_last_result["errors"]: + cloudflare_last_update_completed = "Failed" + else: + cloudflare_last_update_completed = "Failed" + except IndexError: + cloudflare_last_update_requested = "Cloudflare Sync Has Not Been Run Yet" + else: + cloudflare_last_update_requested = "Cloudflare Syncing is Disabled" + # Collect data for cloud monitoring cloud_last_update_completed = "" cloud_last_update_time = "" @@ -831,6 +903,11 @@ def update(request): "namecheap_last_update_completed": namecheap_last_update_completed, "namecheap_last_update_time": namecheap_last_update_time, "namecheap_last_result": namecheap_last_result, + "enable_cloudflare": enable_cloudflare, + "cloudflare_last_update_requested": cloudflare_last_update_requested, + "cloudflare_last_update_completed": cloudflare_last_update_completed, + "cloudflare_last_update_time": cloudflare_last_update_time, + "cloudflare_last_result": cloudflare_last_result, "enable_cloud_monitor": enable_cloud_monitor, "cloud_last_update_requested": cloud_last_update_requested, "cloud_last_update_completed": cloud_last_update_completed, From bf078c6d8bf0ac6b1d7c5d04dee9c170adb5e38c Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:10:57 -0500 Subject: [PATCH 14/30] Update tasks.py --- ghostwriter/shepherd/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index 5d7c783dd..a7526d706 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -918,8 +918,8 @@ def fetch_cloudflare_domains(): try: client = Cloudflare( - api_email=os.environ.get("CLOUDFLARE_EMAIL"), - api_key=os.environ.get("CLOUDFLARE_API_KEY"), + api_email=cloudflare_config.api_username, + api_key=cloudflare_config.api_key, ) page = 1 From 68755c0f2810b9b7c142e4826dd4e9dd68e9c002 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:39:54 -0500 Subject: [PATCH 15/30] Update tasks.py --- ghostwriter/shepherd/tasks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index a7526d706..6268aa437 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -918,8 +918,7 @@ def fetch_cloudflare_domains(): try: client = Cloudflare( - api_email=cloudflare_config.api_username, - api_key=cloudflare_config.api_key, + api_token=cloudflare_config.api_token ) page = 1 From 445b00c756068af5e868397a699addefccbb31cc Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:42:13 -0500 Subject: [PATCH 16/30] Update models.py --- ghostwriter/commandcenter/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ghostwriter/commandcenter/models.py b/ghostwriter/commandcenter/models.py index b821d30a4..569414426 100644 --- a/ghostwriter/commandcenter/models.py +++ b/ghostwriter/commandcenter/models.py @@ -76,7 +76,7 @@ def sanitized_api_key(self): class CloudflareConfiguration(SingletonModel): enable = models.BooleanField(default=False) - api_key = models.CharField(max_length=255, default="Cloudflare API Key", help_text="Your Cloudflare API key") + api_token = models.CharField(max_length=255, default="Cloudflare API Token", help_text="Your Cloudflare API Token") username = models.CharField(max_length=255, default="Account Email", help_text="Your Cloudflare email") api_username = models.CharField( "Account ID", @@ -104,7 +104,7 @@ class Meta: @property def sanitized_api_key(self): - return sanitize(self.api_key) + return sanitize(self.api_token) class ReportConfiguration(SingletonModel): From 59c56ee5105d63f78917938c7efd0adc476f4f90 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:43:28 -0500 Subject: [PATCH 17/30] Update 0001_initial.py --- ghostwriter/commandcenter/migrations/0001_initial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ghostwriter/commandcenter/migrations/0001_initial.py b/ghostwriter/commandcenter/migrations/0001_initial.py index 245f986c0..bfa91bf29 100644 --- a/ghostwriter/commandcenter/migrations/0001_initial.py +++ b/ghostwriter/commandcenter/migrations/0001_initial.py @@ -95,7 +95,7 @@ class Migration(migrations.Migration): fields=[ ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("enable", models.BooleanField(default=False)), - ("api_key", models.CharField(default="Cloudflare API Key", max_length=255)), + ("api_token", models.CharField(default="Cloudflare API Token", max_length=255)), ("username", models.CharField(default="Account email", max_length=255)), ("api_username", models.CharField(default="Account ID", max_length=255)), ("client_ip", models.CharField(default="Whitelisted IP Address", max_length=255)), From 33decb6082ef9bfa38338b7127b6f6da5d7b98ab Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:44:15 -0500 Subject: [PATCH 18/30] Update tasks.py --- ghostwriter/shepherd/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index 6268aa437..e24b006d7 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -8,6 +8,7 @@ from collections import defaultdict from datetime import date, datetime, timedelta from math import ceil +from cloudflare import Cloudflare # Django Imports from django.db.models import Q From 06a15fdf7c37aefa54570bd89e557a3e5bea8ac2 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 13:45:39 -0500 Subject: [PATCH 19/30] Update base.txt --- requirements/base.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/base.txt b/requirements/base.txt index ae87b76bf..b20d48823 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,6 +6,7 @@ argon2-cffi==21.3.0 # https://github.com/hynek/argon2_cffi redis==3.5.3 # https://github.com/antirez/redis uvicorn[standard]==0.22.0 # https://github.com/encode/uvicorn Twisted==20.3.0 # Temporary constraint for channels: https://github.com/django/daphne/pull/359 +cloudflare==4.0.0 # Django # ------------------------------------------------------------------------------ From ffd3370029e8cf3b28b1fc981d9a8c64e369458c Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 14:36:54 -0500 Subject: [PATCH 20/30] Update urls.py --- ghostwriter/shepherd/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ghostwriter/shepherd/urls.py b/ghostwriter/shepherd/urls.py index 98fdbd189..fb0b8eb66 100644 --- a/ghostwriter/shepherd/urls.py +++ b/ghostwriter/shepherd/urls.py @@ -76,7 +76,7 @@ ), path( "ajax/update/cloudflare", - views.RegistrarSynccloudflare.as_view(), + views.RegistrarSyncCloudflare.as_view(), name="ajax_update_cloudflare", ), path( From c2f69f3c9ed3b3ba0a134198747c0fccf1d77df5 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 14:53:34 -0500 Subject: [PATCH 21/30] Update tasks.py --- ghostwriter/shepherd/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index e24b006d7..88a83df1d 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -917,6 +917,8 @@ def fetch_cloudflare_domains(): logger.info("Starting Cloudflare synchronization task at %s", datetime.now()) + cloudflare_config = CloudflareConfiguration.get_solo() + try: client = Cloudflare( api_token=cloudflare_config.api_token From deec6308a792f5665f7faf4d2e3e003604ee7ff1 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:07:12 -0500 Subject: [PATCH 22/30] Update tasks.py --- ghostwriter/shepherd/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index 88a83df1d..ca791859e 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -24,6 +24,7 @@ from ghostwriter.commandcenter.models import ( CloudServicesConfiguration, NamecheapConfiguration, + CLoudflareConfiguration, VirusTotalConfiguration, ) from ghostwriter.modules.cloud_monitors import ( From 36e147ba6921f754fcc8b9e8a84c1cb7ebc26df9 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:34:01 -0500 Subject: [PATCH 23/30] Update tasks.py --- ghostwriter/shepherd/tasks.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index ca791859e..c7aeb66a3 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -926,30 +926,28 @@ def fetch_cloudflare_domains(): ) page = 1 - per_page = 50 # Adjust based on API limits + per_page = 20 # Adjust based on API limits total_pages = 1 - + while page <= total_pages: logger.info("Requesting page %s of %s", page, total_pages) response = client.zones.list(page=page, per_page=per_page) - + if response.success: - result_info = response.result_info - total_pages = ceil(result_info["total_count"] / result_info["per_page"]) - + total_pages = ceil(response.result_info.total_count / response.result_info.per_page) + for domain in response.result: domains_list.append({ - "id": domain["id"], - "name": domain["name"], - "type": domain.get("type", "unknown"), - "status": "active" if not domain["meta"].get("phishing_detected", False) else "burned", + "id": domain.id, + "name": domain.name, + "type": domain.type, + "status": "active" if not domain.meta.phishing_detected else "burned", }) else: - error_messages = " | ".join([error["message"] for error in response.errors]) + error_messages = " | ".join([error.message for error in response.errors]) logger.error("Cloudflare API returned errors: %s", error_messages) - domain_changes["errors"]["cloudflare"] = f"Cloudflare API returned errors: {error_messages}" - return domain_changes - + return {"errors": {"cloudflare": f"Cloudflare API returned errors: {error_messages}"}} + page += 1 except Exception: trace = traceback.format_exc() From fe6ce828c32602870b14bf7a32bcd157062786e1 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:44:54 -0500 Subject: [PATCH 24/30] Update tasks.py --- ghostwriter/shepherd/tasks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index c7aeb66a3..3bac74436 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -984,8 +984,8 @@ def fetch_cloudflare_domains(): domain_changes["updates"][domain.id] = {"domain": domain.name, "change": "renewed"} for domain in domains_list: - entry = {"name": domain["name"], "registrar": "Cloudflare"} - newly_burned = domain["status"] == "burned" + entry = {"name": domain.name, "registrar": "Cloudflare"} + newly_burned = domain.status == "burned" if newly_burned: entry["health_status"] = health_burned_status @@ -993,14 +993,14 @@ def fetch_cloudflare_domains(): entry["burned_explanation"] = "

Cloudflare detected potential phishing activity on this domain.

" try: - instance, created = Domain.objects.update_or_create(name=domain["name"], defaults=entry) + instance, created = Domain.objects.update_or_create(name=domain.name, defaults=entry) instance.save() change_status = "created & burned" if created and newly_burned else "burned" if newly_burned else "created" if created else "updated" - domain_changes["updates"][instance.id] = {"domain": domain["name"], "change": change_status} + domain_changes["updates"][instance.id] = {"domain": domain.name, "change": change_status} except Exception: trace = traceback.format_exc() - logger.exception("Failed to update domain %s", domain["name"]) - domain_changes["errors"][domain["name"]] = {"error": trace} + logger.exception("Failed to update domain %s", domain.name) + domain_changes["errors"][domain.name] = {"error": trace} logger.info("Cloudflare synchronization completed at %s with these changes:\n%s", datetime.now(), domain_changes) else: From 42e2b8390aedd1c294faf4d747aa9d084628ec15 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Mon, 10 Mar 2025 09:46:46 -0400 Subject: [PATCH 25/30] Update models.py --- ghostwriter/commandcenter/models.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ghostwriter/commandcenter/models.py b/ghostwriter/commandcenter/models.py index 569414426..561fdb073 100644 --- a/ghostwriter/commandcenter/models.py +++ b/ghostwriter/commandcenter/models.py @@ -76,9 +76,9 @@ def sanitized_api_key(self): class CloudflareConfiguration(SingletonModel): enable = models.BooleanField(default=False) - api_token = models.CharField(max_length=255, default="Cloudflare API Token", help_text="Your Cloudflare API Token") + api_key = models.CharField(max_length=255, default="Cloudflare API key", help_text="Your Cloudflare API Token") username = models.CharField(max_length=255, default="Account Email", help_text="Your Cloudflare email") - api_username = models.CharField( + account_id = models.CharField( "Account ID", max_length=255, default="Account ID", @@ -106,6 +106,16 @@ class Meta: def sanitized_api_key(self): return sanitize(self.api_token) + def __str__(self): + return "Cloudflare Configuration" + + class Meta: + verbose_name = "Cloudflare Configuration" + + @property + def sanitized_api_key(self): + return sanitize(self.api_token) + class ReportConfiguration(SingletonModel): enable_borders = models.BooleanField(default=False, help_text="Enable borders around images in Word documents") From 888abf8aa3e6dc2dcee4362f0e0745b1796aff6f Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Mon, 10 Mar 2025 10:05:08 -0400 Subject: [PATCH 26/30] Update 0001_initial.py --- ghostwriter/commandcenter/migrations/0001_initial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ghostwriter/commandcenter/migrations/0001_initial.py b/ghostwriter/commandcenter/migrations/0001_initial.py index bfa91bf29..855317bdc 100644 --- a/ghostwriter/commandcenter/migrations/0001_initial.py +++ b/ghostwriter/commandcenter/migrations/0001_initial.py @@ -95,9 +95,9 @@ class Migration(migrations.Migration): fields=[ ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("enable", models.BooleanField(default=False)), - ("api_token", models.CharField(default="Cloudflare API Token", max_length=255)), + ("api_key", models.CharField(default="Cloudflare Global API Key", max_length=255)), ("username", models.CharField(default="Account email", max_length=255)), - ("api_username", models.CharField(default="Account ID", max_length=255)), + ("account_id", models.CharField(default="Account ID", max_length=255)), ("client_ip", models.CharField(default="Whitelisted IP Address", max_length=255)), ("page_size", models.IntegerField(default=100)), ], From 346bec7f24d5f000e2747edb87f8755980a01896 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Mon, 10 Mar 2025 10:05:52 -0400 Subject: [PATCH 27/30] Update management.html --- .../home/templates/home/management.html | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/ghostwriter/home/templates/home/management.html b/ghostwriter/home/templates/home/management.html index 666e3dcd5..28b985716 100644 --- a/ghostwriter/home/templates/home/management.html +++ b/ghostwriter/home/templates/home/management.html @@ -137,6 +137,37 @@

API & Notification Configurations

Disabled {% endif %} + {% if cloudflare_config.enable %} + + Cloudflare API Enabled + {{ cloudflare_config.enable }} + + + Cloudflare Whitelisted IP + {{ cloudflare_config.client_ip }} + + + Cloudflare API Key + {{ cloudflare_config.sanitized_api_key }} + + + Cloudflare Email + {{ cloudflare_config.username }} + + + Cloudflare Account ID + {{ cloudflare_config.account_id }} + + + Cloudflare Page Size + {{ cloudflare_config.page_size }} + + {% else %} + + Cloudflare API Enabled + Disabled + + {% endif %} From aa156d08399023af5aca065387ea66860afa551c Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Thu, 17 Apr 2025 14:46:44 -0400 Subject: [PATCH 28/30] Updated cloudflare and added GCP Updated the cloudflare configs and added GCP infra monitoring --- .../migrations/0034_add_gcp_fields.py | 92 +++++++++ ghostwriter/commandcenter/models.py | 16 +- .../home/templates/home/management.html | 14 ++ ghostwriter/home/urls.py | 5 + ghostwriter/home/views.py | 44 ++++- ghostwriter/modules/cloud_monitors.py | 92 +++++++++ ghostwriter/shepherd/tasks.py | 177 ++++++++++++++++-- 7 files changed, 410 insertions(+), 30 deletions(-) create mode 100644 ghostwriter/commandcenter/migrations/0034_add_gcp_fields.py diff --git a/ghostwriter/commandcenter/migrations/0034_add_gcp_fields.py b/ghostwriter/commandcenter/migrations/0034_add_gcp_fields.py new file mode 100644 index 000000000..086aa72c6 --- /dev/null +++ b/ghostwriter/commandcenter/migrations/0034_add_gcp_fields.py @@ -0,0 +1,92 @@ +# Generated manually to add GCP credential fields + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("commandcenter", "0033_auto_20241204_1810"), # Replace with your actual previous migration + ] + + operations = [ + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_project_id", + field=models.CharField( + max_length=255, + default="your-project-id", + verbose_name="GCP Project ID", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_private_key_id", + field=models.CharField( + max_length=255, + default="your-private-key-id", + verbose_name="GCP Private Key ID", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_private_key", + field=models.TextField( + default="your-private-key", + verbose_name="GCP Private Key", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_client_email", + field=models.CharField( + max_length=255, + default="your-service-account@project.iam.gserviceaccount.com", + verbose_name="GCP Client Email", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_client_id", + field=models.CharField( + max_length=255, + default="your-client-id", + verbose_name="GCP Client ID", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_auth_uri", + field=models.CharField( + max_length=255, + default="https://accounts.google.com/o/oauth2/auth", + verbose_name="GCP Auth URI", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_token_uri", + field=models.CharField( + max_length=255, + default="https://oauth2.googleapis.com/token", + verbose_name="GCP Token URI", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_auth_cert_url", + field=models.CharField( + max_length=255, + default="https://www.googleapis.com/oauth2/v1/certs", + verbose_name="GCP Auth Provider Cert URL", + ), + ), + migrations.AddField( + model_name="cloudservicesconfiguration", + name="gcp_client_cert_url", + field=models.TextField( + default="https://www.googleapis.com/robot/v1/metadata/x509/your-service-account@project.iam.gserviceaccount.com", + verbose_name="GCP Client Cert URL", + ), + ), + ] diff --git a/ghostwriter/commandcenter/models.py b/ghostwriter/commandcenter/models.py index 561fdb073..97a062078 100644 --- a/ghostwriter/commandcenter/models.py +++ b/ghostwriter/commandcenter/models.py @@ -76,7 +76,7 @@ def sanitized_api_key(self): class CloudflareConfiguration(SingletonModel): enable = models.BooleanField(default=False) - api_key = models.CharField(max_length=255, default="Cloudflare API key", help_text="Your Cloudflare API Token") + api_key = models.CharField(max_length=255, default="Cloudflare API key", help_text="Your Cloudflare API key") username = models.CharField(max_length=255, default="Account Email", help_text="Your Cloudflare email") account_id = models.CharField( "Account ID", @@ -104,7 +104,7 @@ class Meta: @property def sanitized_api_key(self): - return sanitize(self.api_token) + return sanitize(self.api_key) def __str__(self): return "Cloudflare Configuration" @@ -114,7 +114,7 @@ class Meta: @property def sanitized_api_key(self): - return sanitize(self.api_token) + return sanitize(self.api_key) class ReportConfiguration(SingletonModel): @@ -323,6 +323,16 @@ class CloudServicesConfiguration(SingletonModel): default=7, help_text="Number of days to delay cloud monitoring notifications for teardown", ) + # GCP-Specific Configuration Fields + gcp_project_id = models.CharField("GCP Project ID", max_length=255, default="your-project-id") + gcp_private_key_id = models.CharField("GCP Private Key ID", max_length=255, default="your-private-key-id") + gcp_private_key = models.TextField("GCP Private Key", default="your-private-key") + gcp_client_email = models.CharField("GCP Client Email", max_length=255, default="your-service-account@example.iam.gserviceaccount.com") + gcp_client_id = models.CharField("GCP Client ID", max_length=255, default="your-client-id") + gcp_auth_uri = models.CharField("GCP Auth URI", max_length=255, default="https://accounts.google.com/o/oauth2/auth") + gcp_token_uri = models.CharField("GCP Token URI", max_length=255, default="https://oauth2.googleapis.com/token") + gcp_auth_cert_url = models.CharField("GCP Auth Provider Cert URL", max_length=255, default="https://www.googleapis.com/oauth2/v1/certs") + gcp_client_cert_url = models.TextField("GCP Client Cert URL", default="https://www.googleapis.com/robot/v1/metadata/x509/your-service-account@example.iam.gserviceaccount.com") def __str__(self): return "Cloud Services Configuration" diff --git a/ghostwriter/home/templates/home/management.html b/ghostwriter/home/templates/home/management.html index 28b985716..aeb9ba2cc 100644 --- a/ghostwriter/home/templates/home/management.html +++ b/ghostwriter/home/templates/home/management.html @@ -197,6 +197,14 @@

API & Notification Configurations

Digital Ocean API Key {{ cloud_config.sanitized_do_api_key }} + + GCP Project ID + {{ cloud_config.gcp_project_id }} + + + GCP Private Key ID + {{ cloud_config.gcp_private_key_id }} + {% else %} Cloud Monitoring Enabled @@ -341,12 +349,18 @@
Test Configurations
+
+ diff --git a/ghostwriter/home/urls.py b/ghostwriter/home/urls.py index b95e8f630..fa19821ff 100644 --- a/ghostwriter/home/urls.py +++ b/ghostwriter/home/urls.py @@ -27,6 +27,11 @@ views.TestDOConnection.as_view(), name="ajax_test_do", ), + path( + "ajax/management/test/gcp", + views.TestGCPConnection.as_view(), + name="ajax_test_gcp", + ), path( "ajax/management/test/namecheap", views.TestNamecheapConnection.as_view(), diff --git a/ghostwriter/home/views.py b/ghostwriter/home/views.py index 8ab6bf08b..cd3ffc826 100644 --- a/ghostwriter/home/views.py +++ b/ghostwriter/home/views.py @@ -47,12 +47,6 @@ def update_session(request): else: request.session["sidebar"] = {} request.session["sidebar"]["sticky"] = True - if req_data == "filter": - if "filter" in request.session.keys(): - request.session["filter"]["sticky"] ^= True - else: - request.session["filter"] = {} - request.session["filter"]["sticky"] = True request.session.save() data = { "result": "success", @@ -228,6 +222,40 @@ def post(self, request, *args, **kwargs): } return JsonResponse(data) +class TestGCPConnection(RoleBasedAccessControlMixin, View): + """ + Create an individual :model:`django_q.Task` under group ``GCP Test`` with + :task:`shepherd.tasks.test_gcp` to test the GCP info stored in + :model:`commandcenter.CloudServicesConfiguration`. + """ + + def test_func(self): + return verify_user_is_privileged(self.request.user) + + def handle_no_permission(self): + messages.error(self.request, "You do not have permission to access that.") + return redirect("home:dashboard") + + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``GCP Test`` + result = "success" + try: + async_task( + "ghostwriter.shepherd.tasks.test_gcp", + self.request.user, + group="GCP Test", + ) + message = "GCP test has been successfully queued." + except Exception: # pragma: no cover + result = "error" + message = "GCP test could not be queued." + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) + class TestNamecheapConnection(RoleBasedAccessControlMixin, View): """ @@ -266,7 +294,7 @@ def post(self, request, *args, **kwargs): class TestCloudflareConnection(RoleBasedAccessControlMixin, View): """ Create an individual :model:`django_q.Task` under group ``Cloudflare Test`` with - :task:`shepherd.tasks.test_Cloudflare` to test the Cloudflare API configuration stored + :task:`shepherd.tasks.test_cloudflare` to test the Cloudflare API configuration stored in :model:`commandcenter.CloudflareConfiguration`. """ @@ -282,7 +310,7 @@ def post(self, request, *args, **kwargs): result = "success" try: async_task( - "ghostwriter.shepherd.tasks.test_Cloudflare", + "ghostwriter.shepherd.tasks.test_cloudflare", self.request.user, group="Cloudflare Test", ) diff --git a/ghostwriter/modules/cloud_monitors.py b/ghostwriter/modules/cloud_monitors.py index 79d23e4f7..2460e7d20 100644 --- a/ghostwriter/modules/cloud_monitors.py +++ b/ghostwriter/modules/cloud_monitors.py @@ -11,6 +11,8 @@ import requests from botocore.config import Config from botocore.exceptions import ClientError, ConnectTimeoutError +from googleapiclient.discovery import build +from google.oauth2 import service_account # Using __name__ resolves to ghostwriter.modules.cloud_monitors logger = logging.getLogger(__name__) @@ -484,3 +486,93 @@ def fetch_digital_ocean(api_key, ignore_tags=None, do_only_running=False): ) return {"capable": capable, "message": message, "instances": instances} + +def format_gcp_private_key(raw_key: str) -> str: + """ + Normalize a GCP private key string to ensure it's in proper PEM format. + Handles keys stored with literal '\\n' as well as already-formatted PEM blocks. + """ + if not raw_key: + return "" + + if "\\n" in raw_key: + raw_key = raw_key.replace("\\n", "\n") + + raw_key = raw_key.strip() + + if not raw_key.startswith("-----BEGIN PRIVATE KEY-----"): + raw_key = f"-----BEGIN PRIVATE KEY-----\n{raw_key}" + if not raw_key.endswith("-----END PRIVATE KEY-----"): + raw_key = f"{raw_key}\n-----END PRIVATE KEY-----" + + return raw_key + +def fetch_gcp_instances(cloud_config): + """ + Fetches all virtual machine instances from GCP using values from the + CloudServicesConfiguration model. + """ + instances = {} + try: + private_key = format_gcp_private_key(cloud_config.gcp_private_key) + + service_account_info = { + "type": "service_account", + "project_id": cloud_config.gcp_project_id, + "private_key_id": cloud_config.gcp_private_key_id, + "private_key": private_key, + "client_email": cloud_config.gcp_client_email, + "client_id": cloud_config.gcp_client_id, + "auth_uri": cloud_config.gcp_auth_uri, + "token_uri": cloud_config.gcp_token_uri, + "auth_provider_x509_cert_url": cloud_config.gcp_auth_cert_url, + "client_x509_cert_url": cloud_config.gcp_client_cert_url + } + + credentials = service_account.Credentials.from_service_account_info(service_account_info) + compute = build('compute', 'v1', credentials=credentials, cache_discovery=False) + + # Get all zones + zones_request = compute.zones().list(project=cloud_config.gcp_project_id) + zones_response = zones_request.execute() + zones = zones_response.get('items', []) + + for zone in zones: + zone_name = zone['name'] + instance_request = compute.instances().list(project=cloud_config.gcp_project_id, zone=zone_name) + response = instance_request.execute() + + if 'items' in response: + for instance in response['items']: + instance_id = instance['id'] + name = instance['name'] + status = instance['status'] + tags = instance.get('tags', {}).get('items', []) + + internal_ip = None + external_ip = None + network_interfaces = instance.get('networkInterfaces', []) + if network_interfaces: + internal_ip = network_interfaces[0].get('networkIP') + access_configs = network_interfaces[0].get('accessConfigs', []) + if access_configs: + external_ip = access_configs[0].get('natIP') + + instances[instance_id] = { + "id": instance_id, + "name": name, + "provider": "gcp", + "zone": zone_name, + "state": status, + "tags": ",".join(tags), + "public_ip": [external_ip] if external_ip else [], + "private_ip": [internal_ip] if internal_ip else [], + "launch_time": instance.get("creationTimestamp", ""), + "ignore": False + } + + return {"message": "", "capable": True, "instances": instances} + + except Exception as e: + logger.exception("GCP instance fetch failed") + return {"message": str(e), "capable": False, "instances": {}} \ No newline at end of file diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index 3bac74436..3e8224cb1 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -9,6 +9,8 @@ from datetime import date, datetime, timedelta from math import ceil from cloudflare import Cloudflare +from googleapiclient.discovery import build +from google.oauth2 import service_account # Django Imports from django.db.models import Q @@ -24,7 +26,7 @@ from ghostwriter.commandcenter.models import ( CloudServicesConfiguration, NamecheapConfiguration, - CLoudflareConfiguration, + CloudflareConfiguration, VirusTotalConfiguration, ) from ghostwriter.modules.cloud_monitors import ( @@ -32,6 +34,7 @@ fetch_aws_lightsail, fetch_aws_s3, fetch_digital_ocean, + fetch_gcp_instances, test_aws, ) from ghostwriter.modules.dns_toolkit import DNSCollector @@ -70,6 +73,22 @@ def __call__(self, r): r.headers["Authorization"] = "Bearer " + self.token return r +def parse_iso_date(date_str): + """ + Convert ISO 8601 datetime string to a Python datetime.date object. + If the string is None or invalid, returns None. + """ + if not date_str: + return None + try: + # This handles the 'Z' suffix for UTC + return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ").date() + except ValueError: + try: + # Fallback if microseconds are missing + return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%SZ").date() + except Exception: + return None def namecheap_reset_dns(namecheap_config, domain): """ @@ -919,16 +938,17 @@ def fetch_cloudflare_domains(): logger.info("Starting Cloudflare synchronization task at %s", datetime.now()) cloudflare_config = CloudflareConfiguration.get_solo() - + try: client = Cloudflare( - api_token=cloudflare_config.api_token + api_email=cloudflare_config.username, + api_key=cloudflare_config.api_key ) - + page = 1 per_page = 20 # Adjust based on API limits total_pages = 1 - + while page <= total_pages: logger.info("Requesting page %s of %s", page, total_pages) response = client.zones.list(page=page, per_page=per_page) @@ -984,23 +1004,45 @@ def fetch_cloudflare_domains(): domain_changes["updates"][domain.id] = {"domain": domain.name, "change": "renewed"} for domain in domains_list: - entry = {"name": domain.name, "registrar": "Cloudflare"} - newly_burned = domain.status == "burned" - + # Use your Cloudflare account ID from config + account_id = cloudflare_config.account_id # Make sure you have this field in your model + + # Make the registrar lookup call + registrar_info = client.registrar.domains.get( + domain_name=domain["name"], + account_id=account_id + ) + + # Enrich the entry with registrar metadata + entry = { + "name": domain["name"], + "registrar": "Cloudflare", + "expiration": parse_iso_date(registrar_info.get("expires_at")), + "creation": parse_iso_date(registrar_info.get("created_at")), + } + + newly_burned = domain["status"] == "burned" + if newly_burned: entry["health_status"] = health_burned_status entry["domain_status"] = burned_status entry["burned_explanation"] = "

Cloudflare detected potential phishing activity on this domain.

" - + try: - instance, created = Domain.objects.update_or_create(name=domain.name, defaults=entry) + instance, created = Domain.objects.update_or_create(name=domain["name"], defaults=entry) instance.save() - change_status = "created & burned" if created and newly_burned else "burned" if newly_burned else "created" if created else "updated" - domain_changes["updates"][instance.id] = {"domain": domain.name, "change": change_status} + change_status = ( + "created & burned" if created and newly_burned else + "burned" if newly_burned else + "created" if created else + "updated" + ) + domain_changes["updates"][instance.id] = {"domain": domain["name"], "change": change_status} except Exception: trace = traceback.format_exc() - logger.exception("Failed to update domain %s", domain.name) - domain_changes["errors"][domain.name] = {"error": trace} + logger.exception("Failed to update domain %s", domain["name"]) + domain_changes["errors"][domain["name"]] = {"error": trace} + logger.info("Cloudflare synchronization completed at %s with these changes:\n%s", datetime.now(), domain_changes) else: @@ -1049,7 +1091,6 @@ def json_datetime_converter(dt): return str(dt) return None - def review_cloud_infrastructure(aws_only_running=False, do_only_running=False): """ Fetch active virtual machines/instances in Digital Ocean, Azure, and AWS and @@ -1132,6 +1173,21 @@ def review_cloud_infrastructure(aws_only_running=False, do_only_running=False): for instance in do_results["instances"]: vps_info["instances"][instance["id"]] = instance + ############### + # GCP Section # + ############### + + # Fetch GCP results using full cloud_config object + gcp_results = fetch_gcp_instances(cloud_config) + + if gcp_results["message"]: + vps_info["errors"]["gcp"] = gcp_results["message"] + else: + if gcp_results["capable"]: + for instance_id, instance in gcp_results["instances"].items(): + vps_info["instances"][instance_id] = instance + + ################## # Notifications # ################## @@ -1533,18 +1589,19 @@ def test_virustotal(user): logger.info("Test of the VirusTotal completed at %s", datetime.now()) return {"result": level, "message": message} -def test_cloudflare_api(user): +def test_cloudflare(user): """ Test the Cloudflare API configuration stored in environment variables stored in :model:`commandcenter.CloudflareConfiguration`. """ level = "error" logger.info("Starting Cloudflare API test at %s", datetime.now()) + cloudflare_config = CloudflareConfiguration.get_solo() try: client = Cloudflare( - api_email=os.environ.get("CLOUDFLARE_EMAIL"), - api_key=os.environ.get("CLOUDFLARE_API_KEY"), + api_email=cloudflare_config.username, + api_key=cloudflare_config.api_key ) - response = client.accounts.tokens.verify() + response = client.zones.list() if response.success: level = "success" message = "Successfully authenticated to Cloudflare" @@ -1555,3 +1612,85 @@ def test_cloudflare_api(user): logger.exception("Cloudflare API request failed") message = f"The Cloudflare API request failed: {trace}" return {"result": level, "message": message} + +def format_gcp_private_key(raw_key: str) -> str: + """ + Normalize a GCP private key string to ensure it's in proper PEM format. + Handles keys stored with literal '\\n' as well as already-formatted PEM blocks. + """ + if not raw_key: + return "" + + if "\\n" in raw_key: + raw_key = raw_key.replace("\\n", "\n") + + raw_key = raw_key.strip() + + if not raw_key.startswith("-----BEGIN PRIVATE KEY-----"): + raw_key = f"-----BEGIN PRIVATE KEY-----\n{raw_key}" + if not raw_key.endswith("-----END PRIVATE KEY-----"): + raw_key = f"{raw_key}\n-----END PRIVATE KEY-----" + + return raw_key + +def test_gcp(user): + """ + Test the GCP service account credentials configured in + :model:`commandcenter.CloudServicesConfiguration`. + """ + cloud_config = CloudServicesConfiguration.get_solo() + level = "error" + logger.info("Starting a test of the GCP credentials at %s", datetime.now()) + + private_key = format_gcp_private_key(cloud_config.gcp_private_key) + + try: + # Build credentials from individual variables (not file) + service_account_info = { + "type": "service_account", + "project_id": cloud_config.gcp_project_id, + "private_key_id": cloud_config.gcp_private_key_id, + "private_key": private_key, + "client_email": cloud_config.gcp_client_email, + "client_id": cloud_config.gcp_client_id, + "auth_uri": cloud_config.gcp_auth_uri, + "token_uri": cloud_config.gcp_token_uri, + "auth_provider_x509_cert_url": cloud_config.gcp_auth_cert_url, + "client_x509_cert_url": cloud_config.gcp_client_cert_url + } + + credentials = service_account.Credentials.from_service_account_info(service_account_info) + compute = build('compute', 'v1', credentials=credentials, cache_discovery=False) + + # Attempt a simple API call — list zones in the project + request = compute.zones().list(project=cloud_config.gcp_project_id) + response = request.execute() + + if "items" in response: + logger.info("GCP credentials are functional, beginning infrastructure review") + logger.info("Successfully verified the GCP service account credentials") + message = "Successfully verified the GCP service account credentials" + level = "success" + else: + logger.warning("GCP credentials did not return zone list as expected") + message = "GCP credentials did not return zone list as expected — please double-check access scopes and permissions" + + except Exception as e: + logger.exception("Testing authentication to GCP failed") + message = f"Testing authentication to GCP failed: {str(e)}" + + # Send a message to the requesting user + async_to_sync(channel_layer.group_send)( + f"notify_{user}", + { + "type": "message", + "message": { + "message": message, + "level": level, + "title": "GCP Test Complete", + }, + }, + ) + + logger.info("Test of the GCP credentials completed at %s", datetime.now()) + return {"result": level, "message": message} \ No newline at end of file From 603a381909d8de8ca34458f5ae60fc22f30dd531 Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Thu, 17 Apr 2025 15:11:42 -0400 Subject: [PATCH 29/30] Update base.txt updated base.txt to include required gcp libraries --- requirements/base.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements/base.txt b/requirements/base.txt index b20d48823..1eee7e89e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -7,6 +7,10 @@ redis==3.5.3 # https://github.com/antirez/redis uvicorn[standard]==0.22.0 # https://github.com/encode/uvicorn Twisted==20.3.0 # Temporary constraint for channels: https://github.com/django/daphne/pull/359 cloudflare==4.0.0 +google-api-python-client>=2.115.0 +google-auth>=2.25.0 +google-auth-httplib2>=0.2.0 +google-auth-oauthlib>=1.1.0 # Django # ------------------------------------------------------------------------------ From bc326cc190a2f3d96a2af84493adc55dd8df58af Mon Sep 17 00:00:00 2001 From: Alexander DeMine <53925659+ToweringDragoon@users.noreply.github.com> Date: Thu, 17 Apr 2025 15:32:12 -0400 Subject: [PATCH 30/30] Update cloud_monitors.py updated gcp info --- ghostwriter/modules/cloud_monitors.py | 29 ++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/ghostwriter/modules/cloud_monitors.py b/ghostwriter/modules/cloud_monitors.py index 2460e7d20..7fdcb83dd 100644 --- a/ghostwriter/modules/cloud_monitors.py +++ b/ghostwriter/modules/cloud_monitors.py @@ -513,6 +513,7 @@ def fetch_gcp_instances(cloud_config): CloudServicesConfiguration model. """ instances = {} + try: private_key = format_gcp_private_key(cloud_config.gcp_private_key) @@ -558,16 +559,38 @@ def fetch_gcp_instances(cloud_config): if access_configs: external_ip = access_configs[0].get('natIP') + # Parse launch time + raw_launch_time = instance.get("creationTimestamp") + launch_time = None + time_up = "Unknown" + if raw_launch_time: + try: + launch_time = datetime.strptime(raw_launch_time, "%Y-%m-%dT%H:%M:%S.%f%z") + now = datetime.now(timezone.utc) + delta = now - launch_time + time_up = f"{delta.days // 30} months" + except Exception: + logger.warning(f"Could not parse launch time: {raw_launch_time}") + + # Determine machine type (extract from URL) + machine_type_full = instance.get("machineType", "") + machine_type = machine_type_full.split("/")[-1] if "/" in machine_type_full else machine_type_full + instances[instance_id] = { "id": instance_id, "name": name, - "provider": "gcp", + "provider": "Google Cloud Platform", + "service": "Compute Engine", + "type": machine_type, + "monthly_cost": 0.0, # Can be updated later via billing API + "cost_to_date": 0.0, # Optional estimate if billing integrated "zone": zone_name, "state": status, - "tags": ",".join(tags), + "tags": ", ".join(tags), "public_ip": [external_ip] if external_ip else [], "private_ip": [internal_ip] if internal_ip else [], - "launch_time": instance.get("creationTimestamp", ""), + "launch_time": launch_time, + "time_up": time_up, "ignore": False }