{{ title }}
{{ subtitle }}
+ + {% endif %}{% endblock %} + {% block content %} + {% block object-tools %}{% endblock %} + {{ content }} + {% endblock %} + {% block sidebar %}{% endblock %} ++
diff --git a/apps/viz/__init__.py b/apps/viz/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/apps/viz/apps.py b/apps/viz/apps.py
new file mode 100644
index 0000000..33dcf49
--- /dev/null
+++ b/apps/viz/apps.py
@@ -0,0 +1,9 @@
+# viz/apps.py
+from django.apps import AppConfig
+from django.utils.translation import gettext_lazy as _
+
+
+class VizConfig(AppConfig):
+ default_auto_field = "django.db.models.BigAutoField"
+ name = "viz"
+ verbose_name = _("viz")
diff --git a/apps/viz/dash/__init__.py b/apps/viz/dash/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/apps/viz/dash/inventory_dashboard/__init__.py b/apps/viz/dash/inventory_dashboard/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/apps/viz/dash/inventory_dashboard/app.py b/apps/viz/dash/inventory_dashboard/app.py
new file mode 100644
index 0000000..2d8a605
--- /dev/null
+++ b/apps/viz/dash/inventory_dashboard/app.py
@@ -0,0 +1,317 @@
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from dash import Input, Output, dcc, html
+
+from ...dash_wrapper import SecureDjangoDash
+from .functions import (
+ get_bss_corrections,
+ get_livelihood_activity_dataframe,
+ get_livelihood_zone_baseline,
+ get_wealthcharactestics,
+)
+
+app = SecureDjangoDash("bss_inventory", suppress_callback_exceptions=True)
+app.title = "BSS Inventory"
+
+
+app.layout = html.Div(
+ [
+ html.Div([html.H1("BSS Inventories", style={"textAlign": "center", "color": "#2c3e50"})]),
+ html.Div(
+ [
+ html.Div(
+ [
+ html.Label("Select Country:", style={"fontWeight": "bold"}),
+ dcc.Dropdown(
+ id="country-dropdown",
+ # Options will be loaded dynamically in callback
+ options=[],
+ value=None,
+ clearable=True,
+ placeholder="Select a country (or leave blank for all)",
+ ),
+ ],
+ style={"width": "30%", "display": "inline-block", "marginRight": "20px"},
+ ),
+ html.Div(
+ [
+ html.Label("Select Livelihood Zone(s):", style={"fontWeight": "bold"}),
+ dcc.Dropdown(id="zone-dropdown", multi=True),
+ ],
+ style={"width": "65%", "display": "inline-block"},
+ ),
+ ],
+ style={"margin": "20px", "padding": "20px", "backgroundColor": "#f8f9fa", "borderRadius": "10px"},
+ ),
+ html.Div(
+ [dcc.Loading(id="loading-overview", type="circle", children=html.Div(id="overview-content"))],
+ style={"padding": "20px"},
+ ),
+ # Hidden div to trigger initial loading
+ html.Div(id="initial-load-trigger", style={"display": "none"}),
+ ],
+ style={"fontFamily": "Arial, sans-serif"},
+)
+
+
+@app.callback(
+ Output("country-dropdown", "options"),
+ Output("country-dropdown", "value"),
+ Input("initial-load-trigger", "children"),
+)
+def initialize_country_dropdown(_):
+ """Initialize country dropdown options on first load."""
+ try:
+ baseline_df = get_livelihood_zone_baseline()
+ countries = sorted(baseline_df["country"].unique().tolist()) if not baseline_df.empty else []
+ options = [{"label": country, "value": country} for country in countries]
+ default_value = countries[0] if countries else None
+ return options, default_value
+ except Exception as e:
+ # Handle gracefully in case of connection issues
+ print(f"Error loading baseline data: {e}")
+ return [], None
+
+
+@app.callback(Output("zone-dropdown", "options"), Output("zone-dropdown", "value"), Input("country-dropdown", "value"))
+def update_livelihood_zone_dropdown(selected_country):
+ """Update zone dropdown based on selected country."""
+ if not selected_country:
+ return [], []
+
+ try:
+ baseline_df = get_livelihood_zone_baseline()
+ zones = sorted(baseline_df[baseline_df["country"] == selected_country]["zone_code"].unique())
+ options = [{"label": zone, "value": zone} for zone in zones]
+ return options, zones
+ except Exception as e:
+ print(f"Error loading zone data: {e}")
+ return [], []
+
+
+@app.callback(
+ Output("overview-content", "children"), Input("country-dropdown", "value"), Input("zone-dropdown", "value")
+)
+def update_kpi_content(selected_country, selected_zone):
+ """Update main content based on selected filters."""
+ try:
+ # Load all required data fresh each time
+ baseline_df = get_livelihood_zone_baseline()
+ livelihood_activity_df = get_livelihood_activity_dataframe()
+ corrections_df = get_bss_corrections()
+ wealthcharactestics_df = get_wealthcharactestics()
+
+ title_segment = selected_country if selected_country else "All Countries"
+
+ if selected_country:
+ filtered_baseline_df = baseline_df[baseline_df["country"] == selected_country].copy()
+ if not livelihood_activity_df.empty:
+ filtered_activity_df = livelihood_activity_df[
+ livelihood_activity_df["country"] == selected_country
+ ].copy()
+ else:
+ filtered_activity_df = pd.DataFrame()
+ filtered_corrections_df = pd.DataFrame()
+ if not corrections_df.empty:
+ filtered_corrections_df = corrections_df[corrections_df["country"] == selected_country].copy()
+
+ most_frequent_wc = ""
+ if not wealthcharactestics_df.empty:
+ filtered_wc = wealthcharactestics_df[wealthcharactestics_df["country"] == selected_country].copy()
+ counts = filtered_wc["wealth_characteristic"].value_counts()
+ # Drop "household size" as it is in almost all bsses
+ counts = counts.drop("household size", errors="ignore")
+ if not counts.empty:
+ most_frequent_wc = counts.idxmax()
+
+ else:
+ filtered_baseline_df = baseline_df.copy()
+ filtered_activity_df = livelihood_activity_df.copy()
+ filtered_corrections_df = corrections_df.copy()
+ most_frequent_wc = ""
+ if not wealthcharactestics_df.empty:
+ counts = wealthcharactestics_df["wealth_characteristic"].value_counts()
+ # Drop "household size" as it is in almost all bsses
+ counts = counts.drop("household size", errors="ignore")
+ if not counts.empty:
+ most_frequent_wc = counts.idxmax()
+
+ if filtered_baseline_df.empty:
+ return html.Div(f"No data available for {title_segment}.", style={"padding": "20px"})
+
+ return create_kpi_layout(filtered_baseline_df, filtered_activity_df, filtered_corrections_df, most_frequent_wc)
+
+ except Exception as e:
+ print(f"Error updating content: {e}")
+ return html.Div(
+ "Error loading data. Please try again later.",
+ style={"padding": "20px", "color": "red", "textAlign": "center"},
+ )
+
+
+def create_livelihood_pie_chart(df):
+ if df.empty:
+ return go.Figure().update_layout(title_text="No Livelihood Category Data")
+ livelihood_counts = df["main_livelihood_category"].value_counts()
+ color_map = {
+ "Agricultural": "#096640", # brand green
+ "Agropastoral": "#0B7A4A", # slightly brighter
+ "Fishing": "#0E8E54", # lighter green
+ "Irrigation": "#12A35F", # medium green
+ "Pastoral": "#27B96B", # fresh green
+ "Peri-Urban": "#52CC85", # soft green
+ "Urban": "#7FE09F", # lightest green
+ }
+ fig = px.pie(
+ values=livelihood_counts.values,
+ names=livelihood_counts.index,
+ title="Distribution of Main Livelihood Categories",
+ hole=0.4,
+ color=livelihood_counts.index,
+ color_discrete_map=color_map,
+ )
+ fig.update_traces(textposition="inside", textinfo="percent+label")
+ fig.update_layout(legend_title_text="Category", uniformtext_minsize=12, uniformtext_mode="hide")
+ return fig
+
+
+def create_organization_treemap(df):
+ if df.empty:
+ return go.Figure().update_layout(title_text="No Source Organization Data")
+ org_counts = df["source_organization"].value_counts().reset_index()
+ org_counts.columns = ["source_organization", "count"]
+ fig = px.treemap(
+ org_counts,
+ path=[px.Constant("All Organizations"), "source_organization"],
+ values="count",
+ title="Data Contribution by Source Organization",
+ color="count",
+ color_continuous_scale="Blues",
+ )
+ fig.update_traces(textinfo="label+value")
+ fig.update_layout(margin=dict(t=50, l=25, r=25, b=25))
+ return fig
+
+
+def create_strategy_barchart(activity_df):
+ """
+ Creates a bar chart showing the count of livelihood activities by strategy type.
+ """
+ if activity_df.empty or "strategy_type" not in activity_df.columns:
+ return go.Figure().update_layout(title_text="No Livelihood Strategy Data")
+
+ strategy_counts = activity_df["strategy_type"].value_counts().reset_index()
+ strategy_counts.columns = ["strategy_type", "count"]
+
+ fig = px.bar(
+ strategy_counts,
+ x="strategy_type",
+ y="count",
+ title="Livelihood Activities by Strategy Type",
+ labels={"strategy_type": "Strategy Type", "count": "Number of Activities"},
+ color="strategy_type",
+ text_auto=True,
+ )
+ fig.update_layout(xaxis_title=None, yaxis_title="Number of Activities", showlegend=False)
+ fig.update_traces(textposition="outside")
+ return fig
+
+
+def get_livelihood_activity_kpis(activity_df):
+ if activity_df.empty:
+ return 0, 0
+ counts = activity_df["data_level"].value_counts().to_dict()
+ return counts.get("community", 0), counts.get("baseline", 0)
+
+
+def create_strategy_sunburst_chart(activity_df):
+ """
+ Creates a sunburst chart for livelihood strategy types.
+ """
+ if activity_df.empty or "strategy_type" not in activity_df.columns:
+ return go.Figure().update_layout(title_text="No Livelihood Activity Data Available")
+
+ strategy_counts = activity_df.groupby(["data_level", "strategy_type"]).size().reset_index(name="count")
+ fig = px.sunburst(
+ strategy_counts,
+ path=["data_level", "strategy_type"],
+ values="count",
+ title="Livelihood Activities by Strategy Type",
+ color="data_level",
+ color_discrete_map={"community": "#2980b9", "baseline": "#8e44ad", "(?)": "#7f8c8d"},
+ )
+ fig.update_traces(textinfo="label+percent parent", insidetextorientation="radial")
+ fig.update_layout(margin=dict(t=50, l=25, r=25, b=25))
+ return fig
+
+
+def create_kpi_layout(filtered_baseline_df, filtered_activity_df, filtered_corrections_df, most_frequent_wc):
+ """
+ All KPI cards and charts into a single layout component.
+ """
+
+ total_bsses = len(filtered_baseline_df)
+ community_count, baseline_count = get_livelihood_activity_kpis(filtered_activity_df)
+ total_corrections = len(filtered_corrections_df)
+ return html.Div(
+ [
+ html.Div(
+ [
+ html.Div(
+ [html.H3(f"{total_bsses}", style={"color": "#2ECC71"}), html.P("Total BSSes loaded")],
+ className="summary-card",
+ ),
+ html.Div(
+ [
+ html.H3("Livelihood Activity Data", style={"color": "#3498db"}),
+ html.Div(
+ [
+ html.Div([html.H4(community_count), html.P("Community")], className="sub-kpi"),
+ html.Div([html.H4(baseline_count), html.P("Baseline")], className="sub-kpi"),
+ ],
+ style={"display": "flex", "justifyContent": "space-around", "marginTop": "10px"},
+ ),
+ ],
+ className="summary-card",
+ ),
+ html.Div(
+ [html.H3(f"{total_corrections}", style={"color": "#9b59b6"}), html.P("Corrections")],
+ className="summary-card",
+ ),
+ html.Div(
+ [
+ html.H3("Most Frequent Wealth Characterstics", style={"color": "#f39c12"}),
+ html.P(f"{most_frequent_wc}"),
+ ],
+ className="summary-card",
+ ),
+ ],
+ className="kpi-card-container",
+ ),
+ html.Div(
+ [
+ html.Div(
+ [dcc.Graph(figure=create_livelihood_pie_chart(filtered_baseline_df))],
+ style={"width": "49%", "display": "inline-block", "verticalAlign": "top"},
+ ),
+ html.Div(
+ [dcc.Graph(figure=create_organization_treemap(filtered_baseline_df))],
+ style={"width": "49%", "display": "inline-block", "verticalAlign": "top"},
+ ),
+ ],
+ style={"marginTop": "30px"},
+ ),
+ html.Div(
+ [
+ html.Div(
+ [dcc.Graph(figure=create_strategy_sunburst_chart(filtered_activity_df))],
+ ),
+ html.Div(
+ [dcc.Graph(figure=create_strategy_barchart(filtered_activity_df))],
+ ),
+ ],
+ style={"marginTop": "30px"},
+ ),
+ ]
+ )
diff --git a/apps/viz/dash/inventory_dashboard/functions.py b/apps/viz/dash/inventory_dashboard/functions.py
new file mode 100644
index 0000000..5a1cfb7
--- /dev/null
+++ b/apps/viz/dash/inventory_dashboard/functions.py
@@ -0,0 +1,114 @@
+import pandas as pd
+
+from baseline.models import (
+ LivelihoodActivity,
+ LivelihoodZoneBaseline,
+ LivelihoodZoneBaselineCorrection,
+ WealthGroupCharacteristicValue,
+)
+
+
+def get_livelihood_zone_baseline():
+ qs = LivelihoodZoneBaseline.objects.select_related(
+ "livelihood_zone__country",
+ "source_organization",
+ ).values_list(
+ "id",
+ "livelihood_zone__country__name",
+ "livelihood_zone__name_en",
+ "source_organization__name",
+ "livelihood_zone__code",
+ "main_livelihood_category__name_en",
+ )
+ columns = [
+ "id",
+ "country",
+ "livelihood_zone",
+ "source_organization",
+ "zone_code",
+ "main_livelihood_category",
+ ]
+ df = pd.DataFrame(list(qs), columns=columns)
+
+ return df
+
+
+def get_livelihood_activity_dataframe():
+ queryset = LivelihoodActivity.objects.select_related(
+ "livelihood_strategy",
+ "livelihood_zone_baseline__livelihood_zone__country",
+ "wealth_group__community__livelihood_zone_baseline__source_organization",
+ "wealth_group__wealth_group_category",
+ ).values(
+ "id",
+ "livelihood_zone_baseline__livelihood_zone__country__name",
+ "livelihood_strategy__strategy_type",
+ "wealth_group",
+ "wealth_group__community",
+ )
+
+ df = pd.DataFrame(list(queryset))
+
+ df = df.rename(
+ columns={
+ "id": "activity_id",
+ "livelihood_zone_baseline__livelihood_zone__country__name": "country",
+ "livelihood_strategy__strategy_type": "strategy_type",
+ "wealth_group": "wealth_group_id",
+ "wealth_group__community": "community_id",
+ }
+ )
+ if df.empty:
+ return df
+ # Add baseline/community flag
+ df["data_level"] = df["community_id"].apply(lambda x: "baseline" if pd.isna(x) else "community")
+
+ return df
+
+
+def get_bss_corrections():
+ queryset = LivelihoodZoneBaselineCorrection.objects.select_related(
+ "livelihood_zone_baseline__livelihood_zone__country",
+ )
+
+ data = []
+ for correction in queryset:
+ data.append(
+ {
+ "id": correction.id,
+ "country": correction.livelihood_zone_baseline.livelihood_zone.country.name,
+ "bss": str(correction.livelihood_zone_baseline),
+ }
+ )
+
+ df = pd.DataFrame(data)
+ return df
+
+
+def get_wealthcharactestics():
+ queryset = (
+ WealthGroupCharacteristicValue.objects.select_related(
+ "wealth_group__livelihood_zone_baseline__livelihood_zone__country", "wealth_characteristic"
+ )
+ .exclude(value__isnull=True)
+ .exclude(value__exact="")
+ )
+
+ data = []
+ for wc in queryset:
+ wealth_characteristic = str(wc.wealth_characteristic)
+ if wc.wealth_characteristic.has_product:
+ wealth_characteristic += f":{wc.product}"
+
+ data.append(
+ {
+ "id": wc.id,
+ "country": wc.wealth_group.livelihood_zone_baseline.livelihood_zone.country.name,
+ "bss": str(wc.wealth_group.livelihood_zone_baseline),
+ "wealth_characteristic": wealth_characteristic,
+ "value": wc.value,
+ }
+ )
+
+ df = pd.DataFrame(data)
+ return df
diff --git a/apps/viz/dash/pipeline_status_dashboard/__init__.py b/apps/viz/dash/pipeline_status_dashboard/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/apps/viz/dash/pipeline_status_dashboard/app.py b/apps/viz/dash/pipeline_status_dashboard/app.py
new file mode 100644
index 0000000..4e35d1c
--- /dev/null
+++ b/apps/viz/dash/pipeline_status_dashboard/app.py
@@ -0,0 +1,378 @@
+import base64
+
+import graphviz
+import pandas as pd
+import plotly.graph_objects as go
+from dash import Input, Output, dcc, html
+from pipelines import defs
+
+from ...dash_wrapper import SecureDjangoDash
+from .functions import get_partion_status_dataframe
+
+app = SecureDjangoDash("pipeline_status_dashboard", suppress_callback_exceptions=True)
+app.title = "Pipeline Run Status"
+
+
+def get_pipeline_data():
+ df = get_partion_status_dataframe()
+ countries = sorted(df["country"].unique().tolist())
+ return df, countries
+
+
+# Layout with data stores and main content
+app.layout = html.Div(
+ [
+ dcc.Store(id="pipeline-data-store"),
+ dcc.Store(id="countries-store"),
+ dcc.Interval(id="init-interval", interval=100, n_intervals=0, max_intervals=1),
+ dcc.Loading(
+ id="loading-initial",
+ type="circle",
+ children=[html.Div(id="initialization-status")],
+ ),
+ html.Div(id="main-dashboard-content", style={"display": "none"}),
+ ],
+ style={"fontFamily": "Arial, sans-serif"},
+)
+
+
+# Separate callback for initial data loading
+@app.callback(
+ [
+ Output("pipeline-data-store", "data"),
+ Output("countries-store", "data"),
+ Output("initialization-status", "children"),
+ Output("main-dashboard-content", "style"),
+ ],
+ [Input("init-interval", "n_intervals")],
+ prevent_initial_call=False,
+)
+def load_initial_data(n_intervals):
+ # Load data only once when dashboard initializes
+ if n_intervals == 0:
+ return {}, [], html.Div("Initializing..."), {"display": "none"}
+
+ try:
+ df, countries = get_pipeline_data()
+
+ df_dict = df.to_dict("records")
+
+ return df_dict, countries, [], {"display": "block"}
+
+ except Exception as e:
+ error_msg = html.Div(
+ [
+ html.H4("Error Loading Dashboard", style={"color": "red", "textAlign": "center"}),
+ html.P(f"Failed to load pipeline data: {str(e)}", style={"textAlign": "center", "color": "red"}),
+ ]
+ )
+ return {}, [], error_msg, {"display": "none"}
+
+
+# Callback to populate main dashboard layout after data is loaded
+@app.callback(
+ Output("main-dashboard-content", "children"), [Input("countries-store", "data")], prevent_initial_call=True
+)
+def create_dashboard_layout(countries):
+ if not countries:
+ return html.Div("No countries data available")
+
+ return create_main_layout(countries)
+
+
+def create_main_layout(countries):
+ return html.Div(
+ [
+ # Header
+ html.Div(
+ [
+ html.H1(
+ "Pipeline Monitoring Dashboard",
+ style={"textAlign": "center", "color": "#2c3e50", "marginBottom": "10px"},
+ ),
+ ]
+ ),
+ # Controls
+ html.Div(
+ [
+ html.Div(
+ [
+ html.Label("Select Country:", style={"fontWeight": "bold"}),
+ dcc.Dropdown(
+ id="country-dropdown",
+ options=[{"label": country, "value": country} for country in countries],
+ value=countries[0] if countries else None,
+ clearable=False,
+ style={"marginTop": "5px"},
+ ),
+ ],
+ style={"width": "30%", "display": "inline-block", "marginRight": "20px"},
+ ),
+ html.Div(
+ [
+ html.Label("Select Partition(s):", style={"fontWeight": "bold"}),
+ dcc.Dropdown(id="partition-dropdown", multi=True, style={"marginTop": "5px"}),
+ ],
+ style={"width": "65%", "display": "inline-block"},
+ ),
+ ],
+ style={"margin": "20px", "padding": "20px", "backgroundColor": "#ecf0f1", "borderRadius": "10px"},
+ ),
+ html.Div(
+ [
+ dcc.Loading(id="loading-overview", type="circle", children=html.Div(id="overview-content")),
+ dcc.Loading(id="loading-matrix", type="circle", children=html.Div(id="matrix-content")),
+ dcc.Loading(id="loading-graph", type="circle", children=html.Div(id="graph-content")),
+ ],
+ style={"padding": "20px"},
+ ),
+ ]
+ )
+
+
+def get_df_from_store(stored_data):
+ if not stored_data:
+ return pd.DataFrame()
+ return pd.DataFrame(stored_data)
+
+
+# Updated callbacks to use stored data instead of global variables
+@app.callback(
+ [Output("partition-dropdown", "options"), Output("partition-dropdown", "value")],
+ [Input("country-dropdown", "value"), Input("pipeline-data-store", "data")],
+)
+def update_partition_dropdown(selected_country, stored_data):
+ if not selected_country or not stored_data:
+ return [], []
+
+ df = get_df_from_store(stored_data)
+ partitions = sorted(df[df["country"] == selected_country]["partition_key"].unique())
+ options = [{"label": partition, "value": partition} for partition in partitions]
+ return options, partitions
+
+
+@app.callback(
+ Output("overview-content", "children"), [Input("country-dropdown", "value"), Input("pipeline-data-store", "data")]
+)
+def update_kpi_content(selected_country, stored_data):
+ if not stored_data:
+ return html.Div("Loading data...", style={"padding": "20px"})
+
+ df = get_df_from_store(stored_data)
+ title_segment = selected_country if selected_country else "All Countries"
+
+ if selected_country:
+ data_df = df[df["country"] == selected_country].copy()
+ else:
+ data_df = df.copy()
+
+ if data_df.empty:
+ return html.Div(f"No data available for {title_segment}.", style={"padding": "20px"})
+
+ return create_kpi_layout(data_df, title_segment)
+
+
+@app.callback(
+ Output("matrix-content", "children"),
+ [Input("country-dropdown", "value"), Input("partition-dropdown", "value"), Input("pipeline-data-store", "data")],
+)
+def update_heat_map_content(selected_country, selected_partitions, stored_data):
+ if not stored_data:
+ return html.Div("Loading data...", style={"padding": "20px"})
+
+ if not selected_country:
+ return html.Div("Please select a country to see the matrix.", style={"padding": "20px"})
+
+ df = get_df_from_store(stored_data)
+ country_df = df[df["country"] == selected_country]
+
+ if selected_partitions:
+ country_df = country_df[country_df["partition_key"].isin(selected_partitions)]
+
+ print(country_df)
+ return create_heatmap(country_df, selected_country)
+
+
+@app.callback(
+ Output("graph-content", "children"),
+ [
+ Input("country-dropdown", "value"),
+ Input("partition-dropdown", "value"),
+ Input("pipeline-data-store", "data"),
+ ],
+)
+def update_graph_content(selected_country, selected_partitions, stored_data):
+ if not stored_data:
+ return html.Div("Loading data...", style={"padding": "20px"})
+
+ if not selected_country or not selected_partitions:
+ return html.Div("Please select a country and partition(s).", style={"padding": "20px", "textAlign": "center"})
+
+ df = get_df_from_store(stored_data)
+
+ if len(selected_partitions) == 1:
+ country_df = df[df["country"] == selected_country]
+ return create_or_update_graph_viz(country_df, selected_partitions[0])
+ else:
+ return html.Div(
+ "Please select a single partition to view its dependency graph.",
+ style={"padding": "20px", "textAlign": "center", "fontWeight": "bold", "color": "#7f8c8d"},
+ )
+
+
+def create_kpi_layout(country_df, title_segment):
+ # Create the KPI cards
+ total_statuses = len(country_df)
+ materialized_count = len(country_df[country_df["status"] == "Materialized"])
+ sucess_rate = (materialized_count / total_statuses * 100) if total_statuses > 0 else 0
+
+ country_df["last_run_date"] = pd.to_datetime(country_df["last_run_date"])
+ last_run = country_df["last_run_date"].max()
+ freshness = last_run.strftime("%Y-%m-%d %H:%M") if pd.notna(last_run) else "N/A"
+
+ total_runs = country_df["run_count"].sum()
+
+ failed_partitions = country_df[country_df["status"] == "Failed"]["partition_key"].value_counts()
+ most_failed_partition = failed_partitions.index[0] if not failed_partitions.empty else "None"
+
+ return html.Div(
+ [
+ # KPI Summary Cards
+ html.Div(
+ [
+ html.Div(
+ [
+ html.H3(f"{sucess_rate:.1f}%", style={"color": "#2ECC71"}),
+ html.P("Success Rate per Country"),
+ ],
+ className="summary-card",
+ ),
+ html.Div(
+ [
+ html.H3(freshness, style={"color": "#3498db"}),
+ html.P("Latest Data per Country"),
+ ],
+ className="summary-card",
+ ),
+ html.Div(
+ [
+ html.H3(f"{total_runs}", style={"color": "#9b59b6"}),
+ html.P("Total Runs for Country"),
+ ],
+ className="summary-card",
+ ),
+ html.Div(
+ [
+ html.H3(most_failed_partition, style={"color": "#f39c12", "fontSize": "1.2em"}),
+ html.P("Top Failing Partition of a Country"),
+ ],
+ className="summary-card",
+ ),
+ ],
+ className="kpi-card-container",
+ ),
+ ],
+ style={"padding": "20px"},
+ )
+
+
+def create_heatmap(country_df, country):
+ # create the matrix heatmap
+ fig_df = country_df.pivot(index="partition_key", columns="asset_key", values="status")
+ status_map = {"Materialized": 2, "Failed": -1, "Missing": 0, None: 0}
+ fig_df_numeric = fig_df.replace(status_map)
+ text_labels = fig_df_numeric.replace({-1: "Failed", 0: "Missing", 2: "Materialized"})
+
+ asset_order = []
+ asset_dependencies = {
+ asset.key.to_user_string(): [dep.to_user_string() for dep in asset.asset_deps.get(asset.key, [])]
+ for asset in defs.assets
+ }
+ while asset_dependencies:
+ no_deps = [key for key, deps in asset_dependencies.items() if not deps]
+ if not no_deps:
+ break
+ asset_order.extend(sorted(no_deps))
+ asset_dependencies = {
+ key: [dep for dep in deps if dep not in no_deps]
+ for key, deps in asset_dependencies.items()
+ if key not in no_deps
+ }
+
+ available_assets = [asset for asset in asset_order if asset in fig_df.columns]
+ fig_df_numeric = fig_df_numeric[available_assets]
+
+ heatmap_fig = go.Figure(
+ data=go.Heatmap(
+ z=fig_df_numeric.values,
+ x=fig_df_numeric.columns,
+ y=fig_df_numeric.index,
+ text=text_labels.values,
+ texttemplate="",
+ colorscale=[[0.0, "#FF6B6B"], [0.5, "#95A5A6"], [1.0, "#2ECC71"]],
+ zmid=0,
+ textfont={"size": 8},
+ hovertemplate="%{y}
Asset: %{x}
Status: %{text}
{{ value }}
\ No newline at end of file diff --git a/apps/viz/templates/viz/blocks/visualization.html b/apps/viz/templates/viz/blocks/visualization.html new file mode 100644 index 0000000..d5dd307 --- /dev/null +++ b/apps/viz/templates/viz/blocks/visualization.html @@ -0,0 +1,3 @@ +0?[0]:[]);if(o.enter().append("g").classed(h.containerClassName,!0).style("cursor","pointer"),o.exit().each((function(){n.select(this).selectAll("g."+h.headerGroupClassName).each(a)})).remove(),0!==r.length){var l=o.selectAll("g."+h.headerGroupClassName).data(r,p);l.enter().append("g").classed(h.headerGroupClassName,!0);for(var c=s.ensureSingle(o,"g",h.dropdownButtonGroupClassName,(function(t){t.style("pointer-events","all")})),u=0;u 90&&i.log("Long binary search..."),f-1},e.sorterAsc=function(t,e){return t-e},e.sorterDes=function(t,e){return e-t},e.distinctVals=function(t){var r,n=t.slice();for(n.sort(e.sorterAsc),r=n.length-1;r>-1&&n[r]===o;r--);for(var i,a=n[r]-n[0]||1,s=a/(r||1)/1e4,l=[],c=0;c<=r;c++){var u=n[c],h=u-i;void 0===i?(l.push(u),i=u):h>s&&(a=Math.min(a,h),l.push(u),i=u)}return{vals:l,minDiff:a}},e.roundUp=function(t,e,r){for(var n,i=0,a=e.length-1,o=0,s=r?0:1,l=r?1:0,c=r?Math.ceil:Math.floor;i0&&(n=1),r&&n)return t.sort(e)}return n?t:t.reverse()},e.findIndexOfMin=function(t,e){e=e||a;for(var r,n=1/0,i=0;il?r.y-l:0;return Math.sqrt(c*c+h*h)}for(var p=f(c);p;){if((c+=p+r)>h)return;p=f(c)}for(p=f(h);p;){if(c>(h-=p+r))return;p=f(h)}return{min:c,max:h,len:h-c,total:u,isClosed:0===c&&h===u&&Math.abs(n.x-i.x)<.1&&Math.abs(n.y-i.y)<.1}},e.findPointOnPath=function(t,e,r,n){for(var i,a,o,s=(n=n||{}).pathLength||t.getTotalLength(),l=n.tolerance||.001,c=n.iterationLimit||30,u=t.getPointAtLength(0)[r]>t.getPointAtLength(s)[r]?-1:1,h=0,f=0,p=s;h
/i;e.BR_TAG_ALL=/
/gi;var b=/(^|[\s"'])style\s*=\s*("([^"]*);?"|'([^']*);?')/i,w=/(^|[\s"'])href\s*=\s*("([^"]*)"|'([^']*)')/i,T=/(^|[\s"'])target\s*=\s*("([^"\s]*)"|'([^'\s]*)')/i,k=/(^|[\s"'])popup\s*=\s*("([\w=,]*)"|'([\w=,]*)')/i;function A(t,e){if(!t)return null;var r=t.match(e),n=r&&(r[3]||r[4]);return n&&C(n)}var M=/(^|;)\s*color:/;e.plainText=function(t,e){for(var r=void 0!==(e=e||{}).len&&-1!==e.len?e.len:1/0,n=void 0!==e.allowedTags?e.allowedTags:["br"],i=t.split(v),a=[],o="",s=0,l=0;l
"+l;e.text=c}(t,o,r,c):"log"===u?function(t,e,r,n,a){var o=t.dtick,l=e.x,c=t.tickformat,u="string"==typeof o&&o.charAt(0);if("never"===a&&(a=""),n&&"L"!==u&&(o="L3",u="L"),c||"L"===u)e.text=wt(Math.pow(10,l),t,a,n);else if(i(o)||"D"===u&&s.mod(l+.01,1)<.1){var h=Math.round(l),f=Math.abs(h),p=t.exponentformat;"power"===p||_t(p)&&bt(h)?(e.text=0===h?1:1===h?"10":"10"+(h>1?"":z)+f+"",e.fontSize*=1.25):("e"===p||"E"===p)&&f>2?e.text="1"+p+(h>0?"+":z)+f:(e.text=wt(Math.pow(10,l),t,"","fakehover"),"D1"===o&&"y"===t._id.charAt(0)&&(e.dy-=e.fontSize/6))}else{if("D"!==u)throw"unrecognized dtick "+String(o);e.text=String(Math.round(Math.pow(10,s.mod(l,1)))),e.fontSize*=.75}if("D1"===t.dtick){var d=String(e.text).charAt(0);"0"!==d&&"1"!==d||("y"===t._id.charAt(0)?e.dx-=e.fontSize/4:(e.dy+=e.fontSize/2,e.dx+=(t.range[1]>t.range[0]?1:-1)*e.fontSize*(l<0?.5:.25)))}}(t,o,0,c,g):"category"===u?function(t,e){var r=t._categories[Math.round(e.x)];void 0===r&&(r=""),e.text=String(r)}(t,o):"multicategory"===u?function(t,e,r){var n=Math.round(e.x),i=t._categories[n]||[],a=void 0===i[1]?"":String(i[1]),o=void 0===i[0]?"":String(i[0]);r?e.text=o+" - "+a:(e.text=a,e.text2=o)}(t,o,r):Rt(t)?function(t,e,r,n,i){if("radians"!==t.thetaunit||r)e.text=wt(e.x,t,i,n);else{var a=e.x/180;if(0===a)e.text="0";else{var o=function(t){function e(t,e){return Math.abs(t-e)<=1e-6}var r=function(t){for(var r=1;!e(Math.round(t*r)/r,t);)r*=10;return r}(t),n=t*r,i=Math.abs(function t(r,n){return e(n,0)?r:t(n,r%n)}(n,r));return[Math.round(n/i),Math.round(r/i)]}(a);if(o[1]>=100)e.text=wt(s.deg2rad(e.x),t,i,n);else{var l=e.x<0;1===o[1]?1===o[0]?e.text="π":e.text=o[0]+"π":e.text=["",o[0],"","⁄","",o[1],"","π"].join(""),l&&(e.text=z+e.text)}}}}(t,o,r,c,g):function(t,e,r,n,i){"never"===i?i="":"all"===t.showexponent&&Math.abs(e.x/t.dtick)<1e-6&&(i="hide"),e.text=wt(e.x,t,i,n)}(t,o,0,c,g),n||(t.tickprefix&&!m(t.showtickprefix)&&(o.text=t.tickprefix+o.text),t.ticksuffix&&!m(t.showticksuffix)&&(o.text+=t.ticksuffix)),t.labelalias&&t.labelalias.hasOwnProperty(o.text)){var y=t.labelalias[o.text];"string"==typeof y&&(o.text=y)}return("boundaries"===t.tickson||t.showdividers)&&(o.xbnd=[f(o.x-.5),f(o.x+t.dtick-.5)]),o},Z.hoverLabelText=function(t,e,r){r&&(t=s.extendFlat({},t,{hoverformat:r}));var n=s.isArrayOrTypedArray(e)?e[0]:e,i=s.isArrayOrTypedArray(e)?e[1]:void 0;if(void 0!==i&&i!==n)return Z.hoverLabelText(t,n,r)+" - "+Z.hoverLabelText(t,i,r);var a="log"===t.type&&n<=0,o=Z.tickText(t,t.c2l(a?-n:n),"hover").text;return a?0===n?"0":z+o:o};var xt=["f","p","n","μ","m","","k","M","G","T"];function _t(t){return"SI"===t||"B"===t}function bt(t){return t>14||t<-15}function wt(t,e,r,n){var a=t<0,o=e._tickround,l=r||e.exponentformat||"B",c=e._tickexponent,u=Z.getTickFormat(e),h=e.separatethousands;if(n){var f={exponentformat:l,minexponent:e.minexponent,dtick:"none"===e.showexponent?e.dtick:i(t)&&Math.abs(t)||1,range:"none"===e.showexponent?e.range.map(e.r2d):[0,t||1]};yt(f),o=(Number(f._tickround)||0)+4,c=f._tickexponent,e.hoverformat&&(u=e.hoverformat)}if(u)return e._numFormat(u)(t).replace(/-/g,z);var p,d=Math.pow(10,-o)/2;if("none"===l&&(c=0),(t=Math.abs(t))
")):x=f.textLabel;var C={x:f.traceCoordinate[0],y:f.traceCoordinate[1],z:f.traceCoordinate[2],data:b._input,fullData:b,curveNumber:b.index,pointNumber:T};d.appendArrayPointValue(C,b,T),t._module.eventData&&(C=b._module.eventData(C,f,b,{},T));var L={points:[C]};if(e.fullSceneLayout.hovermode){var I=[];d.loneHover({trace:b,x:(.5+.5*v[0]/v[3])*s,y:(.5-.5*v[1]/v[3])*l,xLabel:k.xLabel,yLabel:k.yLabel,zLabel:k.zLabel,text:x,name:u.name,color:d.castHoverOption(b,T,"bgcolor")||u.color,borderColor:d.castHoverOption(b,T,"bordercolor"),fontFamily:d.castHoverOption(b,T,"font.family"),fontSize:d.castHoverOption(b,T,"font.size"),fontColor:d.castHoverOption(b,T,"font.color"),nameLength:d.castHoverOption(b,T,"namelength"),textAlign:d.castHoverOption(b,T,"align"),hovertemplate:h.castOption(b,T,"hovertemplate"),hovertemplateLabels:h.extendFlat({},C,k),eventData:[C]},{container:n,gd:r,inOut_bbox:I}),C.bbox=I[0]}f.distance<5&&(f.buttons||w)?r.emit("plotly_click",L):r.emit("plotly_hover",L),this.oldEventData=L}else d.loneUnhover(n),this.oldEventData&&r.emit("plotly_unhover",this.oldEventData),this.oldEventData=void 0;e.drawAnnotations(e)},k.recoverContext=function(){var t=this;t.glplot.dispose();var e=function(){t.glplot.gl.isContextLost()?requestAnimationFrame(e):t.initializeGLPlot()?t.plot.apply(t,t.plotArgs):h.error("Catastrophic and unrecoverable WebGL error. Context lost.")};requestAnimationFrame(e)};var M=["xaxis","yaxis","zaxis"];function S(t,e,r){for(var n=t.fullSceneLayout,i=0;i<3;i++){var a=M[i],o=a.charAt(0),s=n[a],l=e[o],c=e[o+"calendar"],u=e["_"+o+"length"];if(h.isArrayOrTypedArray(l))for(var f,p=0;p<(u||l.length);p++)if(h.isArrayOrTypedArray(l[p]))for(var d=0;d
");y.text(_).attr("data-unformatted",_).call(u.convertToTspans,t),v=c.bBox(y.node())}y.attr("transform",i(-3,8-v.height)),g.insert("rect",".static-attribution").attr({x:-v.width-6,y:-v.height-3,width:v.width+6,height:v.height+3,fill:"rgba(255, 255, 255, 0.75)"});var b=1;v.width+6>x&&(b=x/(v.width+6));var w=[n.l+n.w*p.x[1],n.t+n.h*(1-p.y[0])];g.attr("transform",i(w[0],w[1])+a(b))}},e.updateFx=function(t){for(var e=t._fullLayout,r=e._subplots[f],n=0;n
");_.text(T).attr("data-unformatted",T).call(h.convertToTspans,t),b=u.bBox(_.node())}_.attr("transform",a(-3,8-b.height)),x.insert("rect",".static-attribution").attr({x:-b.width-6,y:-b.height-3,width:b.width+6,height:b.height+3,fill:"rgba(255, 255, 255, 0.75)"});var k=1;b.width+6>w&&(k=w/(b.width+6));var A=[n.l+n.w*f.x[1],n.t+n.h*(1-f.y[0])];x.attr("transform",a(A[0],A[1])+o(k))}},e.updateFx=function(t){for(var e=t._fullLayout,r=e._subplots[p],n=0;n