Skip to content

Commit 59ee6d2

Browse files
knudttyknudtty
authored
usage info (#846)
Ref: HDX-1692
1 parent 92401f2 commit 59ee6d2

File tree

13 files changed

+154
-42
lines changed

13 files changed

+154
-42
lines changed

‎.changeset/honest-taxis-deny.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@hyperdx/api": patch
3+
---
4+
5+
bring usage stats up to date

‎.env

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ ALL_IN_ONE_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-all-in-one
88
ALL_IN_ONE_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-all-in-one
99
OTEL_COLLECTOR_IMAGE_NAME=ghcr.io/hyperdxio/hyperdx-otel-collector
1010
OTEL_COLLECTOR_IMAGE_NAME_DOCKERHUB=hyperdx/hyperdx-otel-collector
11-
CHANGESET_TAG=2.0.0-beta.16
11+
CODE_VERSION=2.0.0-beta.16
1212
IMAGE_VERSION_SUB_TAG=.16
1313
IMAGE_VERSION=2-beta
1414
IMAGE_NIGHTLY_TAG=2-nightly

‎Makefile

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ build-local:
8787
--build-context hyperdx=./docker/hyperdx \
8888
--build-context api=./packages/api \
8989
--build-context app=./packages/app \
90+
--build-arg CODE_VERSION=${CODE_VERSION} \
9091
-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
9192
--target all-in-one-noauth
9293

@@ -98,6 +99,7 @@ build-all-in-one:
9899
--build-context hyperdx=./docker/hyperdx \
99100
--build-context api=./packages/api \
100101
--build-context app=./packages/app \
102+
--build-arg CODE_VERSION=${CODE_VERSION} \
101103
-t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
102104
--target all-in-one-auth
103105

@@ -107,6 +109,7 @@ build-app:
107109
--build-context hyperdx=./docker/hyperdx \
108110
--build-context api=./packages/api \
109111
--build-context app=./packages/app \
112+
--build-arg CODE_VERSION=${CODE_VERSION} \
110113
-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
111114
--target prod
112115

@@ -122,6 +125,7 @@ build-app-nightly:
122125
--build-context hyperdx=./docker/hyperdx \
123126
--build-context api=./packages/api \
124127
--build-context app=./packages/app \
128+
--build-arg CODE_VERSION=${CODE_VERSION} \
125129
-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \
126130
--target prod
127131

@@ -133,6 +137,7 @@ build-local-nightly:
133137
--build-context hyperdx=./docker/hyperdx \
134138
--build-context api=./packages/api \
135139
--build-context app=./packages/app \
140+
--build-arg CODE_VERSION=${CODE_VERSION} \
136141
-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \
137142
--target all-in-one-noauth
138143

@@ -144,6 +149,7 @@ build-all-in-one-nightly:
144149
--build-context hyperdx=./docker/hyperdx \
145150
--build-context api=./packages/api \
146151
--build-context app=./packages/app \
152+
--build-arg CODE_VERSION=${CODE_VERSION} \
147153
-t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \
148154
--target all-in-one-auth
149155

@@ -169,6 +175,7 @@ release-local:
169175
--build-context hyperdx=./docker/hyperdx \
170176
--build-context api=./packages/api \
171177
--build-context app=./packages/app \
178+
--build-arg CODE_VERSION=${CODE_VERSION} \
172179
--platform ${BUILD_PLATFORMS} \
173180
-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} \
174181
-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
@@ -187,6 +194,7 @@ release-all-in-one:
187194
--build-context hyperdx=./docker/hyperdx \
188195
--build-context api=./packages/api \
189196
--build-context app=./packages/app \
197+
--build-arg CODE_VERSION=${CODE_VERSION} \
190198
--platform ${BUILD_PLATFORMS} \
191199
-t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} \
192200
-t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
@@ -203,6 +211,7 @@ release-app:
203211
--build-context hyperdx=./docker/hyperdx \
204212
--build-context api=./packages/api \
205213
--build-context app=./packages/app \
214+
--build-arg CODE_VERSION=${CODE_VERSION} \
206215
--platform ${BUILD_PLATFORMS} \
207216
-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION}${IMAGE_VERSION_SUB_TAG} \
208217
-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_VERSION} \
@@ -228,6 +237,7 @@ release-app-nightly:
228237
--build-context hyperdx=./docker/hyperdx \
229238
--build-context api=./packages/api \
230239
--build-context app=./packages/app \
240+
--build-arg CODE_VERSION=${CODE_VERSION} \
231241
--platform ${BUILD_PLATFORMS} \
232242
-t ${IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \
233243
--target prod \
@@ -243,6 +253,7 @@ release-local-nightly:
243253
--build-context hyperdx=./docker/hyperdx \
244254
--build-context api=./packages/api \
245255
--build-context app=./packages/app \
256+
--build-arg CODE_VERSION=${CODE_VERSION} \
246257
--platform ${BUILD_PLATFORMS} \
247258
-t ${LOCAL_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \
248259
--target all-in-one-noauth \
@@ -258,6 +269,7 @@ release-all-in-one-nightly:
258269
--build-context hyperdx=./docker/hyperdx \
259270
--build-context api=./packages/api \
260271
--build-context app=./packages/app \
272+
--build-arg CODE_VERSION=${CODE_VERSION} \
261273
--platform ${BUILD_PLATFORMS} \
262274
-t ${ALL_IN_ONE_IMAGE_NAME_DOCKERHUB}:${IMAGE_NIGHTLY_TAG} \
263275
--target all-in-one-auth \

‎README.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,15 @@ Here's a high-level list of support we're working on delivering as part of v2:
205205
- [ ] v1 Migration Tooling
206206
- [ ] Public API
207207

208+
## HyperDX Usage Data
209+
210+
HyperDX collects anonymized usage data for open source deployments. This data
211+
supports our mission for observability to be available to any team and helps
212+
support our open source product run in a variety of different environments.
213+
While we hope you will continue to support our mission in this way, you may opt
214+
out of usage data collection by setting the `USAGE_STATS_ENABLED` environment
215+
variable to `false`. Thank you for supporting the development of HyperDX!
216+
208217
## License
209218

210219
[MIT](/LICENSE)

‎docker/hyperdx/Dockerfile

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ RUN rm -rf node_modules && yarn workspaces focus @hyperdx/api @hyperdx/app --pro
5656
# prod ############################################################################################
5757
FROM node:${NODE_VERSION}-alpine AS prod
5858

59+
ARG CODE_VERSION
60+
61+
ENV CODE_VERSION=$CODE_VERSION
5962
ENV NODE_ENV production
6063

6164
# Install libs used for the start script
@@ -86,6 +89,9 @@ ENTRYPOINT ["sh", "/etc/local/entry.sh"]
8689
# all-in-one base ############################################################################################
8790
FROM scratch AS all-in-one-base
8891

92+
ARG CODE_VERSION
93+
94+
ENV CODE_VERSION=$CODE_VERSION
8995
# Copy from clickhouse and otel collector bases
9096
COPY --from=clickhouse_base / /
9197
COPY --from=otel_collector_base --chmod=755 /otelcol-contrib /otelcontribcol

‎packages/api/src/api-app.ts

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ import compression from 'compression';
22
import MongoStore from 'connect-mongo';
33
import express from 'express';
44
import session from 'express-session';
5-
import ms from 'ms';
65
import onHeaders from 'on-headers';
76

87
import * as config from './config';
@@ -72,10 +71,7 @@ app.use(defaultCors);
7271
// ----------------------- Background Jobs -----------------------------
7372
// ---------------------------------------------------------------------
7473
if (config.USAGE_STATS_ENABLED) {
75-
void usageStats();
76-
setInterval(() => {
77-
void usageStats();
78-
}, ms('4h'));
74+
usageStats();
7975
}
8076
// ---------------------------------------------------------------------
8177

‎packages/api/src/config.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ export const NODE_ENV = env.NODE_ENV as string;
1010
export const APP_TYPE = (env.APP_TYPE || DEFAULT_APP_TYPE) as
1111
| 'api'
1212
| 'scheduled-task';
13-
export const CODE_VERSION = env.CODE_VERSION as string;
13+
export const CODE_VERSION = env.CODE_VERSION ?? '';
1414
export const EXPRESS_SESSION_SECRET = (env.EXPRESS_SESSION_SECRET ||
1515
DEFAULT_EXPRESS_SESSION) as string;
1616
export const FRONTEND_URL = (env.FRONTEND_URL ||

‎packages/api/src/models/source.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ export interface ISource extends Omit<TSource, 'connection'> {
1212
connection: ObjectId | string;
1313
}
1414

15+
export type SourceDocument = mongoose.HydratedDocument<ISource>;
16+
1517
export const Source = mongoose.model<ISource>(
1618
'Source',
1719
new Schema<ISource>(

‎packages/api/src/routers/api/clickhouseProxy.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { createProxyMiddleware } from 'http-proxy-middleware';
33
import { z } from 'zod';
44
import { validateRequest } from 'zod-express-middleware';
55

6+
import { CODE_VERSION } from '@/config';
67
import { getConnectionById } from '@/controllers/connection';
78
import { getNonNullUserWithTeam } from '@/middleware/auth';
89
import { validateRequestHeaders } from '@/middleware/validation';
@@ -120,7 +121,9 @@ const proxyMiddleware: RequestHandler =
120121
},
121122
on: {
122123
proxyReq: (proxyReq, _req) => {
123-
const newPath = _req.params[0];
124+
// set user-agent to the hyperdx version identifier
125+
proxyReq.setHeader('user-agent', `hyperdx ${CODE_VERSION}`);
126+
124127
// @ts-expect-error _req.query is type ParamQs, which doesn't play nicely with URLSearchParams. TODO: Replace with getting query params from _req.url eventually
125128
const qparams = new URLSearchParams(_req.query);
126129

@@ -139,6 +142,7 @@ const proxyMiddleware: RequestHandler =
139142
// TODO: Use fixRequestBody after this issue is resolved: https://github.com/chimurai/http-proxy-middleware/issues/1102
140143
proxyReq.write(_req.body);
141144
}
145+
const newPath = _req.params[0];
142146
proxyReq.path = `/${newPath}?${qparams}`;
143147
},
144148
proxyRes: (proxyRes, _req, res) => {

‎packages/api/src/routers/api/me.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import express from 'express';
22

3+
import { USAGE_STATS_ENABLED } from '@/config';
34
import { getTeam } from '@/controllers/team';
45
import { Api404Error } from '@/utils/errors';
56

@@ -29,6 +30,7 @@ router.get('/', async (req, res, next) => {
2930
id,
3031
name,
3132
team,
33+
usageStatsEnabled: USAGE_STATS_ENABLED,
3234
});
3335
} catch (e) {
3436
next(e);

‎packages/api/src/tasks/usageStats.ts

Lines changed: 102 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
import type { ResponseJSON } from '@clickhouse/client';
2+
import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse';
3+
import { MetricsDataType, SourceKind } from '@hyperdx/common-utils/dist/types';
24
import * as HyperDX from '@hyperdx/node-opentelemetry';
5+
import ms from 'ms';
36
import os from 'os';
47
import winston from 'winston';
58

6-
import * as clickhouse from '@/clickhouse';
79
import * as config from '@/config';
10+
import Connection from '@/models/connection';
11+
import { Source, SourceDocument } from '@/models/source';
812
import Team from '@/models/team';
913
import User from '@/models/user';
1014

@@ -13,45 +17,104 @@ const logger = winston.createLogger({
1317
format: winston.format.json(),
1418
transports: [
1519
HyperDX.getWinstonTransport('info', {
16-
apiKey: '3f26ffad-14cf-4fb7-9dc9-e64fa0b84ee0', // hyperdx usage stats service api key
20+
headers: {
21+
Authorization: '3f26ffad-14cf-4fb7-9dc9-e64fa0b84ee0', // hyperdx usage stats service api key
22+
},
1723
baseUrl: 'https://in-otel.hyperdx.io/v1/logs',
1824
maxLevel: 'info',
1925
service: 'hyperdx-oss-usage-stats',
20-
} as any),
26+
}),
2127
],
2228
});
2329

30+
function extractTableNames(source: SourceDocument): string[] {
31+
const tables: string[] = [];
32+
if (source.kind === SourceKind.Metric) {
33+
for (const key of Object.values(MetricsDataType)) {
34+
const metricTable = source.metricTables?.[key];
35+
if (!metricTable) continue;
36+
tables.push(metricTable);
37+
}
38+
} else {
39+
tables.push(source.from.tableName);
40+
}
41+
return tables;
42+
}
43+
2444
const getClickhouseTableSize = async () => {
25-
const rows = await clickhouse.client.query({
26-
query: `
27-
SELECT
28-
table,
29-
sum(bytes) AS size,
30-
sum(rows) AS rows,
31-
min(min_time) AS min_time,
32-
max(max_time) AS max_time,
33-
max(modification_time) AS latestModification,
34-
toUInt32((max_time - min_time) / 86400) AS days,
35-
size / ((max_time - min_time) / 86400) AS avgDaySize
36-
FROM system.parts
37-
WHERE active
38-
AND database = 'default'
39-
AND (table = {table1: String} OR table = {table2: String} OR table = {table3: String})
40-
GROUP BY table
41-
ORDER BY rows DESC
42-
`,
43-
format: 'JSON',
44-
query_params: {
45-
table1: clickhouse.TableName.LogStream,
46-
table2: clickhouse.TableName.Rrweb,
47-
table3: clickhouse.TableName.Metric,
48-
},
49-
});
50-
const result = await rows.json<ResponseJSON<any>>();
51-
return result.data;
45+
// fetch mongo data
46+
const connections = await Connection.find();
47+
const sources = await Source.find();
48+
49+
// build map for each db instance
50+
const distributedTableMap = new Map<string, string[]>();
51+
for (const source of sources) {
52+
const key = `${source.connection.toString()},${source.from.databaseName}`;
53+
if (distributedTableMap.has(key)) {
54+
distributedTableMap.get(key)?.push(...extractTableNames(source));
55+
} else {
56+
distributedTableMap.set(key, extractTableNames(source));
57+
}
58+
}
59+
60+
// fetch usage data
61+
const results: any[] = [];
62+
for (const [key, tables] of distributedTableMap) {
63+
const [connectionId, dbName] = key.split(',');
64+
const tableListString = tables
65+
.map((_, idx) => `table = {table${idx}: String}`)
66+
.join(' OR ');
67+
const query_params = tables.reduce(
68+
(acc, table, idx) => {
69+
acc[`table${idx}`] = table;
70+
return acc;
71+
},
72+
{} as { [key: string]: string },
73+
);
74+
query_params.dbName = dbName;
75+
76+
// find connection
77+
const connection = connections.find(c => c.id === connectionId);
78+
if (!connection) continue;
79+
80+
// query clickhouse
81+
try {
82+
const clickhouseClient = new ClickhouseClient({
83+
host: connection.host,
84+
username: connection.username,
85+
password: connection.password,
86+
});
87+
const _rows = await clickhouseClient.query({
88+
query: `
89+
SELECT
90+
table,
91+
sum(bytes) AS size,
92+
sum(rows) AS rows,
93+
min(min_time) AS min_time,
94+
max(max_time) AS max_time,
95+
max(modification_time) AS latestModification,
96+
toUInt32((max_time - min_time) / 86400) AS days,
97+
size / ((max_time - min_time) / 86400) AS avgDaySize
98+
FROM system.parts
99+
WHERE active
100+
AND database = {dbName: String}
101+
AND (${tableListString})
102+
GROUP BY table
103+
ORDER BY rows DESC
104+
`,
105+
format: 'JSON',
106+
query_params,
107+
});
108+
const res = await _rows.json<ResponseJSON<any>>();
109+
results.push(...res.data);
110+
} catch (error) {
111+
// ignore
112+
}
113+
}
114+
return results;
52115
};
53116

54-
export default async () => {
117+
async function getUsageStats() {
55118
try {
56119
const nowInMs = Date.now();
57120
const [userCounts, team, chTables] = await Promise.all([
@@ -99,4 +162,11 @@ export default async () => {
99162
} catch (err) {
100163
// ignore
101164
}
102-
};
165+
}
166+
167+
export default function () {
168+
void getUsageStats();
169+
setInterval(() => {
170+
void getUsageStats();
171+
}, ms('4h'));
172+
}

0 commit comments

Comments
 (0)