Skip to content

Commit f387161

Browse files
davidabramAnte-Koceicdejan-crocoderivke995
authored
[TOOL-156] Unify Transform (#306)
* init * fixes * changes? * added fks * create big select statement * added mr size * hunks and codegen * getTransformDates * Using UTC values and added week * add getTimelineReviewDepth (incorrect implementation) * getTimelineApproved getTimelineReviewed * getTimelineStartedCodingAt * add startedCodingAt, reviewDepth, approved and reviewed to run function * very good code * fix * Fixed tests * add review time and review depth * add start pickup time * timeline reviewed and approved * add pickup and review times to date dereferentiation * specify columns and cleanup old stuff * fix types and learn more why types arent attractive * Added inserting data into database, lastUpdatedAt in map, missing return values * Calculating time durations * Updated transform metrics data * _ blues * Fixed typos * Added transactions and changes some typings * fix pickupAt? * reset extract migrations * migrate null rows * add seed script; insert null rows? Co-authored-by: ivke995 <ivanivic842@gmail.com> * fix pickupat when no pickup event * fix from in diffs select; remove console.log diffs? * connect transform api to queue which runs the transform timeline per MR Co-authored-by: ivke995 <ivanivic842@gmail.com> * fix tests * fix calculateDuration, add author forge user --------- Co-authored-by: Ante Koceić <ante@crocoder.dev> Co-authored-by: Dejan Tot <dejan@crocoder.dev> Co-authored-by: ivke995 <ivanivic842@gmail.com>
1 parent b5eac5d commit f387161

28 files changed

+2386
-43
lines changed

‎apps/stack/.sst/types/index.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,13 @@ declare module "sst/node/config" {
102102
value: string;
103103
}
104104
}
105+
}import "sst/node/queue";
106+
declare module "sst/node/queue" {
107+
export interface QueueResources {
108+
"TransformTestQueue": {
109+
queueUrl: string;
110+
}
111+
}
105112
}import "sst/node/api";
106113
declare module "sst/node/api" {
107114
export interface ApiResources {

‎apps/stack/src/transform/transform-timeline.ts‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,11 @@ import * as extract from "@acme/extract-schema";
44
import * as transform from "@acme/transform-schema";
55
import { drizzle } from "drizzle-orm/libsql";
66
import { createClient } from "@libsql/client";
7-
import type { Context, ExtractEntities, TransformEntities } from "@acme/transform-functions";
7+
import type { Context, ExtractEntities, TransformEntities, TransformDatabase, ExtractDatabase } from "@acme/transform-functions";
8+
import { run } from "@acme/transform-functions";
89
import { Config } from "sst/node/config";
10+
import { createMessage } from "@stack/config/create-message";
11+
import type { SQSEvent } from "aws-lambda";
912

1013
const apiContextSchema = z.object({
1114
authorizer: z.object({
@@ -32,6 +35,20 @@ const context = {
3235
}
3336
} satisfies Context<Partial<ExtractEntities>, Partial<TransformEntities>>;
3437

38+
const timelineMessageSchema = z.object({
39+
content: z.object({
40+
mergeRequestId: z.number(),
41+
}),
42+
metadata: z.object({}),
43+
kind: z.string()
44+
});
45+
const timelineMessage = createMessage({
46+
kind: 'transform-timeline',
47+
contentShape: timelineMessageSchema.shape.content.shape,
48+
metadataShape: timelineMessageSchema.shape.metadata.shape,
49+
queueId: 'TransformTestQueue',
50+
});
51+
3552
export const apiHandler = ApiHandler(async (ev) => {
3653

3754
const lambdaContextValidation = apiContextSchema.safeParse(ev.requestContext);
@@ -51,9 +68,25 @@ export const apiHandler = ApiHandler(async (ev) => {
5168
body: JSON.stringify({ error: new Error("No extracted merge request found").toString() }),
5269
}
5370

54-
// TODO: This endpoint shouldn't be a teapot
71+
await timelineMessage.sendAll(allMergeRequests, {});
72+
5573
return {
56-
statusCode: 418,
57-
body: JSON.stringify({ message: 'teapot' })
74+
statusCode: 200,
75+
body: JSON.stringify({ message: 'started transform' })
76+
};
77+
});
78+
79+
export const queueHandler = async (event:SQSEvent) => {
80+
if (event.Records.length > 1) console.warn('WARNING: QueueHandler should process 1 message but got', event.Records.length);
81+
for (const record of event.Records) {
82+
const messageValidationResult = timelineMessageSchema.safeParse(JSON.parse(record.body));
83+
if (!messageValidationResult.success) continue;
84+
85+
const messageContent = messageValidationResult.data.content;
86+
87+
await run(messageContent.mergeRequestId, {
88+
extractDatabase: context.extract.db as ExtractDatabase,
89+
transformDatabase: context.transform.db as TransformDatabase,
90+
});
5891
}
59-
});
92+
}

‎apps/stack/stacks/TransformStack.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
Config,
44
type StackContext,
55
Api,
6+
Queue
67
} from "sst/constructs";
78
import { ExtractStack } from "./ExtractStack";
89
import { z } from "zod";
@@ -71,6 +72,28 @@ export function TransformStack({ stack }: StackContext) {
7172
}
7273
});
7374

75+
const transformTestingQueue = new Queue(stack, "TransformTestQueue");
76+
transformTestingQueue.addConsumer(stack, {
77+
cdk: {
78+
eventSource: {
79+
batchSize: 1,
80+
maxConcurrency: 20,
81+
},
82+
},
83+
function: {
84+
bind: [
85+
TRANSFORM_DATABASE_URL,
86+
TRANSFORM_DATABASE_AUTH_TOKEN,
87+
EXTRACT_DATABASE_URL,
88+
EXTRACT_DATABASE_AUTH_TOKEN,
89+
CRAWL_DATABASE_AUTH_TOKEN,
90+
CRAWL_DATABASE_URL,
91+
],
92+
handler: "src/transform/transform-timeline.queueHandler",
93+
},
94+
95+
})
96+
7497
const ENVSchema = z.object({
7598
CLERK_JWT_ISSUER: z.string(),
7699
CLERK_JWT_AUDIENCE: z.string(),
@@ -83,6 +106,7 @@ export function TransformStack({ stack }: StackContext) {
83106
authorizer: "JwtAuthorizer",
84107
function: {
85108
bind: [
109+
transformTestingQueue,
86110
TRANSFORM_DATABASE_URL,
87111
TRANSFORM_DATABASE_AUTH_TOKEN,
88112
EXTRACT_DATABASE_URL,

‎migrations/extract/0000_rare_wraith.sql‎ renamed to ‎migrations/extract/0000_furry_forgotten_one.sql‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ CREATE TABLE `git_identities` (
44
`email` text NOT NULL,
55
`name` text NOT NULL,
66
`__created_at` integer DEFAULT (strftime('%s', 'now')),
7-
`__updated_at` integer DEFAULT (strftime('%s', 'now'))
7+
`__updated_at` integer DEFAULT (strftime('%s', 'now')),
8+
FOREIGN KEY (`repository_id`) REFERENCES `repositories`(`id`) ON UPDATE no action ON DELETE no action
89
);
910
--> statement-breakpoint
1011
CREATE TABLE `members` (
@@ -33,7 +34,8 @@ CREATE TABLE `merge_request_commits` (
3334
`committer_name` text,
3435
`committer_email` text,
3536
`__created_at` integer DEFAULT (strftime('%s', 'now')),
36-
`__updated_at` integer DEFAULT (strftime('%s', 'now'))
37+
`__updated_at` integer DEFAULT (strftime('%s', 'now')),
38+
FOREIGN KEY (`merge_request_id`) REFERENCES `merge_requests`(`id`) ON UPDATE no action ON DELETE no action
3739
);
3840
--> statement-breakpoint
3941
CREATE TABLE `merge_request_diffs` (
@@ -48,7 +50,8 @@ CREATE TABLE `merge_request_diffs` (
4850
`deleted_file` integer NOT NULL,
4951
`diff` text NOT NULL,
5052
`__created_at` integer DEFAULT (strftime('%s', 'now')),
51-
`__updated_at` integer DEFAULT (strftime('%s', 'now'))
53+
`__updated_at` integer DEFAULT (strftime('%s', 'now')),
54+
FOREIGN KEY (`merge_request_id`) REFERENCES `merge_requests`(`id`) ON UPDATE no action ON DELETE no action
5255
);
5356
--> statement-breakpoint
5457
CREATE TABLE `merge_request_notes` (
@@ -62,7 +65,8 @@ CREATE TABLE `merge_request_notes` (
6265
`body` text NOT NULL,
6366
`system` integer NOT NULL,
6467
`__created_at` integer DEFAULT (strftime('%s', 'now')),
65-
`__updated_at` integer DEFAULT (strftime('%s', 'now'))
68+
`__updated_at` integer DEFAULT (strftime('%s', 'now')),
69+
FOREIGN KEY (`merge_request_id`) REFERENCES `merge_requests`(`id`) ON UPDATE no action ON DELETE no action
6670
);
6771
--> statement-breakpoint
6872
CREATE TABLE `merge_requests` (
@@ -83,7 +87,8 @@ CREATE TABLE `merge_requests` (
8387
`target_branch` text,
8488
`source_branch` text,
8589
`__created_at` integer DEFAULT (strftime('%s', 'now')),
86-
`__updated_at` integer DEFAULT (strftime('%s', 'now'))
90+
`__updated_at` integer DEFAULT (strftime('%s', 'now')),
91+
FOREIGN KEY (`repository_id`) REFERENCES `repositories`(`id`) ON UPDATE no action ON DELETE no action
8792
);
8893
--> statement-breakpoint
8994
CREATE TABLE `namespaces` (
@@ -111,7 +116,9 @@ CREATE TABLE `repositories_to_members` (
111116
`member_id` integer NOT NULL,
112117
`__created_at` integer DEFAULT (strftime('%s', 'now')),
113118
`__updated_at` integer DEFAULT (strftime('%s', 'now')),
114-
PRIMARY KEY(`member_id`, `repository_id`)
119+
PRIMARY KEY(`member_id`, `repository_id`),
120+
FOREIGN KEY (`repository_id`) REFERENCES `repositories`(`id`) ON UPDATE no action ON DELETE no action,
121+
FOREIGN KEY (`member_id`) REFERENCES `members`(`id`) ON UPDATE no action ON DELETE no action
115122
);
116123
--> statement-breakpoint
117124
CREATE TABLE `timeline_events` (
@@ -125,7 +132,8 @@ CREATE TABLE `timeline_events` (
125132
`actor_email` text,
126133
`data` text,
127134
`__created_at` integer DEFAULT (strftime('%s', 'now')),
128-
`__updated_at` integer DEFAULT (strftime('%s', 'now'))
135+
`__updated_at` integer DEFAULT (strftime('%s', 'now')),
136+
FOREIGN KEY (`merge_request_id`) REFERENCES `merge_requests`(`id`) ON UPDATE no action ON DELETE no action
129137
);
130138
--> statement-breakpoint
131139
CREATE UNIQUE INDEX `repository_id_email_name_idx` ON `git_identities` (`repository_id`,`email`,`name`);--> statement-breakpoint

‎migrations/extract/meta/0000_snapshot.json‎

Lines changed: 119 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"version": "5",
33
"dialect": "sqlite",
4-
"id": "a32acdeb-db5c-4dc5-a180-8ffc39137a9e",
4+
"id": "1389d00b-f2cd-45af-87ad-91c678a3f8b3",
55
"prevId": "00000000-0000-0000-0000-000000000000",
66
"tables": {
77
"git_identities": {
@@ -63,7 +63,21 @@
6363
"isUnique": true
6464
}
6565
},
66-
"foreignKeys": {},
66+
"foreignKeys": {
67+
"git_identities_repository_id_repositories_id_fk": {
68+
"name": "git_identities_repository_id_repositories_id_fk",
69+
"tableFrom": "git_identities",
70+
"tableTo": "repositories",
71+
"columnsFrom": [
72+
"repository_id"
73+
],
74+
"columnsTo": [
75+
"id"
76+
],
77+
"onDelete": "no action",
78+
"onUpdate": "no action"
79+
}
80+
},
6781
"compositePrimaryKeys": {},
6882
"uniqueConstraints": {}
6983
},
@@ -264,7 +278,21 @@
264278
"isUnique": true
265279
}
266280
},
267-
"foreignKeys": {},
281+
"foreignKeys": {
282+
"merge_request_commits_merge_request_id_merge_requests_id_fk": {
283+
"name": "merge_request_commits_merge_request_id_merge_requests_id_fk",
284+
"tableFrom": "merge_request_commits",
285+
"tableTo": "merge_requests",
286+
"columnsFrom": [
287+
"merge_request_id"
288+
],
289+
"columnsTo": [
290+
"id"
291+
],
292+
"onDelete": "no action",
293+
"onUpdate": "no action"
294+
}
295+
},
268296
"compositePrimaryKeys": {},
269297
"uniqueConstraints": {}
270298
},
@@ -368,7 +396,21 @@
368396
"isUnique": true
369397
}
370398
},
371-
"foreignKeys": {},
399+
"foreignKeys": {
400+
"merge_request_diffs_merge_request_id_merge_requests_id_fk": {
401+
"name": "merge_request_diffs_merge_request_id_merge_requests_id_fk",
402+
"tableFrom": "merge_request_diffs",
403+
"tableTo": "merge_requests",
404+
"columnsFrom": [
405+
"merge_request_id"
406+
],
407+
"columnsTo": [
408+
"id"
409+
],
410+
"onDelete": "no action",
411+
"onUpdate": "no action"
412+
}
413+
},
372414
"compositePrimaryKeys": {},
373415
"uniqueConstraints": {}
374416
},
@@ -465,7 +507,21 @@
465507
"isUnique": true
466508
}
467509
},
468-
"foreignKeys": {},
510+
"foreignKeys": {
511+
"merge_request_notes_merge_request_id_merge_requests_id_fk": {
512+
"name": "merge_request_notes_merge_request_id_merge_requests_id_fk",
513+
"tableFrom": "merge_request_notes",
514+
"tableTo": "merge_requests",
515+
"columnsFrom": [
516+
"merge_request_id"
517+
],
518+
"columnsTo": [
519+
"id"
520+
],
521+
"onDelete": "no action",
522+
"onUpdate": "no action"
523+
}
524+
},
469525
"compositePrimaryKeys": {},
470526
"uniqueConstraints": {}
471527
},
@@ -611,7 +667,21 @@
611667
"isUnique": true
612668
}
613669
},
614-
"foreignKeys": {},
670+
"foreignKeys": {
671+
"merge_requests_repository_id_repositories_id_fk": {
672+
"name": "merge_requests_repository_id_repositories_id_fk",
673+
"tableFrom": "merge_requests",
674+
"tableTo": "repositories",
675+
"columnsFrom": [
676+
"repository_id"
677+
],
678+
"columnsTo": [
679+
"id"
680+
],
681+
"onDelete": "no action",
682+
"onUpdate": "no action"
683+
}
684+
},
615685
"compositePrimaryKeys": {},
616686
"uniqueConstraints": {}
617687
},
@@ -795,7 +865,34 @@
795865
}
796866
},
797867
"indexes": {},
798-
"foreignKeys": {},
868+
"foreignKeys": {
869+
"repositories_to_members_repository_id_repositories_id_fk": {
870+
"name": "repositories_to_members_repository_id_repositories_id_fk",
871+
"tableFrom": "repositories_to_members",
872+
"tableTo": "repositories",
873+
"columnsFrom": [
874+
"repository_id"
875+
],
876+
"columnsTo": [
877+
"id"
878+
],
879+
"onDelete": "no action",
880+
"onUpdate": "no action"
881+
},
882+
"repositories_to_members_member_id_members_id_fk": {
883+
"name": "repositories_to_members_member_id_members_id_fk",
884+
"tableFrom": "repositories_to_members",
885+
"tableTo": "members",
886+
"columnsFrom": [
887+
"member_id"
888+
],
889+
"columnsTo": [
890+
"id"
891+
],
892+
"onDelete": "no action",
893+
"onUpdate": "no action"
894+
}
895+
},
799896
"compositePrimaryKeys": {
800897
"repositories_to_members_repository_id_member_id_pk": {
801898
"columns": [
@@ -900,7 +997,21 @@
900997
"isUnique": true
901998
}
902999
},
903-
"foreignKeys": {},
1000+
"foreignKeys": {
1001+
"timeline_events_merge_request_id_merge_requests_id_fk": {
1002+
"name": "timeline_events_merge_request_id_merge_requests_id_fk",
1003+
"tableFrom": "timeline_events",
1004+
"tableTo": "merge_requests",
1005+
"columnsFrom": [
1006+
"merge_request_id"
1007+
],
1008+
"columnsTo": [
1009+
"id"
1010+
],
1011+
"onDelete": "no action",
1012+
"onUpdate": "no action"
1013+
}
1014+
},
9041015
"compositePrimaryKeys": {},
9051016
"uniqueConstraints": {}
9061017
}

‎migrations/extract/meta/_journal.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
{
66
"idx": 0,
77
"version": "5",
8-
"when": 1700489210258,
9-
"tag": "0000_rare_wraith",
8+
"when": 1700578843881,
9+
"tag": "0000_furry_forgotten_one",
1010
"breakpoints": true
1111
}
1212
]

‎migrations/transform/0001_aromatic_sister_grimm.sql‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
CREATE TABLE `null_rows` (
2+
`id` integer PRIMARY KEY NOT NULL,
3+
`dates_id` integer NOT NULL,
4+
`users_id` integer NOT NULL,
5+
`merge_requests_id` integer NOT NULL,
6+
`repository_id` integer NOT NULL,
7+
`__created_at` integer DEFAULT (strftime('%s', 'now')),
8+
`__updated_at` integer DEFAULT (strftime('%s', 'now'))
9+
);

0 commit comments

Comments
 (0)