From 2735914e08418be9907d295d4df2c2f03d137fa8 Mon Sep 17 00:00:00 2001 From: 0xTitan Date: Sun, 2 Jul 2023 22:47:16 +0200 Subject: [PATCH 1/7] feat: auto-increment for entity id --- package.json | 2 +- src/checkpoint.ts | 20 +++--- src/graphql/controller.ts | 69 +++++++++++++++---- src/utils/graphql.ts | 9 +++ .../__snapshots__/controller.test.ts.snap | 9 +++ test/unit/graphql/controller.test.ts | 43 ++++++++++-- 6 files changed, 124 insertions(+), 28 deletions(-) diff --git a/package.json b/package.json index 5c4d636..ca84740 100644 --- a/package.json +++ b/package.json @@ -61,4 +61,4 @@ "dist/**/*", "src/**/*" ] -} +} \ No newline at end of file diff --git a/src/checkpoint.ts b/src/checkpoint.ts index 20e17bd..9ec06e3 100644 --- a/src/checkpoint.ts +++ b/src/checkpoint.ts @@ -9,7 +9,7 @@ import { CheckpointRecord, CheckpointsStore, MetadataId } from './stores/checkpo import { BaseProvider, StarknetProvider, BlockNotFoundError } from './providers'; import { createLogger, Logger, LogLevel } from './utils/logger'; import { getConfigChecksum, getContractsFromConfig } from './utils/checkpoint'; -import { extendSchema } from './utils/graphql'; +import { ensureGqlCompatibilty, extendSchema } from './utils/graphql'; import { createKnex } from './knex'; import { AsyncMySqlPool, createMySqlPool } from './mysql'; import { createPgPool } from './pg'; @@ -29,7 +29,8 @@ export default class Checkpoint { public config: CheckpointConfig; public writer: CheckpointWriters; public opts?: CheckpointOptions; - public schema: string; + public schemaCustom: string; + public schemaGql: string; private readonly entityController: GqlEntityController; private readonly log: Logger; @@ -47,7 +48,7 @@ export default class Checkpoint { constructor( config: CheckpointConfig, writer: CheckpointWriters, - schema: string, + schemaCustom: string, opts?: CheckpointOptions ) { const validationResult = checkpointConfigSchema.safeParse(config); @@ -58,11 +59,12 @@ export default class Checkpoint { this.config = config; this.writer = writer; this.opts = opts; - this.schema = extendSchema(schema); + this.schemaCustom = extendSchema(schemaCustom); + this.schemaGql = ensureGqlCompatibilty(schemaCustom); this.validateConfig(); - this.entityController = new GqlEntityController(this.schema, config); + this.entityController = new GqlEntityController(this.schemaGql, config); this.sourceContracts = getContractsFromConfig(config); this.cpBlocksCache = []; @@ -72,10 +74,10 @@ export default class Checkpoint { level: opts?.logLevel || LogLevel.Error, ...(opts?.prettifyLogs ? { - transport: { - target: 'pino-pretty' - } + transport: { + target: 'pino-pretty' } + } : {}) }); @@ -191,7 +193,7 @@ export default class Checkpoint { await this.store.createStore(); await this.store.setMetadata(MetadataId.LastIndexedBlock, 0); - await this.entityController.createEntityStores(this.knex); + await this.entityController.createEntityStores(this.knex, this.schemaCustom); } /** diff --git a/src/graphql/controller.ts b/src/graphql/controller.ts index 8a77e6b..f394e27 100644 --- a/src/graphql/controller.ts +++ b/src/graphql/controller.ts @@ -32,6 +32,7 @@ import { } from '../utils/graphql'; import { CheckpointConfig } from '../types'; import { querySingle, queryMulti, ResolverContext, getNestedResolver } from './resolvers'; +import { boolean } from 'yargs'; /** * Type for single and multiple query resolvers @@ -193,10 +194,21 @@ export class GqlEntityController { * ); * ``` * + will execute the following SQL when declaring id as autoincrement: + * ```sql + * DROP TABLE IF EXISTS votes; + * CREATE TABLE votes ( + * id integer not null primary key autoincrement, + * name VARCHAR(128), + * ); + * ``` + * */ - public async createEntityStores(knex: Knex): Promise<{ builder: Knex.SchemaBuilder }> { + public async createEntityStores(knex: Knex, schema: string): Promise<{ builder: Knex.SchemaBuilder }> { let builder = knex.schema; + let autoIncrementFieldsMap = this.extractAutoIncrementFields(schema); + if (this.schemaObjects.length === 0) { return { builder }; } @@ -205,34 +217,65 @@ export class GqlEntityController { const tableName = pluralize(type.name.toLowerCase()); builder = builder.dropTableIfExists(tableName).createTable(tableName, t => { - t.primary(['id']); + if (autoIncrementFieldsMap.get(tableName)?.length === 0) { + t.primary(['id']); + } this.getTypeFields(type).forEach(field => { const fieldType = field.type instanceof GraphQLNonNull ? field.type.ofType : field.type; if (isListType(fieldType) && fieldType.ofType instanceof GraphQLObjectType) return; - const sqlType = this.getSqlType(field.type); + //Check if field is declared as autoincrement + if (autoIncrementFieldsMap.get(tableName)?.includes(field.name)) { + t.increments(field.name, { primaryKey: true }); + } else { + const sqlType = this.getSqlType(field.type); + + let column = + 'options' in sqlType + ? t[sqlType.name](field.name, ...sqlType.options) + : t[sqlType.name](field.name); - let column = - 'options' in sqlType - ? t[sqlType.name](field.name, ...sqlType.options) - : t[sqlType.name](field.name); - if (field.type instanceof GraphQLNonNull) { - column = column.notNullable(); - } - if (!['text', 'json'].includes(sqlType.name)) { - column.index(); + if (field.type instanceof GraphQLNonNull) { + column = column.notNullable(); + } + + if (!['text', 'json'].includes(sqlType.name)) { + column.index(); + } + } + }); }); }); await builder; - return { builder }; } + private extractAutoIncrementFields(schema: string) { + let autoIncrementFieldsMap = new Map(); + const regexTableName = /type\s+(\w+)\s+{/; + const regexAutoIncrementField = /(\w+):\s+(Big)?Int!?\s+@autoIncrement/; + //const regexAutoIncrementField = /(\w+):\s+(Big)?Int!?\s+@autoIncrement/g; + const matchTable = schema.match(regexTableName); + if (matchTable && matchTable[1]) { + const tableName = pluralize.plural(matchTable[1].toLocaleLowerCase()); + const listFields = schema.split('\n').map(line => { + const matchAutoIncrement = line.match(regexAutoIncrementField); + if (matchAutoIncrement && matchAutoIncrement[1]) { + return matchAutoIncrement[1]; + } + }).filter(line => line !== undefined) as string[]; + if (listFields) { + autoIncrementFieldsMap.set(tableName, listFields); + } + } + return autoIncrementFieldsMap; + } + /** * Generates a query based on the first entity discovered * in a schema. If no entities are found in the schema diff --git a/src/utils/graphql.ts b/src/utils/graphql.ts index fb44872..8a6749d 100644 --- a/src/utils/graphql.ts +++ b/src/utils/graphql.ts @@ -79,3 +79,12 @@ export const getNonNullType = (type: GraphQLOutputType): GraphQLOutputType => { return type; }; + +/** + * Gql schema do not manage autoincrement fields. We have to remove the attribute to make schema for gql valid + * @param schema + * @returns schema without autoincrement + */ +export const ensureGqlCompatibilty = (schema: string): string => { + return schema.replace(/@autoIncrement/g, ''); +}; diff --git a/test/unit/graphql/__snapshots__/controller.test.ts.snap b/test/unit/graphql/__snapshots__/controller.test.ts.snap index b168c4d..7232f58 100644 --- a/test/unit/graphql/__snapshots__/controller.test.ts.snap +++ b/test/unit/graphql/__snapshots__/controller.test.ts.snap @@ -16,6 +16,15 @@ create index \`votes_decimal_index\` on \`votes\` (\`decimal\`); create index \`votes_big_decimal_index\` on \`votes\` (\`big_decimal\`)" `; +exports[`GqlEntityController createEntityStores should work with autoincrement 1`] = ` +"drop table if exists \`votes\`; +create table \`votes\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(128), \`authenticators\` json, \`big_number\` bigint, \`decimal\` float, \`big_decimal\` float); +create index \`votes_name_index\` on \`votes\` (\`name\`); +create index \`votes_big_number_index\` on \`votes\` (\`big_number\`); +create index \`votes_decimal_index\` on \`votes\` (\`decimal\`); +create index \`votes_big_decimal_index\` on \`votes\` (\`big_decimal\`)" +`; + exports[`GqlEntityController generateQueryFields should work 1`] = ` "type Query { vote(id: Int!): Vote diff --git a/test/unit/graphql/controller.test.ts b/test/unit/graphql/controller.test.ts index ac91b10..eac70af 100644 --- a/test/unit/graphql/controller.test.ts +++ b/test/unit/graphql/controller.test.ts @@ -1,6 +1,10 @@ import { GraphQLObjectType, GraphQLSchema, printSchema } from 'graphql'; import knex from 'knex'; import { GqlEntityController } from '../../../src/graphql/controller'; +import pluralize, { plural } from 'pluralize'; +import { table } from 'console'; +import ret from 'bluebird/js/release/util'; + describe('GqlEntityController', () => { describe('generateQueryFields', () => { @@ -17,7 +21,6 @@ type Vote { name: 'Query', fields: queryFields }); - const schema = printSchema(new GraphQLSchema({ query: querySchema })); expect(schema).toMatchSnapshot(); }); @@ -56,7 +59,8 @@ type Vote { }); it('should work', async () => { - const controller = new GqlEntityController(` + let autoIncrementFieldsMap = new Map(); + const schema = ` scalar BigInt scalar Decimal scalar BigDecimal @@ -69,10 +73,39 @@ type Vote { decimal: Decimal big_decimal: BigDecimal } - `); - const { builder } = await controller.createEntityStores(mockKnex); + `; + + const controller = new GqlEntityController(schema); + const { builder } = await controller.createEntityStores(mockKnex, schema); + + const createQuery = builder.toString(); + console.log("createQuery :" + createQuery); + expect(createQuery).toMatchSnapshot(); + }); + + + it('should work with autoincrement', async () => { + const schema = ` +scalar BigInt +scalar Decimal +scalar BigDecimal + +type Vote { + id: Int! @autoIncrement + name: String + authenticators: [String] + big_number: BigInt + decimal: Decimal + big_decimal: BigDecimal +} + `; + const schemaGql = schema.replace(/@autoIncrement/g, ''); + const controller = new GqlEntityController(schemaGql); + const { builder } = await controller.createEntityStores(mockKnex, schema); - expect(builder.toString()).toMatchSnapshot(); + const createQuery = builder.toString(); + console.log("createQuery :" + createQuery); + expect(createQuery).toMatchSnapshot(); }); }); From 99b73f179385b9c5e96505ebfa4c8c02b692209f Mon Sep 17 00:00:00 2001 From: 0xTitan Date: Thu, 6 Jul 2023 22:58:14 +0200 Subject: [PATCH 2/7] feat: auto-increment for entity id --- src/graphql/controller.ts | 64 ++++++++++++------- src/types.ts | 20 ++++++ src/utils/graphql.ts | 7 +- .../__snapshots__/controller.test.ts.snap | 13 ++++ test/unit/graphql/controller.test.ts | 41 ++++++++++-- 5 files changed, 111 insertions(+), 34 deletions(-) diff --git a/src/graphql/controller.ts b/src/graphql/controller.ts index f394e27..0b40d0f 100644 --- a/src/graphql/controller.ts +++ b/src/graphql/controller.ts @@ -30,9 +30,8 @@ import { singleEntityQueryName, getNonNullType } from '../utils/graphql'; -import { CheckpointConfig } from '../types'; +import { autoIncrementFieldPattern, CheckpointConfig, tableNamePattern } from '../types'; import { querySingle, queryMulti, ResolverContext, getNestedResolver } from './resolvers'; -import { boolean } from 'yargs'; /** * Type for single and multiple query resolvers @@ -204,10 +203,13 @@ export class GqlEntityController { * ``` * */ - public async createEntityStores(knex: Knex, schema: string): Promise<{ builder: Knex.SchemaBuilder }> { + public async createEntityStores( + knex: Knex, + schema: string + ): Promise<{ builder: Knex.SchemaBuilder }> { let builder = knex.schema; - let autoIncrementFieldsMap = this.extractAutoIncrementFields(schema); + const autoIncrementFieldsMap = this.extractAutoIncrementFields(schema); if (this.schemaObjects.length === 0) { return { builder }; @@ -217,7 +219,11 @@ export class GqlEntityController { const tableName = pluralize(type.name.toLowerCase()); builder = builder.dropTableIfExists(tableName).createTable(tableName, t => { - if (autoIncrementFieldsMap.get(tableName)?.length === 0) { + //if there is no autoIncrement fields on current table, mark the id as primary + if ( + autoIncrementFieldsMap.size == 0 || + autoIncrementFieldsMap.get(tableName)?.length === 0 + ) { t.primary(['id']); } @@ -229,14 +235,11 @@ export class GqlEntityController { t.increments(field.name, { primaryKey: true }); } else { const sqlType = this.getSqlType(field.type); - let column = 'options' in sqlType ? t[sqlType.name](field.name, ...sqlType.options) : t[sqlType.name](field.name); - - if (field.type instanceof GraphQLNonNull) { column = column.notNullable(); } @@ -244,9 +247,7 @@ export class GqlEntityController { if (!['text', 'json'].includes(sqlType.name)) { column.index(); } - } - }); }); }); @@ -255,24 +256,39 @@ export class GqlEntityController { return { builder }; } + /** + * Parse schema to extract table and fields witrh annotation autoIncrement + * @param schema + * @returns A map where key is table name and values a list of fields with annotation autoIncrement + */ private extractAutoIncrementFields(schema: string) { - let autoIncrementFieldsMap = new Map(); - const regexTableName = /type\s+(\w+)\s+{/; - const regexAutoIncrementField = /(\w+):\s+(Big)?Int!?\s+@autoIncrement/; - //const regexAutoIncrementField = /(\w+):\s+(Big)?Int!?\s+@autoIncrement/g; - const matchTable = schema.match(regexTableName); - if (matchTable && matchTable[1]) { - const tableName = pluralize.plural(matchTable[1].toLocaleLowerCase()); - const listFields = schema.split('\n').map(line => { - const matchAutoIncrement = line.match(regexAutoIncrementField); + const autoIncrementFieldsMap = new Map(); + let currentTable = ''; + + schema.split('\n').forEach(line => { + const matchTable = line.match(tableNamePattern); + //check for current table name + if (matchTable && matchTable[1]) { + const tableName = pluralize.plural(matchTable[1].toLocaleLowerCase()); + if (tableName !== currentTable) { + currentTable = tableName; + } + } else { + //check for fields + const matchAutoIncrement = line.match(autoIncrementFieldPattern()); if (matchAutoIncrement && matchAutoIncrement[1]) { - return matchAutoIncrement[1]; + const field = matchAutoIncrement[1]; + // Check if the key already exists in the map + if (autoIncrementFieldsMap.has(currentTable)) { + const existingFields = autoIncrementFieldsMap.get(currentTable); + existingFields?.push(field); + } else { + autoIncrementFieldsMap.set(currentTable, [field]); + } } - }).filter(line => line !== undefined) as string[]; - if (listFields) { - autoIncrementFieldsMap.set(tableName, listFields); } - } + }); + return autoIncrementFieldsMap; } diff --git a/src/types.ts b/src/types.ts index c5e5384..f102494 100644 --- a/src/types.ts +++ b/src/types.ts @@ -112,3 +112,23 @@ export function isFullBlock(block: Block): block is FullBlock { export function isDeployTransaction(tx: Transaction | PendingTransaction): tx is DeployTransaction { return tx.type === 'DEPLOY'; } + +/** + * AutoIncrement annotation in schema.gql + */ +export const autoIncrementTag = /@autoIncrement/; + +/** + * Regex pattern to find table name in schema.gql + */ +export const tableNamePattern = /type\s+(\w+)\s+{/; + +/** + * Regex pattern to find field name having annotion "@autoIncrement" in schema.gql + * Consider only field type : Int!,ID!, BigInt, + * @returns regex expression : /(\w+):\s(Int|ID|BigInt)!?\s+@autoIncrement/ + */ +export function autoIncrementFieldPattern(): RegExp { + const fieldPattern = /(\w+):\s(Int|ID|BigInt)!?\s+/; + return new RegExp(fieldPattern.source + autoIncrementTag.source); +} diff --git a/src/utils/graphql.ts b/src/utils/graphql.ts index 8a6749d..2c28cd8 100644 --- a/src/utils/graphql.ts +++ b/src/utils/graphql.ts @@ -8,6 +8,7 @@ import { } from 'graphql'; import { jsonToGraphQLQuery } from 'json-to-graphql-query'; import pluralize from 'pluralize'; +import { autoIncrementTag } from '../types'; export const extendSchema = (schema: string): string => { return `directive @derivedFrom(field: String!) on FIELD_DEFINITION @@ -82,9 +83,9 @@ export const getNonNullType = (type: GraphQLOutputType): GraphQLOutputType => { /** * Gql schema do not manage autoincrement fields. We have to remove the attribute to make schema for gql valid - * @param schema - * @returns schema without autoincrement + * @param schema + * @returns schema without autoincrement annotation */ export const ensureGqlCompatibilty = (schema: string): string => { - return schema.replace(/@autoIncrement/g, ''); + return schema.replace(new RegExp(autoIncrementTag.source, 'g'), ''); }; diff --git a/test/unit/graphql/__snapshots__/controller.test.ts.snap b/test/unit/graphql/__snapshots__/controller.test.ts.snap index 7232f58..087a410 100644 --- a/test/unit/graphql/__snapshots__/controller.test.ts.snap +++ b/test/unit/graphql/__snapshots__/controller.test.ts.snap @@ -25,6 +25,19 @@ create index \`votes_decimal_index\` on \`votes\` (\`decimal\`); create index \`votes_big_decimal_index\` on \`votes\` (\`big_decimal\`)" `; +exports[`GqlEntityController createEntityStores should work with autoincrement with nested objects 1`] = ` +"drop table if exists \`votes\`; +create table \`votes\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(128), \`authenticators\` json, \`big_number\` bigint, \`decimal\` float, \`big_decimal\` float, \`poster\` varchar(128)); +create index \`votes_name_index\` on \`votes\` (\`name\`); +create index \`votes_big_number_index\` on \`votes\` (\`big_number\`); +create index \`votes_decimal_index\` on \`votes\` (\`decimal\`); +create index \`votes_big_decimal_index\` on \`votes\` (\`big_decimal\`); +create index \`votes_poster_index\` on \`votes\` (\`poster\`); +drop table if exists \`posters\`; +create table \`posters\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(128) not null); +create index \`posters_name_index\` on \`posters\` (\`name\`)" +`; + exports[`GqlEntityController generateQueryFields should work 1`] = ` "type Query { vote(id: Int!): Vote diff --git a/test/unit/graphql/controller.test.ts b/test/unit/graphql/controller.test.ts index eac70af..237aa86 100644 --- a/test/unit/graphql/controller.test.ts +++ b/test/unit/graphql/controller.test.ts @@ -1,10 +1,10 @@ import { GraphQLObjectType, GraphQLSchema, printSchema } from 'graphql'; import knex from 'knex'; import { GqlEntityController } from '../../../src/graphql/controller'; -import pluralize, { plural } from 'pluralize'; -import { table } from 'console'; import ret from 'bluebird/js/release/util'; +import { autoIncrementTag } from '../../../src/types'; +const regex = new RegExp(autoIncrementTag.source, 'g'); describe('GqlEntityController', () => { describe('generateQueryFields', () => { @@ -59,7 +59,6 @@ type Vote { }); it('should work', async () => { - let autoIncrementFieldsMap = new Map(); const schema = ` scalar BigInt scalar Decimal @@ -79,11 +78,9 @@ type Vote { const { builder } = await controller.createEntityStores(mockKnex, schema); const createQuery = builder.toString(); - console.log("createQuery :" + createQuery); expect(createQuery).toMatchSnapshot(); }); - it('should work with autoincrement', async () => { const schema = ` scalar BigInt @@ -98,13 +95,43 @@ type Vote { decimal: Decimal big_decimal: BigDecimal } + `; - const schemaGql = schema.replace(/@autoIncrement/g, ''); + + const schemaGql = schema.replace(regex, ''); const controller = new GqlEntityController(schemaGql); const { builder } = await controller.createEntityStores(mockKnex, schema); + const createQuery = builder.toString(); + expect(createQuery).toMatchSnapshot(); + }); + + it('should work with autoincrement with nested objects', async () => { + const schema = ` +scalar BigInt +scalar Decimal +scalar BigDecimal + +type Vote { + id: Int! @autoIncrement + name: String + authenticators: [String] + big_number: BigInt + decimal: Decimal + big_decimal: BigDecimal + poster : Poster +} + +type Poster { + id: ID! @autoIncrement + name: String! +} + `; + + const schemaGql = schema.replace(new RegExp(autoIncrementTag.source, 'g'), ''); + const controller = new GqlEntityController(schemaGql); + const { builder } = await controller.createEntityStores(mockKnex, schema); const createQuery = builder.toString(); - console.log("createQuery :" + createQuery); expect(createQuery).toMatchSnapshot(); }); }); From 05305c217ec8fe9d2da8b485daa96072920af889 Mon Sep 17 00:00:00 2001 From: 0xTitan Date: Thu, 6 Jul 2023 23:09:48 +0200 Subject: [PATCH 3/7] feat: auto-increment for entity id --- test/unit/graphql/__snapshots__/controller.test.ts.snap | 2 +- test/unit/graphql/controller.test.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/unit/graphql/__snapshots__/controller.test.ts.snap b/test/unit/graphql/__snapshots__/controller.test.ts.snap index 087a410..d424d34 100644 --- a/test/unit/graphql/__snapshots__/controller.test.ts.snap +++ b/test/unit/graphql/__snapshots__/controller.test.ts.snap @@ -25,7 +25,7 @@ create index \`votes_decimal_index\` on \`votes\` (\`decimal\`); create index \`votes_big_decimal_index\` on \`votes\` (\`big_decimal\`)" `; -exports[`GqlEntityController createEntityStores should work with autoincrement with nested objects 1`] = ` +exports[`GqlEntityController createEntityStores should work with autoincrement and nested objects 1`] = ` "drop table if exists \`votes\`; create table \`votes\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(128), \`authenticators\` json, \`big_number\` bigint, \`decimal\` float, \`big_decimal\` float, \`poster\` varchar(128)); create index \`votes_name_index\` on \`votes\` (\`name\`); diff --git a/test/unit/graphql/controller.test.ts b/test/unit/graphql/controller.test.ts index 237aa86..1e67720 100644 --- a/test/unit/graphql/controller.test.ts +++ b/test/unit/graphql/controller.test.ts @@ -1,7 +1,6 @@ import { GraphQLObjectType, GraphQLSchema, printSchema } from 'graphql'; import knex from 'knex'; import { GqlEntityController } from '../../../src/graphql/controller'; -import ret from 'bluebird/js/release/util'; import { autoIncrementTag } from '../../../src/types'; const regex = new RegExp(autoIncrementTag.source, 'g'); @@ -105,7 +104,7 @@ type Vote { expect(createQuery).toMatchSnapshot(); }); - it('should work with autoincrement with nested objects', async () => { + it('should work with autoincrement and nested objects', async () => { const schema = ` scalar BigInt scalar Decimal From 23698a4587b4fe2b1160fe0224401e0d5d1bb90f Mon Sep 17 00:00:00 2001 From: 0xTitan Date: Wed, 26 Jul 2023 23:03:29 +0200 Subject: [PATCH 4/7] feat: auto-increment for entity id --- src/checkpoint.ts | 14 ++++++-------- src/utils/graphql.ts | 15 +++------------ test/unit/graphql/controller.test.ts | 16 +++++++--------- 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/src/checkpoint.ts b/src/checkpoint.ts index 9ec06e3..7b33ba5 100644 --- a/src/checkpoint.ts +++ b/src/checkpoint.ts @@ -9,7 +9,7 @@ import { CheckpointRecord, CheckpointsStore, MetadataId } from './stores/checkpo import { BaseProvider, StarknetProvider, BlockNotFoundError } from './providers'; import { createLogger, Logger, LogLevel } from './utils/logger'; import { getConfigChecksum, getContractsFromConfig } from './utils/checkpoint'; -import { ensureGqlCompatibilty, extendSchema } from './utils/graphql'; +import { extendSchema } from './utils/graphql'; import { createKnex } from './knex'; import { AsyncMySqlPool, createMySqlPool } from './mysql'; import { createPgPool } from './pg'; @@ -29,8 +29,7 @@ export default class Checkpoint { public config: CheckpointConfig; public writer: CheckpointWriters; public opts?: CheckpointOptions; - public schemaCustom: string; - public schemaGql: string; + public schema: string; private readonly entityController: GqlEntityController; private readonly log: Logger; @@ -48,7 +47,7 @@ export default class Checkpoint { constructor( config: CheckpointConfig, writer: CheckpointWriters, - schemaCustom: string, + schema: string, opts?: CheckpointOptions ) { const validationResult = checkpointConfigSchema.safeParse(config); @@ -59,12 +58,11 @@ export default class Checkpoint { this.config = config; this.writer = writer; this.opts = opts; - this.schemaCustom = extendSchema(schemaCustom); - this.schemaGql = ensureGqlCompatibilty(schemaCustom); + this.schema = extendSchema(schema); this.validateConfig(); - this.entityController = new GqlEntityController(this.schemaGql, config); + this.entityController = new GqlEntityController(this.schema, config); this.sourceContracts = getContractsFromConfig(config); this.cpBlocksCache = []; @@ -193,7 +191,7 @@ export default class Checkpoint { await this.store.createStore(); await this.store.setMetadata(MetadataId.LastIndexedBlock, 0); - await this.entityController.createEntityStores(this.knex, this.schemaCustom); + await this.entityController.createEntityStores(this.knex, this.schema); } /** diff --git a/src/utils/graphql.ts b/src/utils/graphql.ts index 2c28cd8..fe2bc23 100644 --- a/src/utils/graphql.ts +++ b/src/utils/graphql.ts @@ -8,11 +8,11 @@ import { } from 'graphql'; import { jsonToGraphQLQuery } from 'json-to-graphql-query'; import pluralize from 'pluralize'; -import { autoIncrementTag } from '../types'; export const extendSchema = (schema: string): string => { - return `directive @derivedFrom(field: String!) on FIELD_DEFINITION -${schema}`; + return `directive @derivedFrom(field: String!) on FIELD_DEFINITION + directive @autoIncrement on FIELD_DEFINITION + ${schema}`; }; /** @@ -80,12 +80,3 @@ export const getNonNullType = (type: GraphQLOutputType): GraphQLOutputType => { return type; }; - -/** - * Gql schema do not manage autoincrement fields. We have to remove the attribute to make schema for gql valid - * @param schema - * @returns schema without autoincrement annotation - */ -export const ensureGqlCompatibilty = (schema: string): string => { - return schema.replace(new RegExp(autoIncrementTag.source, 'g'), ''); -}; diff --git a/test/unit/graphql/controller.test.ts b/test/unit/graphql/controller.test.ts index 1e67720..28100b8 100644 --- a/test/unit/graphql/controller.test.ts +++ b/test/unit/graphql/controller.test.ts @@ -1,9 +1,7 @@ import { GraphQLObjectType, GraphQLSchema, printSchema } from 'graphql'; import knex from 'knex'; import { GqlEntityController } from '../../../src/graphql/controller'; -import { autoIncrementTag } from '../../../src/types'; - -const regex = new RegExp(autoIncrementTag.source, 'g'); +import { extendSchema } from '../../../src/utils/graphql'; describe('GqlEntityController', () => { describe('generateQueryFields', () => { @@ -81,7 +79,7 @@ type Vote { }); it('should work with autoincrement', async () => { - const schema = ` + let schema = ` scalar BigInt scalar Decimal scalar BigDecimal @@ -97,15 +95,15 @@ type Vote { `; - const schemaGql = schema.replace(regex, ''); - const controller = new GqlEntityController(schemaGql); + schema = extendSchema(schema); + const controller = new GqlEntityController(schema); const { builder } = await controller.createEntityStores(mockKnex, schema); const createQuery = builder.toString(); expect(createQuery).toMatchSnapshot(); }); it('should work with autoincrement and nested objects', async () => { - const schema = ` + let schema = ` scalar BigInt scalar Decimal scalar BigDecimal @@ -127,8 +125,8 @@ type Poster { `; - const schemaGql = schema.replace(new RegExp(autoIncrementTag.source, 'g'), ''); - const controller = new GqlEntityController(schemaGql); + schema = extendSchema(schema); + const controller = new GqlEntityController(schema); const { builder } = await controller.createEntityStores(mockKnex, schema); const createQuery = builder.toString(); expect(createQuery).toMatchSnapshot(); From 359e10a44bd1b715acc2a8628e9c27c7195752a3 Mon Sep 17 00:00:00 2001 From: 0xTitan Date: Sat, 4 Nov 2023 08:54:09 +0100 Subject: [PATCH 5/7] use directive to check autoincrement field --- src/graphql/controller.ts | 58 +++++-------------- src/types.ts | 22 +------ .../__snapshots__/controller.test.ts.snap | 6 +- 3 files changed, 17 insertions(+), 69 deletions(-) diff --git a/src/graphql/controller.ts b/src/graphql/controller.ts index c0f3bf4..f5832a4 100644 --- a/src/graphql/controller.ts +++ b/src/graphql/controller.ts @@ -30,7 +30,7 @@ import { singleEntityQueryName, getNonNullType } from '../utils/graphql'; -import { autoIncrementFieldPattern, CheckpointConfig, tableNamePattern } from '../types'; +import { CheckpointConfig } from '../types'; import { querySingle, queryMulti, ResolverContext, getNestedResolver } from './resolvers'; /** @@ -216,8 +216,6 @@ export class GqlEntityController { ): Promise<{ builder: Knex.SchemaBuilder }> { let builder = knex.schema; - const autoIncrementFieldsMap = this.extractAutoIncrementFields(schema); - if (this.schemaObjects.length === 0) { return { builder }; } @@ -227,18 +225,24 @@ export class GqlEntityController { builder = builder.dropTableIfExists(tableName).createTable(tableName, t => { //if there is no autoIncrement fields on current table, mark the id as primary - if ( - autoIncrementFieldsMap.size == 0 || - autoIncrementFieldsMap.get(tableName)?.length === 0 - ) { + const tableHasAutoIncrement = this.getTypeFields(type).some(field => { + const directives = field.astNode?.directives ?? []; + const autoIncrementDirective = directives.find(dir => dir.name.value === 'autoIncrement'); + return autoIncrementDirective; + }) + + if (!tableHasAutoIncrement) t.primary(['id']); - } this.getTypeFields(type).forEach(field => { const fieldType = field.type instanceof GraphQLNonNull ? field.type.ofType : field.type; if (isListType(fieldType) && fieldType.ofType instanceof GraphQLObjectType) return; + + const directives = field.astNode?.directives ?? []; + const autoIncrementDirective = directives.find(dir => dir.name.value === 'autoIncrement'); + //Check if field is declared as autoincrement - if (autoIncrementFieldsMap.get(tableName)?.includes(field.name)) { + if (autoIncrementDirective) { t.increments(field.name, { primaryKey: true }); } else { const sqlType = this.getSqlType(field.type); @@ -263,42 +267,6 @@ export class GqlEntityController { return { builder }; } - /** - * Parse schema to extract table and fields witrh annotation autoIncrement - * @param schema - * @returns A map where key is table name and values a list of fields with annotation autoIncrement - */ - private extractAutoIncrementFields(schema: string) { - const autoIncrementFieldsMap = new Map(); - let currentTable = ''; - - schema.split('\n').forEach(line => { - const matchTable = line.match(tableNamePattern); - //check for current table name - if (matchTable && matchTable[1]) { - const tableName = pluralize.plural(matchTable[1].toLocaleLowerCase()); - if (tableName !== currentTable) { - currentTable = tableName; - } - } else { - //check for fields - const matchAutoIncrement = line.match(autoIncrementFieldPattern()); - if (matchAutoIncrement && matchAutoIncrement[1]) { - const field = matchAutoIncrement[1]; - // Check if the key already exists in the map - if (autoIncrementFieldsMap.has(currentTable)) { - const existingFields = autoIncrementFieldsMap.get(currentTable); - existingFields?.push(field); - } else { - autoIncrementFieldsMap.set(currentTable, [field]); - } - } - } - }); - - return autoIncrementFieldsMap; - } - /** * Generates a query based on the first entity discovered * in a schema. If no entities are found in the schema diff --git a/src/types.ts b/src/types.ts index 901dcb7..a1d845a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -111,24 +111,4 @@ export function isFullBlock(block: Block): block is FullBlock { export function isDeployTransaction(tx: Transaction | PendingTransaction): tx is DeployTransaction { return tx.type === 'DEPLOY'; -} - -/** - * AutoIncrement annotation in schema.gql - */ -export const autoIncrementTag = /@autoIncrement/; - -/** - * Regex pattern to find table name in schema.gql - */ -export const tableNamePattern = /type\s+(\w+)\s+{/; - -/** - * Regex pattern to find field name having annotion "@autoIncrement" in schema.gql - * Consider only field type : Int!,ID!, BigInt, - * @returns regex expression : /(\w+):\s(Int|ID|BigInt)!?\s+@autoIncrement/ - */ -export function autoIncrementFieldPattern(): RegExp { - const fieldPattern = /(\w+):\s(Int|ID|BigInt)!?\s+/; - return new RegExp(fieldPattern.source + autoIncrementTag.source); -} +} \ No newline at end of file diff --git a/test/unit/graphql/__snapshots__/controller.test.ts.snap b/test/unit/graphql/__snapshots__/controller.test.ts.snap index e36c102..932de03 100644 --- a/test/unit/graphql/__snapshots__/controller.test.ts.snap +++ b/test/unit/graphql/__snapshots__/controller.test.ts.snap @@ -18,7 +18,7 @@ create index \`votes_big_decimal_index\` on \`votes\` (\`big_decimal\`)" exports[`GqlEntityController createEntityStores should work with autoincrement 1`] = ` "drop table if exists \`votes\`; -create table \`votes\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(128), \`authenticators\` json, \`big_number\` bigint, \`decimal\` float, \`big_decimal\` float); +create table \`votes\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(256), \`authenticators\` json, \`big_number\` bigint, \`decimal\` float, \`big_decimal\` float); create index \`votes_name_index\` on \`votes\` (\`name\`); create index \`votes_big_number_index\` on \`votes\` (\`big_number\`); create index \`votes_decimal_index\` on \`votes\` (\`decimal\`); @@ -27,14 +27,14 @@ create index \`votes_big_decimal_index\` on \`votes\` (\`big_decimal\`)" exports[`GqlEntityController createEntityStores should work with autoincrement and nested objects 1`] = ` "drop table if exists \`votes\`; -create table \`votes\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(128), \`authenticators\` json, \`big_number\` bigint, \`decimal\` float, \`big_decimal\` float, \`poster\` varchar(128)); +create table \`votes\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(256), \`authenticators\` json, \`big_number\` bigint, \`decimal\` float, \`big_decimal\` float, \`poster\` varchar(256)); create index \`votes_name_index\` on \`votes\` (\`name\`); create index \`votes_big_number_index\` on \`votes\` (\`big_number\`); create index \`votes_decimal_index\` on \`votes\` (\`decimal\`); create index \`votes_big_decimal_index\` on \`votes\` (\`big_decimal\`); create index \`votes_poster_index\` on \`votes\` (\`poster\`); drop table if exists \`posters\`; -create table \`posters\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(128) not null); +create table \`posters\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(256) not null); create index \`posters_name_index\` on \`posters\` (\`name\`)" `; From d49c3178d196680a338b560900c6a6a5d3604c75 Mon Sep 17 00:00:00 2001 From: 0xTitan Date: Sat, 4 Nov 2023 11:47:38 +0100 Subject: [PATCH 6/7] fix format, comments, tests, duplicate loop --- src/checkpoint.ts | 8 +++--- src/graphql/controller.ts | 28 ++++--------------- src/types.ts | 2 +- .../__snapshots__/controller.test.ts.snap | 9 ------ test/unit/graphql/controller.test.ts | 28 ++----------------- 5 files changed, 12 insertions(+), 63 deletions(-) diff --git a/src/checkpoint.ts b/src/checkpoint.ts index 6a0b258..8700b9e 100644 --- a/src/checkpoint.ts +++ b/src/checkpoint.ts @@ -73,10 +73,10 @@ export default class Checkpoint { level: opts?.logLevel || LogLevel.Error, ...(opts?.prettifyLogs ? { - transport: { - target: 'pino-pretty' + transport: { + target: 'pino-pretty' + } } - } : {}) }); @@ -205,7 +205,7 @@ export default class Checkpoint { await this.store.createStore(); await this.store.setMetadata(MetadataId.LastIndexedBlock, 0); - await this.entityController.createEntityStores(this.knex, this.schema); + await this.entityController.createEntityStores(this.knex); } /** diff --git a/src/graphql/controller.ts b/src/graphql/controller.ts index f5832a4..cf85550 100644 --- a/src/graphql/controller.ts +++ b/src/graphql/controller.ts @@ -199,21 +199,8 @@ export class GqlEntityController { * INDEX name (name) * ); * ``` - * - will execute the following SQL when declaring id as autoincrement: - * ```sql - * DROP TABLE IF EXISTS votes; - * CREATE TABLE votes ( - * id integer not null primary key autoincrement, - * name VARCHAR(128), - * ); - * ``` - * */ - public async createEntityStores( - knex: Knex, - schema: string - ): Promise<{ builder: Knex.SchemaBuilder }> { + public async createEntityStores(knex: Knex): Promise<{ builder: Knex.SchemaBuilder }> { let builder = knex.schema; if (this.schemaObjects.length === 0) { @@ -224,15 +211,7 @@ export class GqlEntityController { const tableName = pluralize(type.name.toLowerCase()); builder = builder.dropTableIfExists(tableName).createTable(tableName, t => { - //if there is no autoIncrement fields on current table, mark the id as primary - const tableHasAutoIncrement = this.getTypeFields(type).some(field => { - const directives = field.astNode?.directives ?? []; - const autoIncrementDirective = directives.find(dir => dir.name.value === 'autoIncrement'); - return autoIncrementDirective; - }) - - if (!tableHasAutoIncrement) - t.primary(['id']); + let tableHasAutoIncrement = false; this.getTypeFields(type).forEach(field => { const fieldType = field.type instanceof GraphQLNonNull ? field.type.ofType : field.type; @@ -244,6 +223,7 @@ export class GqlEntityController { //Check if field is declared as autoincrement if (autoIncrementDirective) { t.increments(field.name, { primaryKey: true }); + tableHasAutoIncrement = true; } else { const sqlType = this.getSqlType(field.type); let column = @@ -260,6 +240,8 @@ export class GqlEntityController { } } }); + + if (!tableHasAutoIncrement) t.primary(['id']); }); }); diff --git a/src/types.ts b/src/types.ts index a1d845a..dd3e301 100644 --- a/src/types.ts +++ b/src/types.ts @@ -111,4 +111,4 @@ export function isFullBlock(block: Block): block is FullBlock { export function isDeployTransaction(tx: Transaction | PendingTransaction): tx is DeployTransaction { return tx.type === 'DEPLOY'; -} \ No newline at end of file +} diff --git a/test/unit/graphql/__snapshots__/controller.test.ts.snap b/test/unit/graphql/__snapshots__/controller.test.ts.snap index 932de03..ba6996e 100644 --- a/test/unit/graphql/__snapshots__/controller.test.ts.snap +++ b/test/unit/graphql/__snapshots__/controller.test.ts.snap @@ -16,15 +16,6 @@ create index \`votes_decimal_index\` on \`votes\` (\`decimal\`); create index \`votes_big_decimal_index\` on \`votes\` (\`big_decimal\`)" `; -exports[`GqlEntityController createEntityStores should work with autoincrement 1`] = ` -"drop table if exists \`votes\`; -create table \`votes\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(256), \`authenticators\` json, \`big_number\` bigint, \`decimal\` float, \`big_decimal\` float); -create index \`votes_name_index\` on \`votes\` (\`name\`); -create index \`votes_big_number_index\` on \`votes\` (\`big_number\`); -create index \`votes_decimal_index\` on \`votes\` (\`decimal\`); -create index \`votes_big_decimal_index\` on \`votes\` (\`big_decimal\`)" -`; - exports[`GqlEntityController createEntityStores should work with autoincrement and nested objects 1`] = ` "drop table if exists \`votes\`; create table \`votes\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(256), \`authenticators\` json, \`big_number\` bigint, \`decimal\` float, \`big_decimal\` float, \`poster\` varchar(256)); diff --git a/test/unit/graphql/controller.test.ts b/test/unit/graphql/controller.test.ts index 28100b8..0d9ae48 100644 --- a/test/unit/graphql/controller.test.ts +++ b/test/unit/graphql/controller.test.ts @@ -72,36 +72,12 @@ type Vote { `; const controller = new GqlEntityController(schema); - const { builder } = await controller.createEntityStores(mockKnex, schema); + const { builder } = await controller.createEntityStores(mockKnex); const createQuery = builder.toString(); expect(createQuery).toMatchSnapshot(); }); - it('should work with autoincrement', async () => { - let schema = ` -scalar BigInt -scalar Decimal -scalar BigDecimal - -type Vote { - id: Int! @autoIncrement - name: String - authenticators: [String] - big_number: BigInt - decimal: Decimal - big_decimal: BigDecimal -} - - `; - - schema = extendSchema(schema); - const controller = new GqlEntityController(schema); - const { builder } = await controller.createEntityStores(mockKnex, schema); - const createQuery = builder.toString(); - expect(createQuery).toMatchSnapshot(); - }); - it('should work with autoincrement and nested objects', async () => { let schema = ` scalar BigInt @@ -127,7 +103,7 @@ type Poster { schema = extendSchema(schema); const controller = new GqlEntityController(schema); - const { builder } = await controller.createEntityStores(mockKnex, schema); + const { builder } = await controller.createEntityStores(mockKnex); const createQuery = builder.toString(); expect(createQuery).toMatchSnapshot(); }); From 2e92f083d4547aaf748c98cf8b6d5f43a95394e8 Mon Sep 17 00:00:00 2001 From: 0xTitan Date: Mon, 6 Nov 2023 22:48:51 +0100 Subject: [PATCH 7/7] Rm comments, one graph test, fix package.json --- package.json | 2 +- src/graphql/controller.ts | 1 - .../__snapshots__/controller.test.ts.snap | 10 -------- test/unit/graphql/controller.test.ts | 23 ------------------- 4 files changed, 1 insertion(+), 35 deletions(-) diff --git a/package.json b/package.json index 2cc1044..53dffe9 100644 --- a/package.json +++ b/package.json @@ -61,4 +61,4 @@ "dist/**/*", "src/**/*" ] -} \ No newline at end of file +} diff --git a/src/graphql/controller.ts b/src/graphql/controller.ts index cf85550..278f9b4 100644 --- a/src/graphql/controller.ts +++ b/src/graphql/controller.ts @@ -220,7 +220,6 @@ export class GqlEntityController { const directives = field.astNode?.directives ?? []; const autoIncrementDirective = directives.find(dir => dir.name.value === 'autoIncrement'); - //Check if field is declared as autoincrement if (autoIncrementDirective) { t.increments(field.name, { primaryKey: true }); tableHasAutoIncrement = true; diff --git a/test/unit/graphql/__snapshots__/controller.test.ts.snap b/test/unit/graphql/__snapshots__/controller.test.ts.snap index ba6996e..5c652aa 100644 --- a/test/unit/graphql/__snapshots__/controller.test.ts.snap +++ b/test/unit/graphql/__snapshots__/controller.test.ts.snap @@ -8,16 +8,6 @@ exports[` 3`] = `"'id' field for type Participant is not a scalar type."`; exports[`GqlEntityController createEntityStores should work 1`] = ` "drop table if exists \`votes\`; -create table \`votes\` (\`id\` integer not null, \`name\` varchar(256), \`authenticators\` json, \`big_number\` bigint, \`decimal\` float, \`big_decimal\` float, primary key (\`id\`)); -create index \`votes_id_index\` on \`votes\` (\`id\`); -create index \`votes_name_index\` on \`votes\` (\`name\`); -create index \`votes_big_number_index\` on \`votes\` (\`big_number\`); -create index \`votes_decimal_index\` on \`votes\` (\`decimal\`); -create index \`votes_big_decimal_index\` on \`votes\` (\`big_decimal\`)" -`; - -exports[`GqlEntityController createEntityStores should work with autoincrement and nested objects 1`] = ` -"drop table if exists \`votes\`; create table \`votes\` (\`id\` integer not null primary key autoincrement, \`name\` varchar(256), \`authenticators\` json, \`big_number\` bigint, \`decimal\` float, \`big_decimal\` float, \`poster\` varchar(256)); create index \`votes_name_index\` on \`votes\` (\`name\`); create index \`votes_big_number_index\` on \`votes\` (\`big_number\`); diff --git a/test/unit/graphql/controller.test.ts b/test/unit/graphql/controller.test.ts index 0d9ae48..b962d78 100644 --- a/test/unit/graphql/controller.test.ts +++ b/test/unit/graphql/controller.test.ts @@ -56,29 +56,6 @@ type Vote { }); it('should work', async () => { - const schema = ` -scalar BigInt -scalar Decimal -scalar BigDecimal - -type Vote { - id: Int! - name: String - authenticators: [String] - big_number: BigInt - decimal: Decimal - big_decimal: BigDecimal -} - `; - - const controller = new GqlEntityController(schema); - const { builder } = await controller.createEntityStores(mockKnex); - - const createQuery = builder.toString(); - expect(createQuery).toMatchSnapshot(); - }); - - it('should work with autoincrement and nested objects', async () => { let schema = ` scalar BigInt scalar Decimal