diff --git a/.eslintrc b/.eslintrc index 9b220f8cf0..454761980f 100644 --- a/.eslintrc +++ b/.eslintrc @@ -2,7 +2,12 @@ "parser": "@typescript-eslint/parser", // Specifies the ESLint parser "parserOptions": { "ecmaVersion": 2020, // Allows for the parsing of modern ECMAScript features - "sourceType": "module" // Allows for the use of imports + "sourceType": "module", // Allows for the use of imports + // "babelOptions": { + // "plugins": [ + // "@babel/plugin-syntax-import-assertions" + // ], + // }, }, "extends": [ "plugin:import/errors", diff --git a/documentation/ai-scraper/README.md b/documentation/ai-scraper/README.md index 74636201fd..e0a777a8ce 100644 --- a/documentation/ai-scraper/README.md +++ b/documentation/ai-scraper/README.md @@ -78,7 +78,7 @@ participant SJR as Scraper Job Repository participant LLM as LLM Scraper participant PU as PromptUtils participant LLMProvider as LLM Provider -participant T as Text Preprocessor +participant Pre as HTML Preprocessor participant DTU as DomainTaskUtils loop Periodically @@ -95,8 +95,8 @@ loop Periodically PU-->LLM: Prompt loop for each job - LLM->>T: convert HTML to text and minimize - T-->>LLM: minimzed Text + LLM->>Pre: convert HTML to text and minimize + Pre-->>LLM: minimzed Text LLM->>PU: add to Prompt if have token budget end @@ -118,3 +118,66 @@ end 1. Web-workers 2. IPFS to collect URLs? + +### Prompt Builder + +```mermaid +classDiagram + + class PromptDirector { + +makePurchaseHistoryPrompt(data) Prompt + } + + PromptDirector --> PromptBuilder + + class PromptBuilder { + <> + +setExemplars(exemplars) + +setRole(role) + +setQuestion(question) + +setAnswerStructure(structure) + +setData(data) + +getPrompt() Prompt + } + + class PurchaseHistoryPromptBuilder { + + } + PromptBuilder <|-- PurchaseHistoryPromptBuilder + + class ShoppingCartPromptBuilder { + + } + PromptBuilder <|-- ShoppingCartPromptBuilder + + %% Collection prompts + + class CollectionPromptBuilder { + + <> + } + PromptBuilder <|-- CollectionPromptBuilder + + class ProductCollectionPromptBuilder { + + } + CollectionPromptBuilder <|-- ProductCollectionPromptBuilder + + class GameCollectionPromptBuilder { + + } + CollectionPromptBuilder <|-- GameCollectionPromptBuilder + + + %% Single Item Prompts + class ItemDetailsPromptBuilder { + + } + PromptBuilder <|-- ItemDetailsPromptBuilder + + class OrderDetailsPromptBuilder { + + } + ItemDetailsPromptBuilder <|-- OrderDetailsPromptBuilder + +``` \ No newline at end of file diff --git a/documentation/persistence layer/README.md b/documentation/persistence layer/README.md index faaa907a59..39e3669835 100644 --- a/documentation/persistence layer/README.md +++ b/documentation/persistence layer/README.md @@ -44,13 +44,18 @@ We add the migrator definition to the same file where we defined the Animal clas } } ``` -3. Every entity requires a schema which is analogous to table definitions in SQL. We add the schema to [VolatileStorageSchema.ts](./../../packages/persistence/src/volatile/VolatileStorageSchema.ts) by adding an object of type [VolatileTableIndex](./../../packages/persistence/src/volatile/VolatileTableIndex.ts). +3. Every entity requires a schema which is analogous to table definitions in SQL. We add the schema to [VolatileStorageSchemaProvider.ts](./../../packages/persistence/src/volatile/VolatileStorageSchemaProvider.ts) by adding an object of type [VolatileTableIndex](./../../packages/persistence/src/volatile/VolatileTableIndex.ts). ``` new VolatileTableIndex( ERecordKey.ANIMAL, // The name of our object store / table "id", // primary key field. false, // false disables the auto-increment key generator. new AnimalMigrator(), // migrator that our database client will use to convert data into animal objects. + + EBackupPriority.NORMAL, // Backup priority + 3600 * 1000, // Backup Interval in milliseconds + config.backupChunkSizeTarget, + [], // Index ), ``` @@ -65,7 +70,12 @@ We add the migrator definition to the same file where we defined the Animal clas "id", // primary key field. false, // false disables the auto-increment key generator. new AnimalMigrator(), - [['name', false], ['someOtherField', false], [['comp1', 'comp2'], true] + + EBackupPriority.NORMAL, // Backup priority + 3600 * 1000, // Backup Interval in milliseconds + config.backupChunkSizeTarget, + + [['name', false], ['someOtherField', false], [['comp1', 'comp2'], true]] ), @@ -78,12 +88,7 @@ In the examples, **this.persistence** is an instance of DataWalletPersistence. A **Add an animal**: We add a new object to the store by wrapping it in a [VolatileStorageMetadata](./../../packages/objects/src/businessObjects/VolatileStorageMetadata.ts) object. ``` const myDog = new Animal("XX12", "Tom"); - const metadata = new VolatileStorageMetadata( - EBackupPriority.NORMAL, - myDog, - Animal.CURRENT_VERSION, - ); - return this.persistence.updateRecord(ERecordKey.ANIMAL, metadata); + return this.persistence.updateRecord(ERecordKey.ANIMAL, myDog); ``` **Update an animal**: Update works exactly the same way as adding a new object. The engine will update and existing object if an object with the same primary key exists. @@ -91,12 +96,12 @@ In the examples, **this.persistence** is an instance of DataWalletPersistence. A **Delete an animal**: Current we only support deleting by the primary key. The value of the primary key needs to be wrapped in a [VolatileStorageKey](./../../packages/objects/src/primitives/VolatileStorageKey.ts) object. ``` - return this.persistence.deleteRecord(ERecordKey.ANIMAL, VolatileStorageKey("XX12"), EBackupPriority.NORMAL); // Deletes Tom from the animal store. + return this.persistence.deleteRecord(ERecordKey.ANIMAL, "XX12", EBackupPriority.NORMAL); // Deletes Tom from the animal store. ``` **Find an animal by primary key**: ``` - return this.persistence.getObject(ERecordKey.ANIMAL, VolatileStorageKey("XX12"), EBackupPriority.NORMAL); // Deletes Tom from the animal store. + return this.persistence.getObject(ERecordKey.ANIMAL, "XX12", EBackupPriority.NORMAL); // Deletes Tom from the animal store. ``` **Get all animals**: @@ -105,7 +110,7 @@ In the examples, **this.persistence** is an instance of DataWalletPersistence. A ``` **Get all animals by an index**: ``` - return this.persistence.getAllByIndex(ERecordKey.ANIMAL, "name", IDBValidKey("Tom")); // Analogous to the getCursor function. But it returns all the objects. + return this.persistence.getAllByIndex(ERecordKey.ANIMAL, "name", "Tom"); // Analogous to the getCursor function. But it returns all the objects. ``` **Get all primary keys**: ``` @@ -115,7 +120,7 @@ In the examples, **this.persistence** is an instance of DataWalletPersistence. A **Get cursor**: Cursors can return all the objects or a subset matching an index field. For details, please check [IndexDB](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API/Using_IndexedDB) document. ``` - return this.persistence.getCursor(ERecordKey.ANIMAL, "name", IDBValidKey("Tom")); // will return a cursor with all the Toms. + return this.persistence.getCursor(ERecordKey.ANIMAL, "name", "Tom"); // will return a cursor with all the Toms. ``` diff --git a/documentation/shopping-data/README.md b/documentation/shopping-data/README.md new file mode 100644 index 0000000000..f87f5c14cb --- /dev/null +++ b/documentation/shopping-data/README.md @@ -0,0 +1 @@ +# TODO \ No newline at end of file diff --git a/packages/ai-scraper/package.json b/packages/ai-scraper/package.json index f65bf73e76..fa56d0502e 100644 --- a/packages/ai-scraper/package.json +++ b/packages/ai-scraper/package.json @@ -1,7 +1,7 @@ { "name": "@snickerdoodlelabs/ai-scraper", "version": "0.0.18", - "description": "Utilities for parsing and understanding SDQL Queries", + "description": "Web scraper for Data Wallet browser extension", "license": "MIT", "repository": { "type": "git", @@ -41,9 +41,14 @@ "dependencies": { "@snickerdoodlelabs/common-utils": "workspace:^", "@snickerdoodlelabs/objects": "workspace:^", + "@snickerdoodlelabs/persistence": "workspace:^", + "@snickerdoodlelabs/shopping-data": "workspace:^", "ethers": "^6.10.0", + "html-to-text": "^9.0.5", "inversify": "^6.0.2", + "js-tiktoken": "^1.0.8", "neverthrow": "^5.1.0", - "neverthrow-result-utils": "^2.0.2" + "neverthrow-result-utils": "^2.0.2", + "openai": "^4.0.1" } } diff --git a/packages/ai-scraper/src/implementations/business/LLMScraperService.ts b/packages/ai-scraper/src/implementations/business/LLMScraperService.ts new file mode 100644 index 0000000000..39e2a776e4 --- /dev/null +++ b/packages/ai-scraper/src/implementations/business/LLMScraperService.ts @@ -0,0 +1,271 @@ +/*** + * The main class that connects everything together + */ + +import { ILogUtils, ILogUtilsType } from "@snickerdoodlelabs/common-utils"; +import { + URLString, + HTMLString, + ScraperError, + LLMError, + PersistenceError, + ELanguageCode, + DomainTask, + ETask, + LLMData, + LLMResponse, + Prompt, + EKnownDomains, + ProductCategories, + PurchasedProduct, + UnknownProductCategory, + InvalidURLError, +} from "@snickerdoodlelabs/objects"; +import { + IPurchaseRepository, + IPurchaseRepositoryType, + IPurchaseUtils, + IPurchaseUtilsType, +} from "@snickerdoodlelabs/shopping-data"; +import { inject, injectable } from "inversify"; +import { ResultAsync, errAsync, ok, okAsync } from "neverthrow"; +import { ResultUtils } from "neverthrow-result-utils"; + +import { + IAmazonNavigationUtils, + IAmazonNavigationUtilsType, + IHTMLPreProcessor, + IHTMLPreProcessorType, + ILLMProductMetaUtils, + ILLMProductMetaUtilsType, + ILLMRepository, + ILLMRepositoryType, + ILLMPurchaseHistoryUtils, + ILLMPurchaseHistoryUtilsType, + IPromptDirector, + IPromptDirectorType, + IScraperService, + IWebpageClassifier, + IWebpageClassifierType, + ILLMPurchaseValidatorType, + ILLMPurchaseValidator, +} from "@ai-scraper/interfaces/index.js"; + +@injectable() +export class LLMScraperService implements IScraperService { + public constructor( + @inject(ILogUtilsType) + private logUtils: ILogUtils, + @inject(IHTMLPreProcessorType) + private htmlPreProcessor: IHTMLPreProcessor, + @inject(ILLMRepositoryType) + private llmRepository: ILLMRepository, + @inject(IPromptDirectorType) + private promptDirector: IPromptDirector, + @inject(IWebpageClassifierType) + private webpageClassifier: IWebpageClassifier, + @inject(IPurchaseUtilsType) + private purchaseUtils: IPurchaseUtils, + @inject(ILLMPurchaseHistoryUtilsType) + private purchaseHistoryLLMUtils: ILLMPurchaseHistoryUtils, + @inject(IPurchaseRepositoryType) + private purchaseRepository: IPurchaseRepository, + @inject(ILLMProductMetaUtilsType) + private productMetaUtils: ILLMProductMetaUtils, + @inject(IAmazonNavigationUtilsType) + private amazonNavigationUtils: IAmazonNavigationUtils, + @inject(ILLMPurchaseValidatorType) + private llmPurchaseValidator: ILLMPurchaseValidator, + ) {} + + public poll(): ResultAsync { + return errAsync(new ScraperError("poll not implemented")); + } + + public classifyURL( + url: URLString, + language: ELanguageCode, + ): ResultAsync { + return this.webpageClassifier.classify(url, language); + } + /** + * Now we will scrape it immmediately and assume the task is a Amazon Purchase History Taks. In future it's done by a job executor with a rate limiter + */ + public scrape( + url: URLString, + html: HTMLString, + suggestedDomainTask: DomainTask, + ): ResultAsync { + if (suggestedDomainTask.taskType == ETask.PurchaseHistory) { + return this.scrapePurchaseHistory(url, html, suggestedDomainTask); + } + return errAsync(new ScraperError("Task type not supported.")); + } + + private scrapePurchaseHistory( + url: URLString, + html: HTMLString, + suggestedDomainTask: DomainTask, + ): ResultAsync { + /* + This is a two step process. + Step 1: get purchase information from LLM + Step 2: for each purchase, if category is null/unknown, then add it to the next prompt to get meta information + */ + // throw new Error("Method not implemented."); + // 1. build prompt + // 2. execute prompt + // 3. parse response for information + // 4. persist information + + const languageResult = this.htmlPreProcessor.getLanguage(html); + const promptResult = this.buildPrompt(url, html, suggestedDomainTask); + + return ResultUtils.combine([languageResult, promptResult]).andThen( + ([language, prompt]) => { + return this.llmRepository + .executePrompt(prompt) + .andThen((llmResponse) => { + return this.llmPurchaseValidator + .fixMalformedJSONArrayResponse(llmResponse) + .andThen((sanitizedLLMResponse) => { + return this.processLLMPurchaseResponse( + prompt, + suggestedDomainTask, + language, + sanitizedLLMResponse, + ); + }); + }); + }, + ); + } + + private buildPrompt( + url: URLString, + html: HTMLString, + domainTask: DomainTask, + ): ResultAsync { + if ( + domainTask.taskType == ETask.PurchaseHistory && + domainTask.domain == EKnownDomains.Amazon + ) { + const preprocessingOptions = + this.amazonNavigationUtils.getPurchaseHistoryPagePreprocessingOptions(); + return this.htmlPreProcessor + .htmlToText(html, preprocessingOptions) + .andThen((text) => { + text = text.substring( + 0, + this.llmRepository.defaultMaxTokens() - 1000, + ); + return this.promptDirector.makePurchaseHistoryPrompt(LLMData(text)); + }); + } + return errAsync(new LLMError("Task type or domain not supported.")); + } + + private processLLMPurchaseResponse( + prompt: Prompt, + domainTask: DomainTask, + language: ELanguageCode, + sanitizedLLMResponse: LLMResponse, + ): ResultAsync { + if (domainTask.taskType == ETask.PurchaseHistory) { + return this.purchaseHistoryLLMUtils + .parsePurchases(domainTask.domain, language, sanitizedLLMResponse) + .andThen((purchases) => { + // Find a better way to refactor it + // return this.savePurchases(purchases); + return this.llmPurchaseValidator + .trimHalucinatedPurchases(prompt, purchases) + .andThen((validPurchases) => { + if (validPurchases.length < purchases.length) { + } + return this.savePurchases(validPurchases).andThen(() => { + return this.scrapeProductMeta( + domainTask, + language, + validPurchases, + ); + }); + }); + }); + } + return errAsync(new LLMError("Task type not supported.")); + } + + private savePurchases( + purchases: PurchasedProduct[], + ): ResultAsync { + // return errAsync(new PersistenceError("savePurchases not implemented.")); + const results = purchases.map((purchase) => { + return this.purchaseRepository.add(purchase); + }); + + return ResultUtils.combine(results).map(() => {}); + } + + private scrapeProductMeta( + domainTask: DomainTask, + language: ELanguageCode, + purchases: PurchasedProduct[], + ): ResultAsync { + // convert purchases to LLM data first + const nullCategoryPurchases = + this.purchaseUtils.getNullCategoryPurchases(purchases); + + if (nullCategoryPurchases.length == 0) { + return okAsync(undefined); + } + + return this.scrapeCategory(domainTask, language, nullCategoryPurchases); + } + + private scrapeCategory( + domainTask: DomainTask, + language: ELanguageCode, + nullCategoryPurchases: PurchasedProduct[], + ) { + const purchaseJsonArr = nullCategoryPurchases.map((purchase, idx) => { + return { + product_id: idx, + product_name: purchase.name, + }; + }); + const llmData = LLMData(JSON.stringify(purchaseJsonArr)); + + return this.promptDirector + .makeProductMetaPrompt(llmData) + .andThen((prompt) => { + return this.llmRepository + .executePrompt(prompt) + .andThen((llmResponse) => { + return this.llmPurchaseValidator + .fixMalformedJSONArrayResponse(llmResponse) + .andThen((sanitizedLLMResponse) => { + const productMetas = this.productMetaUtils.parseMeta( + domainTask.domain, + language, + sanitizedLLMResponse, + ); + // TODO + return productMetas.andThen((metas) => { + const purchasesToUpdate = metas.map((meta) => { + const purchase = + nullCategoryPurchases[parseInt(meta.productId)]; // this indexing is not correct + purchase.category = meta.category ?? UnknownProductCategory; // TODO convert to enum + purchase.keywords = meta.keywords; + return purchase; + }); + + return this.savePurchases(purchasesToUpdate); + }); + }); + }); + }) + .mapErr((err) => { + return new ScraperError(err.message, err); + }); + } +} diff --git a/packages/ai-scraper/src/implementations/business/PromptDirector.ts b/packages/ai-scraper/src/implementations/business/PromptDirector.ts new file mode 100644 index 0000000000..6122369773 --- /dev/null +++ b/packages/ai-scraper/src/implementations/business/PromptDirector.ts @@ -0,0 +1,80 @@ +import { + Exemplar, + LLMAnswerStructure, + LLMData, + LLMError, + LLMQuestion, + LLMRole, + Prompt, +} from "@snickerdoodlelabs/objects"; +import { inject, injectable } from "inversify"; +import { ResultAsync } from "neverthrow"; + +import { + IPromptDirector, + ILLMPurchaseHistoryUtilsType, + ILLMPurchaseHistoryUtils, + IPromptBuilderFactoryType, + IPromptBuilderFactory, + IPromptBuilder, + ILLMProductMetaUtilsType, + ILLMProductMetaUtils, +} from "@ai-scraper/interfaces/index.js"; + +@injectable() +export class PromptDirector implements IPromptDirector { + constructor( + @inject(IPromptBuilderFactoryType) + private promptBuilderFactory: IPromptBuilderFactory, + @inject(ILLMPurchaseHistoryUtilsType) + private purchaseHistoryLLMUtils: ILLMPurchaseHistoryUtils, + @inject(ILLMProductMetaUtilsType) + private productMetaUtils: ILLMProductMetaUtils, + ) {} + + /** + * @description + */ + public makePurchaseHistoryPrompt( + data: LLMData, + ): ResultAsync { + // Acquire + const builder = this.promptBuilderFactory.purchaseHistory(); + const role = this.purchaseHistoryLLMUtils.getRole(); + const question = this.purchaseHistoryLLMUtils.getQuestion(); + const answerStructure = this.purchaseHistoryLLMUtils.getAnswerStructure(); + + // Attend + return this.make(builder, [], role, question, answerStructure, data); + } + + public makeProductMetaPrompt(data: LLMData): ResultAsync { + // Acquire + const builder = this.promptBuilderFactory.productMeta(); + const role = this.productMetaUtils.getRole(); + const question = this.productMetaUtils.getQuestion(); + const answerStructure = this.productMetaUtils.getAnswerStructure(); + + // Attend + return this.make(builder, [], role, question, answerStructure, data); + } + + private make( + builder: IPromptBuilder, + exemplars: Exemplar[], + role: LLMRole, + question: LLMQuestion, + answerStructure: LLMAnswerStructure, + data: LLMData, + ): ResultAsync { + // Assemble + builder.setExemplars(exemplars); + builder.setRole(role); + builder.setQuestion(question); + builder.setAnswerStructure(answerStructure); + builder.setData(data); + + // Attend + return builder.getPrompt(); + } +} diff --git a/packages/ai-scraper/src/implementations/business/index.ts b/packages/ai-scraper/src/implementations/business/index.ts index f1fb22390e..bf4bb80c63 100644 --- a/packages/ai-scraper/src/implementations/business/index.ts +++ b/packages/ai-scraper/src/implementations/business/index.ts @@ -1 +1,3 @@ +export * from "@ai-scraper/implementations/business/LLMScraperService.js"; +export * from "@ai-scraper/implementations/business/PromptDirector.js"; export * from "@ai-scraper/implementations/business/utils/index.js"; diff --git a/packages/ai-scraper/src/implementations/business/utils/KeywordUtils.ts b/packages/ai-scraper/src/implementations/business/utils/KeywordUtils.ts index f2a184962c..16fc7a118b 100644 --- a/packages/ai-scraper/src/implementations/business/utils/KeywordUtils.ts +++ b/packages/ai-scraper/src/implementations/business/utils/KeywordUtils.ts @@ -1,13 +1,9 @@ -import { ELanguageCode } from "@snickerdoodlelabs/objects"; +import { ELanguageCode, ETask, Keyword } from "@snickerdoodlelabs/objects"; import { inject, injectable } from "inversify"; import { ResultAsync, okAsync } from "neverthrow"; -import { DefaultKeywords } from "@ai-scraper/data/index.js"; import { IKeywordUtils, - Keyword, - ETask, - Keywords, IKeywordRepository, IKeywordRepositoryType, } from "@ai-scraper/interfaces/index.js"; diff --git a/packages/ai-scraper/src/implementations/business/utils/LLMProductMetaUtilsChatGPT.ts b/packages/ai-scraper/src/implementations/business/utils/LLMProductMetaUtilsChatGPT.ts new file mode 100644 index 0000000000..eb32515462 --- /dev/null +++ b/packages/ai-scraper/src/implementations/business/utils/LLMProductMetaUtilsChatGPT.ts @@ -0,0 +1,110 @@ +import { + ILogUtils, + ILogUtilsType, + ITimeUtils, + ITimeUtilsType, +} from "@snickerdoodlelabs/common-utils"; +import { + DomainName, + ELanguageCode, + LLMAnswerStructure, + LLMError, + LLMQuestion, + LLMResponse, + LLMRole, + UnixTimestamp, + ProductKeyword, + PurchaseId, + PurchasedProduct, + ProductCategories, + ProductMeta, + ProductId, +} from "@snickerdoodlelabs/objects"; +import { inject, injectable } from "inversify"; +import { ResultAsync, errAsync, okAsync } from "neverthrow"; + +import { + IPurchaseBlock, + ILLMProductMetaUtils, + IProductMetaBlock, +} from "@ai-scraper/interfaces/index.js"; + +/** + * @description We will make this updateable via ipfs in future. For now, + * it will implement utils for all the tasks. Later we can break this into multiple classes + */ +@injectable() +export class LLMProductMetaUtilsChatGPT implements ILLMProductMetaUtils { + public constructor( + @inject(ITimeUtilsType) + private timeUtils: ITimeUtils, + @inject(ILogUtilsType) + private logUtils: ILogUtils, + ) {} + public getRole(): LLMRole { + return LLMRole( + "You are an expert in understanding product categories and keywords.", + ); + } + + public getQuestion(): LLMQuestion { + // return LLMQuestion( + // "Can you get the product names from the following text? I also need the product brand, price, classification, keywords, and date purchased. Give response in a JSON array in the preceding format.", + // ); + return LLMQuestion( + `Classification denotes the category of the product and keywords describe the products using a few keywords. For categories choose from [${ProductCategories.join( + ", ", + )}] only. A product has one category and multiple keywords. Here is a list of products seperated by new lines.`, + ); + } + + public getAnswerStructure(): LLMAnswerStructure { + return LLMAnswerStructure( + `I need to extract categories and keywords of some products. Sub-category is more specific. I need all the output in this format: + \n\nJSON format for each product: \n + { + product_id: number, + sub_category: string, + category: string, + keywords: string[], + } + \n\nGive response in a JSON array in the preceding format. The array is enclosed in third brackets.`, + ); + } + + public parseMeta( + domain: DomainName, + language: ELanguageCode, + llmResponse: LLMResponse, + ): ResultAsync { + let metas: IProductMetaBlock[] = []; + try { + metas = JSON.parse(llmResponse); + } catch (e) { + // this.logUtils.warning(`No product meta. LLMRReponse: ${llmResponse}`); + // console.log(`No product meta. LLMRReponse: ${llmResponse}`); + return errAsync( + new LLMError(`No product meta. LLMRReponse: ${llmResponse}`, e), + ); + } + + const validMetas = metas.reduce((accumulator, meta) => { + if (meta.product_id == null) { + this.logUtils.debug(`Invalid product id ${meta.product_id} in ${meta}`); + // console.log(`Invalid product id ${meta.product_id} in ${meta}`); + } else { + // console.log(`got valid meta ${meta}`); + accumulator.push( + new ProductMeta( + ProductId(meta.product_id.toString()), + meta.category, + (meta.keywords ?? []) as ProductKeyword[], + ), + ); + } + return accumulator; + }, [] as ProductMeta[]); + + return okAsync(validMetas); + } +} diff --git a/packages/ai-scraper/src/implementations/business/utils/LLMPurchaseHistoryUtilsChatGPT.ts b/packages/ai-scraper/src/implementations/business/utils/LLMPurchaseHistoryUtilsChatGPT.ts new file mode 100644 index 0000000000..7240c9e4ca --- /dev/null +++ b/packages/ai-scraper/src/implementations/business/utils/LLMPurchaseHistoryUtilsChatGPT.ts @@ -0,0 +1,155 @@ +import { + ILogUtils, + ILogUtilsType, + ITimeUtils, + ITimeUtilsType, +} from "@snickerdoodlelabs/common-utils"; +import { + DomainName, + ELanguageCode, + LLMAnswerStructure, + LLMError, + LLMQuestion, + LLMResponse, + LLMRole, + UnixTimestamp, + ProductKeyword, + PurchaseId, + PurchasedProduct, + UnknownProductCategory, +} from "@snickerdoodlelabs/objects"; +import { inject, injectable } from "inversify"; +import { ResultAsync, errAsync, okAsync } from "neverthrow"; + +import { + IPurchaseBlock, + ILLMPurchaseHistoryUtils, +} from "@ai-scraper/interfaces/index.js"; + +/** + * @description We will make this updateable via ipfs in future. For now, + * it will implement utils for all the tasks. Later we can break this into multiple classes + */ +@injectable() +export class LLMPurchaseHistoryUtilsChatGPT + implements ILLMPurchaseHistoryUtils +{ + public constructor( + @inject(ITimeUtilsType) + private timeUtils: ITimeUtils, + @inject(ILogUtilsType) + private logUtils: ILogUtils, + ) {} + public getRole(): LLMRole { + return LLMRole("You are an expert in understanding e-commerce."); + } + + public getQuestion(): LLMQuestion { + // return LLMQuestion( + // "Can you get the product names from the following text? I also need the product brand, price, classification, keywords, and date purchased. Give response in a JSON array in the preceding format.", + // ); + return LLMQuestion( + `I need the purchase history from the following content. You need to follow theses rules: + A purchase history must have a product name, price, and date of purchase. It can also have brand, classification, keywords which are optional. + Classification denotes the category of the product and keywords describe the products using a few key words. + The purchase date and price cannot be null. + Do not include a purchase information in the output if the purchase date or price is missing. + Do not push yourself hard and do not generate imaginary purchases. + Give response in a JSON array in the preceding format.`, + ); + } + + public makePurchaseId( + purchase: IPurchaseBlock, + timestampPurchased: UnixTimestamp, + ): PurchaseId { + return PurchaseId( + `${purchase.name}-${timestampPurchased}`.replace(" ", "-"), + ); + } + + public getAnswerStructure(): LLMAnswerStructure { + return LLMAnswerStructure( + `I need to extract purchase information. I need all the output in this format: + \n\nJSON format: \n + { + name: string, + brand: string, + price: number, + classification: string, + keywords: string[], + date: string + }`, + ); + } + + public parsePurchases( + domain: DomainName, + language: ELanguageCode, + llmResponse: LLMResponse, + ): ResultAsync { + let purchases: IPurchaseBlock[] = []; + try { + purchases = JSON.parse(llmResponse); + } catch (e) { + // return errAsync(new LLMError((e as Error).message, e)); + // this.logUtils.warning(`No product history. LLMRReponse: ${llmResponse}`); + return errAsync( + new LLMError(`No product history. LLMRReponse: ${llmResponse}`, e), + ); + } + // worst possible parser + const purchasedProducts = purchases.reduce((accumulator, purchase) => { + const timestampPurchased = this.timeUtils.parseToSDTimestamp( + purchase.date, + ); + const price = this.parsePrice(purchase.price); + + if ( + timestampPurchased == null || + timestampPurchased > this.timeUtils.getUnixNow() + ) { + this.logUtils.debug( + `Invalid purchase date ${purchase.date} for ${purchase.name}`, + ); + } else if (price == 0) { + this.logUtils.debug( + `Invalid price ${purchase.price} for ${purchase.name}`, + ); + } else { + const category = purchase.classification ?? UnknownProductCategory; + accumulator.push( + new PurchasedProduct( + domain, + language, + this.makePurchaseId(purchase, timestampPurchased), + purchase.name, + purchase.brand, + price, + timestampPurchased, + this.timeUtils.getUnixNow(), + null, + null, + null, + category, + purchase.keywords as ProductKeyword[], + ), + ); + } + return accumulator; + }, [] as PurchasedProduct[]); + + return okAsync(purchasedProducts); + } + + public parsePrice(priceStr: string | number): number { + if (typeof priceStr === "number") { + return priceStr; + } + try { + return parseFloat(priceStr.replace("$", "")); // TODO make a regex to extract the decimal number instead of this. + } catch (e) { + return 0; + } + } +} diff --git a/packages/ai-scraper/src/implementations/business/utils/OpenAIUtils.ts b/packages/ai-scraper/src/implementations/business/utils/OpenAIUtils.ts new file mode 100644 index 0000000000..12c0f68cb9 --- /dev/null +++ b/packages/ai-scraper/src/implementations/business/utils/OpenAIUtils.ts @@ -0,0 +1,28 @@ +import { LLMError, LLMResponse } from "@snickerdoodlelabs/objects"; +import { injectable } from "inversify"; +import { ResultAsync, errAsync, okAsync } from "neverthrow"; +import OpenAI from "openai"; +import { + ChatCompletion, + ChatCompletionCreateParamsNonStreaming, +} from "openai/resources/chat"; + +@injectable() +export class OpenAIUtils { + public getLLMResponseNonStreaming( + client: OpenAI, + params: ChatCompletionCreateParamsNonStreaming, + ): ResultAsync { + const completionResult = ResultAsync.fromPromise( + client.chat.completions.create(params), + (e) => new LLMError((e as Error).message, e), + ); + return completionResult.andThen((completion) => { + const content = completion.choices[0].message.content; + if (content == null) { + return errAsync(new LLMError("No content in response")); + } + return okAsync(LLMResponse(content)); + }); + } +} diff --git a/packages/ai-scraper/src/implementations/business/utils/ProductMetaPromptBuilder.ts b/packages/ai-scraper/src/implementations/business/utils/ProductMetaPromptBuilder.ts new file mode 100644 index 0000000000..fe3173097e --- /dev/null +++ b/packages/ai-scraper/src/implementations/business/utils/ProductMetaPromptBuilder.ts @@ -0,0 +1,59 @@ +import { Exemplar, LLMError, Prompt } from "@snickerdoodlelabs/objects"; +import { ResultAsync, errAsync, okAsync } from "neverthrow"; + +import { PromptBuilder } from "@ai-scraper/implementations/business/utils/PromptBuilder.js"; +import { IProductMetaPromptBuilder } from "@ai-scraper/interfaces/index.js"; + +/** + * @description this class is responsible for building prompts for purchase history + */ +export class ProductMetaPromptBuilder + extends PromptBuilder + implements IProductMetaPromptBuilder +{ + /** + * + * @description ignore exemplars for purchase history as this breaks the token limit easily + */ + public setExemplars(exemplars: Exemplar[]): void {} + public getPrompt(): ResultAsync { + return this.assureValid().map(() => { + let orderedInstructions: (string | null)[] = []; + // 1. role + if (this.role != null) { + orderedInstructions = [this.role, ...orderedInstructions]; + } + + orderedInstructions = [ + ...orderedInstructions, + this.answerStructure, + this.question, + "\n\n", + this.data, + ]; + + return Prompt(orderedInstructions.join(" ")); + }); + } + + private assureValid(): ResultAsync { + if (this.question == null) { + return errAsync( + new LLMError("Missing question for purchase history prompts", this), + ); + } else if (this.answerStructure == null) { + return errAsync( + new LLMError( + "Missing answerStructure for purchase history prompts", + this, + ), + ); + } else if (this.data == null) { + return errAsync( + new LLMError("Missing data for purchase history prompts", this), + ); + } + + return okAsync(undefined); + } +} diff --git a/packages/ai-scraper/src/implementations/business/utils/PromptBuilder.ts b/packages/ai-scraper/src/implementations/business/utils/PromptBuilder.ts new file mode 100644 index 0000000000..7d81167981 --- /dev/null +++ b/packages/ai-scraper/src/implementations/business/utils/PromptBuilder.ts @@ -0,0 +1,69 @@ +import { + Exemplar, + LLMAnswerStructure, + LLMData, + LLMError, + LLMQuestion, + LLMRole, + Prompt, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync, errAsync, okAsync } from "neverthrow"; + +import { IPromptBuilder } from "@ai-scraper/interfaces/business/utils/IPromptBuilder.js"; + +/** + * @description All the prompt builders should extend this class as + * it provides a generic implementation of the IPromptBuilder interface and should work in most of the cases + */ +export abstract class PromptBuilder implements IPromptBuilder { + protected exemplars: Exemplar[] | null = null; + protected role: LLMRole | null = null; + protected question: LLMQuestion | null = null; + protected answerStructure: LLMAnswerStructure | null = null; + protected data: LLMData | null = null; + + public setExemplars(exemplars: Exemplar[]): void { + this.exemplars = exemplars; + } + public setRole(role: LLMRole): void { + this.role = role; + } + public setQuestion(question: LLMQuestion): void { + this.question = question; + } + public setAnswerStructure(answerStructure: LLMAnswerStructure): void { + this.answerStructure = answerStructure; + } + public setData(data: LLMData): void { + this.data = data; + } + public getPrompt(): ResultAsync { + if (this.question == null) { + return errAsync(new LLMError("Missing question for prompts", this)); + } + + let orderedInstructions: (string | null)[] = []; + // 1. role + if (this.role != null) { + orderedInstructions = [this.role, ...orderedInstructions]; + } + + // 2. exemplars + if (this.exemplars != null) { + orderedInstructions = [...this.exemplars, "\n\n"]; + } + + // 3. question + // 4. answer structure + // 5. data + orderedInstructions = [ + ...orderedInstructions, + this.question, + this.answerStructure, + "\n\n", + this.data, + ]; + + return okAsync(Prompt(orderedInstructions.join(" "))); + } +} diff --git a/packages/ai-scraper/src/implementations/business/utils/PromptBuilderFactory.ts b/packages/ai-scraper/src/implementations/business/utils/PromptBuilderFactory.ts new file mode 100644 index 0000000000..739c94e492 --- /dev/null +++ b/packages/ai-scraper/src/implementations/business/utils/PromptBuilderFactory.ts @@ -0,0 +1,19 @@ +import { injectable } from "inversify"; + +import { ProductMetaPromptBuilder } from "@ai-scraper/implementations/business/utils/ProductMetaPromptBuilder.js"; +import { PurchaseHistoryPromptBuilder } from "@ai-scraper/implementations/business/utils/PurchaseHistoryPromptBuilder.js"; +import { + IProductMetaPromptBuilder, + IPromptBuilderFactory, + IPurchaseHistoryPromptBuilder, +} from "@ai-scraper/interfaces/index.js"; + +@injectable() +export class PromptBuilderFactory implements IPromptBuilderFactory { + public productMeta(): IProductMetaPromptBuilder { + return new ProductMetaPromptBuilder(); + } + public purchaseHistory(): IPurchaseHistoryPromptBuilder { + return new PurchaseHistoryPromptBuilder(); + } +} diff --git a/packages/ai-scraper/src/implementations/business/utils/PurchaseHistoryPromptBuilder.ts b/packages/ai-scraper/src/implementations/business/utils/PurchaseHistoryPromptBuilder.ts new file mode 100644 index 0000000000..8aff2f49c6 --- /dev/null +++ b/packages/ai-scraper/src/implementations/business/utils/PurchaseHistoryPromptBuilder.ts @@ -0,0 +1,59 @@ +import { Exemplar, LLMError, Prompt } from "@snickerdoodlelabs/objects"; +import { ResultAsync, errAsync, okAsync } from "neverthrow"; + +import { PromptBuilder } from "@ai-scraper/implementations/business/utils/PromptBuilder.js"; +import { IPurchaseHistoryPromptBuilder } from "@ai-scraper/interfaces/index.js"; + +/** + * @description this class is responsible for building prompts for purchase history + */ +export class PurchaseHistoryPromptBuilder + extends PromptBuilder + implements IPurchaseHistoryPromptBuilder +{ + /** + * + * @description ignore exemplars for purchase history as this breaks the token limit easily + */ + public setExemplars(exemplars: Exemplar[]): void {} + public getPrompt(): ResultAsync { + const error = this.validateBeforeBuild(); + if (error != null) { + return errAsync(error); + } + + let orderedInstructions: (string | null)[] = []; + // 1. role + if (this.role != null) { + orderedInstructions = [this.role, ...orderedInstructions]; + } + + orderedInstructions = [ + ...orderedInstructions, + this.answerStructure, + this.question, + "\n\n", + this.data, + ]; + + return okAsync(Prompt(orderedInstructions.join(" "))); + } + + private validateBeforeBuild(): LLMError | null { + if (this.question == null) { + return new LLMError( + "Missing question for purchase history prompts", + this, + ); + } else if (this.answerStructure == null) { + return new LLMError( + "Missing answerStructure for purchase history prompts", + this, + ); + } else if (this.data == null) { + return new LLMError("Missing data for purchase history prompts", this); + } + + return null; + } +} diff --git a/packages/ai-scraper/src/implementations/business/utils/index.ts b/packages/ai-scraper/src/implementations/business/utils/index.ts index d23da1bd13..72339de74e 100644 --- a/packages/ai-scraper/src/implementations/business/utils/index.ts +++ b/packages/ai-scraper/src/implementations/business/utils/index.ts @@ -1 +1,6 @@ export * from "@ai-scraper/implementations/business/utils/KeywordUtils.js"; +export * from "@ai-scraper/implementations/business/utils/LLMProductMetaUtilsChatGPT.js"; +export * from "@ai-scraper/implementations/business/utils/LLMPurchaseHistoryUtilsChatGPT.js"; +export * from "@ai-scraper/implementations/business/utils/OpenAIUtils.js"; +export * from "@ai-scraper/implementations/business/utils/PromptBuilderFactory.js"; +export * from "@ai-scraper/implementations/business/utils/PurchaseHistoryPromptBuilder.js"; diff --git a/packages/ai-scraper/src/implementations/data/ChatGPTRepository.ts b/packages/ai-scraper/src/implementations/data/ChatGPTRepository.ts new file mode 100644 index 0000000000..738d7d279a --- /dev/null +++ b/packages/ai-scraper/src/implementations/data/ChatGPTRepository.ts @@ -0,0 +1,101 @@ +import { ILogUtilsType, ILogUtils } from "@snickerdoodlelabs/common-utils"; +import { LLMError, LLMResponse, Prompt } from "@snickerdoodlelabs/objects"; +import { inject, injectable } from "inversify"; +import { + getEncoding, + encodingForModel, + TiktokenModel, + Tiktoken, +} from "js-tiktoken"; +import { ResultAsync, okAsync, errAsync } from "neverthrow"; +import OpenAI from "openai"; +import { + ChatCompletion, + ChatCompletionCreateParamsNonStreaming, + CompletionCreateParamsNonStreaming, +} from "openai/resources/chat"; + +import { + ILLMRepository, + IOpenAIUtils, + IOpenAIUtilsType, + IScraperConfigProvider, + IScraperConfigProviderType, +} from "@ai-scraper/interfaces/index.js"; + +@injectable() +export class ChatGPTRepository implements ILLMRepository { + private chatModel: TiktokenModel = "gpt-3.5-turbo"; // back to 4k + private temperature: number; + private chatEncoder: Tiktoken; + // private timeout = 5 * 60 * 1000; // 5 minutes + + public constructor( + @inject(IScraperConfigProviderType) + private configProvider: IScraperConfigProvider, + @inject(ILogUtilsType) + private logUtils: ILogUtils, + @inject(IOpenAIUtilsType) + private openAIUtils: IOpenAIUtils, + ) { + this.temperature = 0.1; + this.chatEncoder = encodingForModel(this.chatModel); + } + + public defaultMaxTokens(): number { + return 4096; + } + + public maxTokens(model: TiktokenModel): number { + switch (model) { + case "gpt-3.5-turbo": + return 4096; + case "gpt-3.5-turbo-16k": + return 4096 * 4; + case "gpt-4": + return 8192; + } + return 0; + } + + public getPromptTokens(prompt: Prompt): ResultAsync { + const tokens = this.chatEncoder.encode(prompt); // This might take a while + return okAsync(tokens.length); + } + + public executePrompt(prompt: Prompt): ResultAsync { + const messages = [ + { role: "system", content: "You are an helpful assistant." }, + { role: "user", content: prompt }, + ]; + + return this.chatOnce(messages); + } + + private getClient(): ResultAsync { + return this.configProvider.getConfig().andThen((config) => { + try { + const clientOptions = { + apiKey: config.scraper.OPENAI_API_KEY, + timeout: config.scraper.timeout, + }; + // this.logUtils.debug("ChatGPTProvider", "constructor", clientOptions); + return okAsync(new OpenAI(clientOptions)); + } catch (e) { + return errAsync(new LLMError((e as Error).message, e)); + } + }); + } + + private chatOnce(messages): ResultAsync { + return this.getClient().andThen((openai) => { + const params: ChatCompletionCreateParamsNonStreaming = { + model: this.chatModel, + messages: messages, + temperature: this.temperature, + }; + + return this.openAIUtils.getLLMResponseNonStreaming(openai, params); + }); + } +} diff --git a/packages/ai-scraper/src/implementations/data/KeywordRepository.ts b/packages/ai-scraper/src/implementations/data/KeywordRepository.ts index 48d15552c9..67a85c55ad 100644 --- a/packages/ai-scraper/src/implementations/data/KeywordRepository.ts +++ b/packages/ai-scraper/src/implementations/data/KeywordRepository.ts @@ -1,11 +1,9 @@ -import { IpfsCID, JSONString, ELanguageCode } from "@snickerdoodlelabs/objects"; +import { IpfsCID, JSONString, ELanguageCode, ETask, Keyword } from "@snickerdoodlelabs/objects"; import { injectable } from "inversify"; import { ResultAsync, okAsync } from "neverthrow"; import { DefaultKeywords } from "@ai-scraper/data/index.js"; import { - Keyword, - ETask, IKeywordRepository, Keywords, TaskKeywords, diff --git a/packages/ai-scraper/src/implementations/data/index.ts b/packages/ai-scraper/src/implementations/data/index.ts index 01280fcbca..01827a046a 100644 --- a/packages/ai-scraper/src/implementations/data/index.ts +++ b/packages/ai-scraper/src/implementations/data/index.ts @@ -1 +1,2 @@ export * from "@ai-scraper/implementations/data/KeywordRepository.js"; +export * from "@ai-scraper/implementations/data/ChatGPTRepository.js"; diff --git a/packages/ai-scraper/src/implementations/utils/AmazonNavigationUtils.ts b/packages/ai-scraper/src/implementations/utils/AmazonNavigationUtils.ts new file mode 100644 index 0000000000..5b462a2b03 --- /dev/null +++ b/packages/ai-scraper/src/implementations/utils/AmazonNavigationUtils.ts @@ -0,0 +1,65 @@ +import { ITimeUtils, ITimeUtilsType } from "@snickerdoodlelabs/common-utils"; +import { + ELanguageCode, + HTMLString, + PageNumber, + URLString, + Year, +} from "@snickerdoodlelabs/objects"; +import { inject, injectable } from "inversify"; + +import { + IAmazonNavigationUtils, + IHTMLPreProcessor, + IHTMLPreProcessorOptions, + IHTMLPreProcessorType, +} from "@ai-scraper/interfaces/index.js"; + +@injectable() +export class AmazonNavigationUtils implements IAmazonNavigationUtils { + constructor( + @inject(ITimeUtilsType) + protected readonly timeUtils: ITimeUtils, + @inject(IHTMLPreProcessorType) + protected readonly htmlPreProcessor: IHTMLPreProcessor, + ) {} + public getOrderHistoryPage(lang: ELanguageCode, page: PageNumber): URLString { + return URLString("https://www.amazon.com/your-orders/orders?startIndex=0"); + } + + public getYears(html: HTMLString): Year[] { + const curYear = this.timeUtils.getCurYear(); + const years = [curYear]; + for (let i = 1; i < 5; i++) { + years.push(Year(curYear - i)); + } + return years; + } + public getPageCount(html: HTMLString, year: Year): number { + // TODO, parse the page and get number of pages. + return 5; + } + + public getOrderHistoryPageByYear( + lang: ELanguageCode, + year: Year, + page: PageNumber, + ): URLString { + // TODO: this URL structure may break very easily. We should find a better way to do this. + const startIndex = (page - 1) * 10; + return URLString( + `https://www.amazon.com/gp/your-account/order-history?orderFilter=year-${year}&startIndex=${startIndex}`, + ); + } + + public getPurchaseHistoryPagePreprocessingOptions(): IHTMLPreProcessorOptions { + const options = { + baseElements: { selectors: [".your-orders-content-container"] }, + selectors: [ + { selector: "a", options: { ignoreHref: true } }, + { selector: "img", format: "skip" }, + ], + }; + return options as IHTMLPreProcessorOptions; + } +} diff --git a/packages/ai-scraper/src/implementations/utils/HTMLPreProcessor.ts b/packages/ai-scraper/src/implementations/utils/HTMLPreProcessor.ts new file mode 100644 index 0000000000..c684c44e74 --- /dev/null +++ b/packages/ai-scraper/src/implementations/utils/HTMLPreProcessor.ts @@ -0,0 +1,79 @@ +import { + ELanguageCode, + HTMLString, + ScraperError, +} from "@snickerdoodlelabs/objects"; +import { compile, convert } from "html-to-text"; +import { injectable } from "inversify"; +import { ResultAsync, okAsync } from "neverthrow"; + +import { IHTMLPreProcessor } from "@ai-scraper/interfaces/index.js"; + +type Converter = (html: string) => string; + +@injectable() +export class HTMLPreProcessor implements IHTMLPreProcessor { + private converter: Converter; + private converterWithImages: Converter; + private converterWithLinks: Converter; + private headConverter: Converter; + + public constructor() { + const options = { + wordwrap: false, + selectors: [ + { selector: "a", options: { ignoreHref: true } }, + { selector: "img", format: "skip" }, + ], + }; + this.converter = compile(options); + + const optionsImages = { + wordwrap: false, + baseElements: { selectors: ["body"] }, + selectors: [{ selector: "a", options: { ignoreHref: true } }], + }; + + this.converterWithImages = compile(optionsImages); + + const optionsLinks = { + wordwrap: false, + baseElements: { selectors: ["body"] }, + selectors: [{ selector: "img", format: "skip" }], + }; + this.converterWithLinks = compile(optionsLinks); + + const headOptions = { + wordwrap: false, + baseElements: { selectors: ["head"] }, + }; + this.headConverter = compile(headOptions); + } + + public getLanguage( + html: HTMLString, + ): ResultAsync { + return okAsync(ELanguageCode.English); // TODO parse html tag for language. if not found, use third party library (nlp.js) to detect language such as google translate. + } + + public htmlToText( + html: HTMLString, + options: unknown | null, + ): ResultAsync { + if (options == null) { + return okAsync(this.converter(html)); + } else { + return okAsync(convert(html, options)); + } + } + public htmlToTextWithImages( + html: HTMLString, + ): ResultAsync { + return okAsync(this.converterWithImages(html)); + } + public htmlToTextWithLinks( + html: HTMLString, + ): ResultAsync { + return okAsync(this.converterWithLinks(html)); + } +} diff --git a/packages/ai-scraper/src/implementations/utils/LLMPurchaseValidator.ts b/packages/ai-scraper/src/implementations/utils/LLMPurchaseValidator.ts new file mode 100644 index 0000000000..98438ff970 --- /dev/null +++ b/packages/ai-scraper/src/implementations/utils/LLMPurchaseValidator.ts @@ -0,0 +1,53 @@ +import { LLMResponse, PurchasedProduct } from "@snickerdoodlelabs/objects"; +import { ResultAsync, okAsync } from "neverthrow"; + +import { ILLMPurchaseValidator } from "@ai-scraper/interfaces/utils/ILLMPurchaseValidator.js"; + +import { injectable } from "inversify"; +@injectable() +export class LLMPurchaseValidator implements ILLMPurchaseValidator { + public trimHalucinatedPurchases( + promptText: string, + purchases: PurchasedProduct[], + ): ResultAsync { + const validPurchases = purchases.reduce((acc, purchase) => { + // we cannot add price here as multiple order pricing can be hacked a bit + if (promptText.includes(purchase.name)) { + return [...acc, purchase]; + } + return acc; + }, [] as PurchasedProduct[]); + + return okAsync(validPurchases); + } + + + fixMalformedJSONArrayResponse( + llmResponse: LLMResponse + ): ResultAsync { + + // There can be two issues + // 1. The reponse has a valid JSON, but with extra text or characters around the array + // 2. LLM can forget to add the array brackets around the response + + // First, we try to find the maximal array in the response + const arrayExpression = /\[.*\]/gs; // s to match newlines with . + const objectExpression = /\{.*\}/gs; // s to match newlines with . + const arrayMatch = llmResponse.match(arrayExpression); + if (arrayMatch) { + const array = arrayMatch[0]; + return okAsync(LLMResponse(array)); + } else { + // IF we fail, we try to wrap the response in array brackets + // we match indibidual objects in the response and wrap them in array brackets + const objectMatch = llmResponse.match(objectExpression); + if (objectMatch) { + const object = objectMatch[0]; + return okAsync(LLMResponse(`[${object}]`)); + } + } + + return okAsync(LLMResponse("[]")); + } + +} diff --git a/packages/ai-scraper/src/implementations/utils/ProductUtils.ts b/packages/ai-scraper/src/implementations/utils/ProductUtils.ts new file mode 100644 index 0000000000..1282ac32c0 --- /dev/null +++ b/packages/ai-scraper/src/implementations/utils/ProductUtils.ts @@ -0,0 +1,5 @@ +export class ProductUtils { + public hash(name: string) { + return name; // substring? how do we identify unique products? + } +} diff --git a/packages/ai-scraper/src/implementations/utils/URLUtils.ts b/packages/ai-scraper/src/implementations/utils/URLUtils.ts index 88245bbc34..b839e4526e 100644 --- a/packages/ai-scraper/src/implementations/utils/URLUtils.ts +++ b/packages/ai-scraper/src/implementations/utils/URLUtils.ts @@ -4,6 +4,10 @@ import { HostName, HexString, ELanguageCode, + EKnownDomains, + ETask, + Keyword, + InvalidURLError, } from "@snickerdoodlelabs/objects"; import { inject, injectable } from "inversify"; import { Result, ResultAsync, err, errAsync, ok, okAsync } from "neverthrow"; @@ -13,9 +17,6 @@ import { IKeywordUtils, IKeywordUtilsType, IURLUtils, - Keyword, - EKnownDomains, - ETask, } from "@ai-scraper/interfaces/index.js"; @injectable() @@ -33,15 +34,15 @@ export class URLUtils implements IURLUtils { return false; } } - public getHostname(url: URLString): ResultAsync { + public getHostname(url: URLString): ResultAsync { try { return okAsync(HostName(new URL(url).hostname)); } catch (error) { - return errAsync(new TypeError((error as Error).message)); + return errAsync(new InvalidURLError((error as Error).message)); } } - public getDomain(url: URLString): ResultAsync { + public getDomain(url: URLString): ResultAsync { return this.getHostname(url).map((hostname) => { if (hostname.includes(EKnownDomains.Amazon)) { return DomainName(EKnownDomains.Amazon); @@ -54,7 +55,7 @@ export class URLUtils implements IURLUtils { public getKeywords( url: URLString, language: ELanguageCode, - ): ResultAsync, TypeError> { + ): ResultAsync, InvalidURLError> { // keywords are in the path or in search params const uniqueKeywords = new Set(); @@ -75,14 +76,14 @@ export class URLUtils implements IURLUtils { public getHash( url: URLString, language: ELanguageCode, - ): ResultAsync { + ): ResultAsync { throw new Error("Method not implemented."); } public getTask( url: URLString, language: ELanguageCode, - ): ResultAsync { + ): ResultAsync { // 1. get domain // 2. get urlKeywords // 3. get task diff --git a/packages/ai-scraper/src/implementations/utils/WebpageClassifier.ts b/packages/ai-scraper/src/implementations/utils/WebpageClassifier.ts index 740b34a68f..ad7b17a1b5 100644 --- a/packages/ai-scraper/src/implementations/utils/WebpageClassifier.ts +++ b/packages/ai-scraper/src/implementations/utils/WebpageClassifier.ts @@ -1,9 +1,13 @@ -import { ELanguageCode, URLString } from "@snickerdoodlelabs/objects"; +import { + DomainTask, + ELanguageCode, + InvalidURLError, + URLString, +} from "@snickerdoodlelabs/objects"; import { inject, injectable } from "inversify"; import { ResultAsync, okAsync } from "neverthrow"; import { - DomainTask, IKeywordRepository, IKeywordRepositoryType, IURLUtils, @@ -22,7 +26,7 @@ export class WebpageClassifier implements IWebpageClassifier { public classify( url: URLString, language: ELanguageCode, - ): ResultAsync { + ): ResultAsync { // Simplest version return this.urlUtils.getDomain(url).andThen((domain) => { return this.urlUtils.getTask(url, language).andThen((task) => { diff --git a/packages/ai-scraper/src/implementations/utils/index.ts b/packages/ai-scraper/src/implementations/utils/index.ts index 1f4d6f789e..99614ae9c7 100644 --- a/packages/ai-scraper/src/implementations/utils/index.ts +++ b/packages/ai-scraper/src/implementations/utils/index.ts @@ -1,2 +1,5 @@ +export * from "@ai-scraper/implementations/utils/AmazonNavigationUtils.js"; +export * from "@ai-scraper/implementations/utils/HTMLPreProcessor.js"; +export * from "@ai-scraper/implementations/utils/LLMPurchaseValidator.js"; export * from "@ai-scraper/implementations/utils/WebpageClassifier.js"; export * from "@ai-scraper/implementations/utils/URLUtils.js"; diff --git a/packages/ai-scraper/src/index.ts b/packages/ai-scraper/src/index.ts new file mode 100644 index 0000000000..a2425b7497 --- /dev/null +++ b/packages/ai-scraper/src/index.ts @@ -0,0 +1,3 @@ +export * from "@ai-scraper/scraper.module.js"; +export * from "@ai-scraper/interfaces/index.js"; +export * from "@ai-scraper/implementations/index.js"; diff --git a/packages/ai-scraper/src/interfaces/IProductMetaBlock.ts b/packages/ai-scraper/src/interfaces/IProductMetaBlock.ts new file mode 100644 index 0000000000..597d8946e5 --- /dev/null +++ b/packages/ai-scraper/src/interfaces/IProductMetaBlock.ts @@ -0,0 +1,6 @@ +export interface IProductMetaBlock { + product_id: number; + sub_category: string; + category: string; + keywords: string[]; +} diff --git a/packages/ai-scraper/src/interfaces/IPurchaseBlock.ts b/packages/ai-scraper/src/interfaces/IPurchaseBlock.ts new file mode 100644 index 0000000000..c2b290b8c6 --- /dev/null +++ b/packages/ai-scraper/src/interfaces/IPurchaseBlock.ts @@ -0,0 +1,8 @@ +export interface IPurchaseBlock { + name: string; + brand: string; + price: number; + classification: string; + keywords: string[]; + date: string; +} diff --git a/packages/ai-scraper/src/interfaces/IScraperConfig.ts b/packages/ai-scraper/src/interfaces/IScraperConfig.ts new file mode 100644 index 0000000000..1d62b4dd63 --- /dev/null +++ b/packages/ai-scraper/src/interfaces/IScraperConfig.ts @@ -0,0 +1,6 @@ +export interface IScraperConfig { + scraper: { + OPENAI_API_KEY: string; + timeout: number; + }; +} diff --git a/packages/ai-scraper/src/interfaces/IScraperConfigProvider.ts b/packages/ai-scraper/src/interfaces/IScraperConfigProvider.ts new file mode 100644 index 0000000000..96b88d2c2a --- /dev/null +++ b/packages/ai-scraper/src/interfaces/IScraperConfigProvider.ts @@ -0,0 +1,8 @@ +import { ResultAsync } from "neverthrow"; + +import { IScraperConfig } from "@ai-scraper/interfaces/IScraperConfig.js"; + +export interface IScraperConfigProvider { + getConfig(): ResultAsync; +} +export const IScraperConfigProviderType = Symbol.for("IScraperConfigProvider"); diff --git a/packages/ai-scraper/src/interfaces/TaskKeywords.ts b/packages/ai-scraper/src/interfaces/TaskKeywords.ts index bf91b0fc3e..69ec880574 100644 --- a/packages/ai-scraper/src/interfaces/TaskKeywords.ts +++ b/packages/ai-scraper/src/interfaces/TaskKeywords.ts @@ -1,4 +1,4 @@ -import { Keyword } from "@ai-scraper/interfaces/primitives/Keyword.js"; +import { Keyword } from "@snickerdoodlelabs/objects"; export interface TaskKeywords { [task: string]: Keyword[]; diff --git a/packages/ai-scraper/src/interfaces/business/ILLMResponseProcessor.ts b/packages/ai-scraper/src/interfaces/business/ILLMResponseProcessor.ts new file mode 100644 index 0000000000..c21b184225 --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/ILLMResponseProcessor.ts @@ -0,0 +1,3 @@ +export interface ILLMResponseProcessor {} + +export const ILLMResponseProcessorType = Symbol.for("ILLMResponseProcessor"); diff --git a/packages/ai-scraper/src/interfaces/business/ILLMScraper.ts b/packages/ai-scraper/src/interfaces/business/ILLMScraper.ts new file mode 100644 index 0000000000..5169f2a01b --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/ILLMScraper.ts @@ -0,0 +1,7 @@ +import { ScraperJob } from "@snickerdoodlelabs/objects"; + +export interface ILLMScraper { + scrape(jobs: ScraperJob[]); +} + +export const ILLMScraperType = Symbol.for("ILLMScraper"); diff --git a/packages/ai-scraper/src/interfaces/business/IPromptDirector.ts b/packages/ai-scraper/src/interfaces/business/IPromptDirector.ts new file mode 100644 index 0000000000..bcc79e38f1 --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/IPromptDirector.ts @@ -0,0 +1,9 @@ +import { LLMData, LLMError, Prompt } from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface IPromptDirector { + makePurchaseHistoryPrompt(data: LLMData): ResultAsync; + makeProductMetaPrompt(data: LLMData): ResultAsync; +} + +export const IPromptDirectorType = Symbol.for("IPromptDirector"); diff --git a/packages/ai-scraper/src/interfaces/business/IScraperJobService.ts b/packages/ai-scraper/src/interfaces/business/IScraperJobService.ts new file mode 100644 index 0000000000..9a74af465b --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/IScraperJobService.ts @@ -0,0 +1,14 @@ +import { + URLString, + HTMLString, + ScraperError, + ScraperJob, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface IScraperJobService { + add(job: ScraperJob): ResultAsync; + poll(): ResultAsync; +} + +export const IScraperJobServiceType = Symbol.for("IScraperJobService"); diff --git a/packages/ai-scraper/src/interfaces/business/IScraperService.ts b/packages/ai-scraper/src/interfaces/business/IScraperService.ts new file mode 100644 index 0000000000..5e1c6d4b8c --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/IScraperService.ts @@ -0,0 +1,40 @@ +import { + URLString, + HTMLString, + ScraperError, + ELanguageCode, + DomainTask, + PersistenceError, + LLMError, + InvalidURLError, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface IScraperService { + /** + * This method to extract information from a website + * @param url + * @param html + * @param suggestedDomainTask + */ + scrape( + url: URLString, + html: HTMLString, + suggestedDomainTask: DomainTask, + ): ResultAsync; + + /** + * + * @param url url of a website to classify + * @param language language. Just pass en for now + */ + + classifyURL( + url: URLString, + language: ELanguageCode, + ): ResultAsync; + + poll(): ResultAsync; +} + +export const IScraperServiceType = Symbol.for("IScraperService"); diff --git a/packages/ai-scraper/src/interfaces/business/index.ts b/packages/ai-scraper/src/interfaces/business/index.ts index e69de29bb2..f8e95c88bb 100644 --- a/packages/ai-scraper/src/interfaces/business/index.ts +++ b/packages/ai-scraper/src/interfaces/business/index.ts @@ -0,0 +1,6 @@ +export * from "@ai-scraper/interfaces/business/ILLMResponseProcessor.js"; +export * from "@ai-scraper/interfaces/business/ILLMScraper.js"; +export * from "@ai-scraper/interfaces/business/IPromptDirector.js"; +export * from "@ai-scraper/interfaces/business/IScraperJobService.js"; +export * from "@ai-scraper/interfaces/business/IScraperService.js"; +export * from "@ai-scraper/interfaces/business/utils/index.js"; diff --git a/packages/ai-scraper/src/interfaces/business/utils/IDomainTaskUtils.ts b/packages/ai-scraper/src/interfaces/business/utils/IDomainTaskUtils.ts new file mode 100644 index 0000000000..1052108c6e --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/utils/IDomainTaskUtils.ts @@ -0,0 +1,2 @@ +export interface IDomainTaskUtils {} +export const IDomainTaskUtilsType = Symbol.for("IDomainTaskUtils"); diff --git a/packages/ai-scraper/src/interfaces/business/utils/ILLMProductMetaUtils.ts b/packages/ai-scraper/src/interfaces/business/utils/ILLMProductMetaUtils.ts new file mode 100644 index 0000000000..b81cdf6f46 --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/utils/ILLMProductMetaUtils.ts @@ -0,0 +1,25 @@ +import { + LLMError, + DomainName, + ELanguageCode, + LLMAnswerStructure, + LLMQuestion, + LLMResponse, + LLMRole, + ProductMeta, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface ILLMProductMetaUtils { + getRole(): LLMRole; + getQuestion(): LLMQuestion; + getAnswerStructure(): LLMAnswerStructure; + + parseMeta( + domain: DomainName, + language: ELanguageCode, + llmResponse: LLMResponse, + ): ResultAsync; +} + +export const ILLMProductMetaUtilsType = Symbol.for("ILLMProductMetaUtils"); diff --git a/packages/ai-scraper/src/interfaces/business/utils/ILLMPurchaseHistoryUtils.ts b/packages/ai-scraper/src/interfaces/business/utils/ILLMPurchaseHistoryUtils.ts new file mode 100644 index 0000000000..6e66db78b2 --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/utils/ILLMPurchaseHistoryUtils.ts @@ -0,0 +1,27 @@ +import { + LLMError, + DomainName, + ELanguageCode, + LLMAnswerStructure, + LLMQuestion, + LLMResponse, + LLMRole, + PurchasedProduct, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface ILLMPurchaseHistoryUtils { + getRole(): LLMRole; + getQuestion(): LLMQuestion; + getAnswerStructure(): LLMAnswerStructure; + + parsePurchases( + domain: DomainName, + language: ELanguageCode, + llmResponse: LLMResponse, + ): ResultAsync; +} + +export const ILLMPurchaseHistoryUtilsType = Symbol.for( + "ILLMPurchaseHistoryUtils", +); diff --git a/packages/ai-scraper/src/interfaces/business/utils/ILLMPurchaseHistoryUtilsChatGPT.ts b/packages/ai-scraper/src/interfaces/business/utils/ILLMPurchaseHistoryUtilsChatGPT.ts new file mode 100644 index 0000000000..4a467d545b --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/utils/ILLMPurchaseHistoryUtilsChatGPT.ts @@ -0,0 +1,11 @@ +import { ILLMPurchaseHistoryUtils } from "@ai-scraper/interfaces/business/utils/ILLMPurchaseHistoryUtils.js"; + +/** + * Currently we don't need it. But in future if we want a fallback LLM provider for this task, we will need it + */ +export interface ILLMPurchaseHistoryUtilsChatGPT + extends ILLMPurchaseHistoryUtils {} + +export const ILLMPurchaseHistoryUtilsChatGPTType = Symbol.for( + "ILLMPurchaseHistoryUtilsChatGPT", +); diff --git a/packages/ai-scraper/src/interfaces/business/utils/IOpenAIUtils.ts b/packages/ai-scraper/src/interfaces/business/utils/IOpenAIUtils.ts new file mode 100644 index 0000000000..7547c05f01 --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/utils/IOpenAIUtils.ts @@ -0,0 +1,12 @@ +import { LLMError, LLMResponse } from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; +import OpenAI from "openai"; +import { ChatCompletionCreateParamsNonStreaming } from "openai/resources/chat"; + +export interface IOpenAIUtils { + getLLMResponseNonStreaming( + client: OpenAI, + params: ChatCompletionCreateParamsNonStreaming, + ): ResultAsync; +} +export const IOpenAIUtilsType = Symbol.for("IOpenAIUtils"); diff --git a/packages/ai-scraper/src/interfaces/business/utils/IProductMetaPromptBuilder.ts b/packages/ai-scraper/src/interfaces/business/utils/IProductMetaPromptBuilder.ts new file mode 100644 index 0000000000..a1582e2946 --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/utils/IProductMetaPromptBuilder.ts @@ -0,0 +1,6 @@ +import { IPromptBuilder } from "@ai-scraper/interfaces/business/utils/IPromptBuilder.js"; + +export interface IProductMetaPromptBuilder extends IPromptBuilder {} +export const IProductMetaPromptBuilderType = Symbol.for( + "IProductMetaPromptBuilder", +); diff --git a/packages/ai-scraper/src/interfaces/business/utils/IPromptBuilder.ts b/packages/ai-scraper/src/interfaces/business/utils/IPromptBuilder.ts new file mode 100644 index 0000000000..7d93cb4b8b --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/utils/IPromptBuilder.ts @@ -0,0 +1,21 @@ +import { + Exemplar, + LLMAnswerStructure, + LLMData, + LLMError, + LLMQuestion, + LLMRole, + Prompt, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface IPromptBuilder { + setExemplars(exemplars: Exemplar[]): void; + setRole(role: LLMRole): void; + setQuestion(question: LLMQuestion): void; + setAnswerStructure(structure: LLMAnswerStructure): void; + setData(data: LLMData): void; + getPrompt(): ResultAsync; +} + +export const IPromptBuilderType = Symbol.for("IPromptBuilder"); diff --git a/packages/ai-scraper/src/interfaces/business/utils/IPromptBuilderFactory.ts b/packages/ai-scraper/src/interfaces/business/utils/IPromptBuilderFactory.ts new file mode 100644 index 0000000000..5de92f358b --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/utils/IPromptBuilderFactory.ts @@ -0,0 +1,9 @@ +import { IProductMetaPromptBuilder } from "@ai-scraper/interfaces/business/utils/IProductMetaPromptBuilder.js"; +import { IPurchaseHistoryPromptBuilder } from "@ai-scraper/interfaces/business/utils/IPurchaseHistoryPromptBuilder.js"; + +export interface IPromptBuilderFactory { + productMeta(): IProductMetaPromptBuilder; + purchaseHistory(): IPurchaseHistoryPromptBuilder; +} + +export const IPromptBuilderFactoryType = Symbol.for("IPromptBuilderFactory"); diff --git a/packages/ai-scraper/src/interfaces/business/utils/IPurchaseHistoryPromptBuilder.ts b/packages/ai-scraper/src/interfaces/business/utils/IPurchaseHistoryPromptBuilder.ts new file mode 100644 index 0000000000..e81f88638d --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/utils/IPurchaseHistoryPromptBuilder.ts @@ -0,0 +1,6 @@ +import { IPromptBuilder } from "@ai-scraper/interfaces/business/utils/IPromptBuilder.js"; + +export interface IPurchaseHistoryPromptBuilder extends IPromptBuilder {} +export const IPurchaseHistoryPromptBuilderType = Symbol.for( + "IPurchaseHistoryPromptBuilder", +); diff --git a/packages/ai-scraper/src/interfaces/business/utils/index.ts b/packages/ai-scraper/src/interfaces/business/utils/index.ts new file mode 100644 index 0000000000..d61cbaabae --- /dev/null +++ b/packages/ai-scraper/src/interfaces/business/utils/index.ts @@ -0,0 +1,8 @@ +export * from "@ai-scraper/interfaces/business/utils/ILLMPurchaseHistoryUtils.js"; +export * from "@ai-scraper/interfaces/business/utils/ILLMProductMetaUtils.js"; +export * from "@ai-scraper/interfaces/business/utils/ILLMPurchaseHistoryUtilsChatGPT.js"; +export * from "@ai-scraper/interfaces/business/utils/IOpenAIUtils.js"; +export * from "@ai-scraper/interfaces/business/utils/IPromptBuilder.js"; +export * from "@ai-scraper/interfaces/business/utils/IPromptBuilderFactory.js"; +export * from "@ai-scraper/interfaces/business/utils/IProductMetaPromptBuilder.js"; +export * from "@ai-scraper/interfaces/business/utils/IPurchaseHistoryPromptBuilder.js"; diff --git a/packages/ai-scraper/src/interfaces/data/IKeywordRepository.ts b/packages/ai-scraper/src/interfaces/data/IKeywordRepository.ts index 018b94b377..0ce13edfb5 100644 --- a/packages/ai-scraper/src/interfaces/data/IKeywordRepository.ts +++ b/packages/ai-scraper/src/interfaces/data/IKeywordRepository.ts @@ -1,8 +1,11 @@ -import { IpfsCID, ELanguageCode } from "@snickerdoodlelabs/objects"; +import { + IpfsCID, + ELanguageCode, + ETask, + Keyword, +} from "@snickerdoodlelabs/objects"; import { ResultAsync } from "neverthrow"; -import { ETask } from "@ai-scraper/interfaces/enums/ETask.js"; -import { Keyword } from "@ai-scraper/interfaces/primitives/Keyword.js"; import { TaskKeywords } from "@ai-scraper/interfaces/TaskKeywords.js"; export interface IKeywordRepository { diff --git a/packages/ai-scraper/src/interfaces/data/ILLMRepository.ts b/packages/ai-scraper/src/interfaces/data/ILLMRepository.ts new file mode 100644 index 0000000000..970c67a85b --- /dev/null +++ b/packages/ai-scraper/src/interfaces/data/ILLMRepository.ts @@ -0,0 +1,11 @@ +import { LLMError, LLMResponse, Prompt } from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface ILLMRepository { + defaultMaxTokens(): number; + maxTokens(model: string): number; + getPromptTokens(prompt: Prompt): ResultAsync; + executePrompt(prompt: Prompt): ResultAsync; +} + +export const ILLMRepositoryType = Symbol.for("ILLMRepository"); diff --git a/packages/ai-scraper/src/interfaces/data/index.ts b/packages/ai-scraper/src/interfaces/data/index.ts index 1092f33934..96998389f8 100644 --- a/packages/ai-scraper/src/interfaces/data/index.ts +++ b/packages/ai-scraper/src/interfaces/data/index.ts @@ -1 +1,2 @@ export * from "@ai-scraper/interfaces/data/IKeywordRepository.js"; +export * from "@ai-scraper/interfaces/data/ILLMRepository.js"; diff --git a/packages/ai-scraper/src/interfaces/enums/index.ts b/packages/ai-scraper/src/interfaces/enums/index.ts deleted file mode 100644 index 0bfc28c5ae..0000000000 --- a/packages/ai-scraper/src/interfaces/enums/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "@ai-scraper/interfaces/enums/EKnownDomains.js"; -export * from "@ai-scraper/interfaces/enums/ETask.js"; diff --git a/packages/ai-scraper/src/interfaces/index.ts b/packages/ai-scraper/src/interfaces/index.ts index cfe6f6ac1b..6c2e5c1bd5 100644 --- a/packages/ai-scraper/src/interfaces/index.ts +++ b/packages/ai-scraper/src/interfaces/index.ts @@ -1,8 +1,10 @@ -// export * from "@ai-scraper/interfaces/business/index.js"; +export * from "@ai-scraper/interfaces/business/index.js"; export * from "@ai-scraper/interfaces/data/index.js"; -export * from "@ai-scraper/interfaces/enums/index.js"; -export * from "@ai-scraper/interfaces/objects/index.js"; -export * from "@ai-scraper/interfaces/primitives/index.js"; export * from "@ai-scraper/interfaces/utils/index.js"; -export * from "@ai-scraper/interfaces/TaskKeywords.js"; + +export * from "@ai-scraper/interfaces/IScraperConfig.js"; +export * from "@ai-scraper/interfaces/IScraperConfigProvider.js"; +export * from "@ai-scraper/interfaces/IProductMetaBlock.js"; +export * from "@ai-scraper/interfaces/IPurchaseBlock.js"; export * from "@ai-scraper/interfaces/Keywords.js"; +export * from "@ai-scraper/interfaces/TaskKeywords.js"; diff --git a/packages/ai-scraper/src/interfaces/objects/DomainTask.ts b/packages/ai-scraper/src/interfaces/objects/DomainTask.ts deleted file mode 100644 index cbbf20aeaa..0000000000 --- a/packages/ai-scraper/src/interfaces/objects/DomainTask.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { DomainName } from "@snickerdoodlelabs/objects"; - -import { ETask } from "@ai-scraper/interfaces/enums/ETask.js"; - -export class DomainTask { - constructor(readonly domain: DomainName, readonly taskType: ETask) {} -} diff --git a/packages/ai-scraper/src/interfaces/objects/index.ts b/packages/ai-scraper/src/interfaces/objects/index.ts deleted file mode 100644 index ef419533ae..0000000000 --- a/packages/ai-scraper/src/interfaces/objects/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "@ai-scraper/interfaces/objects/DomainTask.js"; diff --git a/packages/ai-scraper/src/interfaces/primitives/index.ts b/packages/ai-scraper/src/interfaces/primitives/index.ts deleted file mode 100644 index 06126a97ab..0000000000 --- a/packages/ai-scraper/src/interfaces/primitives/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "@ai-scraper/interfaces/primitives/Keyword.js"; diff --git a/packages/ai-scraper/src/interfaces/utils/IAmazonNavigationUtils.ts b/packages/ai-scraper/src/interfaces/utils/IAmazonNavigationUtils.ts new file mode 100644 index 0000000000..0007de3ff2 --- /dev/null +++ b/packages/ai-scraper/src/interfaces/utils/IAmazonNavigationUtils.ts @@ -0,0 +1,22 @@ +import { + ELanguageCode, + HTMLString, + PageNumber, + URLString, + Year, +} from "@snickerdoodlelabs/objects"; + +import { IHTMLPreProcessorOptions } from "@ai-scraper/interfaces/utils/IHTMLPreProcessorOptions.js"; + +export interface IAmazonNavigationUtils { + getOrderHistoryPage(lang: ELanguageCode, page: PageNumber): URLString; + getYears(html: HTMLString): Year[]; + getOrderHistoryPageByYear( + lang: ELanguageCode, + year: Year, + page: PageNumber, + ): URLString; + getPageCount(html: HTMLString, year: Year): number; + getPurchaseHistoryPagePreprocessingOptions(): IHTMLPreProcessorOptions; +} +export const IAmazonNavigationUtilsType = Symbol.for("IAmazonNavigationUtils"); diff --git a/packages/ai-scraper/src/interfaces/utils/IHTMLPreProcessor.ts b/packages/ai-scraper/src/interfaces/utils/IHTMLPreProcessor.ts new file mode 100644 index 0000000000..48b9a3a46f --- /dev/null +++ b/packages/ai-scraper/src/interfaces/utils/IHTMLPreProcessor.ts @@ -0,0 +1,19 @@ +import { + ELanguageCode, + HTMLString, + ScraperError, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface IHTMLPreProcessor { + getLanguage(html: HTMLString): ResultAsync; + htmlToText( + html: HTMLString, + options: unknown | null, + ): ResultAsync; + + htmlToTextWithImages(html: HTMLString): ResultAsync; + htmlToTextWithLinks(html: HTMLString): ResultAsync; +} + +export const IHTMLPreProcessorType = Symbol.for("IHTMLPreProcessor"); diff --git a/packages/ai-scraper/src/interfaces/utils/IHTMLPreProcessorOptions.ts b/packages/ai-scraper/src/interfaces/utils/IHTMLPreProcessorOptions.ts new file mode 100644 index 0000000000..92239176cc --- /dev/null +++ b/packages/ai-scraper/src/interfaces/utils/IHTMLPreProcessorOptions.ts @@ -0,0 +1,7 @@ +export interface IHTMLPreProcessorOptions { + baseElements: { selectors: string[] }; + selectors: [ + { selector: "a"; options?: { ignoreHref: boolean } }, + { selector: "img"; format: string }, + ]; +} diff --git a/packages/ai-scraper/src/interfaces/utils/IKeywordUtils.ts b/packages/ai-scraper/src/interfaces/utils/IKeywordUtils.ts index 0f34726637..6b253be5df 100644 --- a/packages/ai-scraper/src/interfaces/utils/IKeywordUtils.ts +++ b/packages/ai-scraper/src/interfaces/utils/IKeywordUtils.ts @@ -1,10 +1,6 @@ -import { ELanguageCode } from "@snickerdoodlelabs/objects"; +import { ELanguageCode, ETask, Keyword } from "@snickerdoodlelabs/objects"; import { ResultAsync } from "neverthrow"; -import { IKeywordRepository } from "@ai-scraper/interfaces/data/IKeywordRepository.js"; -import { ETask } from "@ai-scraper/interfaces/enums/ETask.js"; -import { Keyword } from "@ai-scraper/interfaces/primitives/Keyword.js"; - export interface IKeywordUtils { getTaskByKeywords( language: ELanguageCode, diff --git a/packages/ai-scraper/src/interfaces/utils/ILLMPurchaseValidator.ts b/packages/ai-scraper/src/interfaces/utils/ILLMPurchaseValidator.ts new file mode 100644 index 0000000000..3b660bcf64 --- /dev/null +++ b/packages/ai-scraper/src/interfaces/utils/ILLMPurchaseValidator.ts @@ -0,0 +1,15 @@ +import { LLMResponse, Prompt, PurchasedProduct } from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface ILLMPurchaseValidator { + trimHalucinatedPurchases( + prompt: Prompt, + purchases: PurchasedProduct[], + ): ResultAsync; + + fixMalformedJSONArrayResponse( + llmResponse: LLMResponse + ): ResultAsync; +} + +export const ILLMPurchaseValidatorType = Symbol.for("ILLMPurchaseValidator"); diff --git a/packages/ai-scraper/src/interfaces/utils/IURLUtils.ts b/packages/ai-scraper/src/interfaces/utils/IURLUtils.ts index 258cde0eda..6c8f82cbd2 100644 --- a/packages/ai-scraper/src/interfaces/utils/IURLUtils.ts +++ b/packages/ai-scraper/src/interfaces/utils/IURLUtils.ts @@ -4,28 +4,27 @@ import { HostName, HexString, ELanguageCode, + ETask, + Keyword, + InvalidURLError, } from "@snickerdoodlelabs/objects"; import { Result, ResultAsync } from "neverthrow"; -import { IKeywordRepository } from "@ai-scraper/interfaces/data/IKeywordRepository.js"; -import { ETask } from "@ai-scraper/interfaces/enums/ETask.js"; -import { Keyword } from "@ai-scraper/interfaces/primitives/Keyword.js"; - export interface IURLUtils { - getHostname(url: URLString): ResultAsync; - getDomain(url: URLString): ResultAsync; + getHostname(url: URLString): ResultAsync; + getDomain(url: URLString): ResultAsync; getKeywords( url: URLString, language: ELanguageCode, - ): ResultAsync, TypeError>; + ): ResultAsync, InvalidURLError>; getHash( url: URLString, language: ELanguageCode, - ): ResultAsync; + ): ResultAsync; getTask( url: URLString, language: ELanguageCode, - ): ResultAsync; + ): ResultAsync; } export const IURLUtilsType = Symbol.for("IURLUtils"); diff --git a/packages/ai-scraper/src/interfaces/utils/IWebpageClassifier.ts b/packages/ai-scraper/src/interfaces/utils/IWebpageClassifier.ts index 538a8bfe94..bf99027c88 100644 --- a/packages/ai-scraper/src/interfaces/utils/IWebpageClassifier.ts +++ b/packages/ai-scraper/src/interfaces/utils/IWebpageClassifier.ts @@ -1,13 +1,16 @@ -import { ELanguageCode, URLString } from "@snickerdoodlelabs/objects"; +import { + DomainTask, + ELanguageCode, + InvalidURLError, + URLString, +} from "@snickerdoodlelabs/objects"; import { ResultAsync } from "neverthrow"; -import { DomainTask } from "@ai-scraper/interfaces/index.js"; - export interface IWebpageClassifier { classify( url: URLString, language: ELanguageCode, - ): ResultAsync; + ): ResultAsync; } export const IWebpageClassifierType = Symbol.for("IWebpageClassifier"); diff --git a/packages/ai-scraper/src/interfaces/utils/index.ts b/packages/ai-scraper/src/interfaces/utils/index.ts index ac8f3029dc..2ea53b6f57 100644 --- a/packages/ai-scraper/src/interfaces/utils/index.ts +++ b/packages/ai-scraper/src/interfaces/utils/index.ts @@ -1,3 +1,7 @@ +export * from "@ai-scraper/interfaces/utils/IAmazonNavigationUtils.js"; +export * from "@ai-scraper/interfaces/utils/IHTMLPreProcessor.js"; +export * from "@ai-scraper/interfaces/utils/IHTMLPreProcessorOptions.js"; export * from "@ai-scraper/interfaces/utils/IKeywordUtils.js"; +export * from "@ai-scraper/interfaces/utils/ILLMPurchaseValidator.js"; export * from "@ai-scraper/interfaces/utils/IWebpageClassifier.js"; export * from "@ai-scraper/interfaces/utils/IURLUtils.js"; diff --git a/packages/ai-scraper/src/scraper.module.ts b/packages/ai-scraper/src/scraper.module.ts new file mode 100644 index 0000000000..b171e5e3b6 --- /dev/null +++ b/packages/ai-scraper/src/scraper.module.ts @@ -0,0 +1,103 @@ +import { ContainerModule, interfaces } from "inversify"; + +import { + OpenAIUtils, + LLMScraperService, + ChatGPTRepository, + HTMLPreProcessor, + PromptDirector, + LLMPurchaseHistoryUtilsChatGPT, + PromptBuilderFactory, + AmazonNavigationUtils, + WebpageClassifier, + URLUtils, + KeywordRepository, + KeywordUtils, + LLMProductMetaUtilsChatGPT, + LLMPurchaseValidator, +} from "@ai-scraper/implementations/index.js"; +import { + IOpenAIUtils, + IOpenAIUtilsType, + IScraperService, + IScraperServiceType, + ILLMRepository, + ILLMRepositoryType, + IHTMLPreProcessor, + IHTMLPreProcessorType, + IPromptDirector, + IPromptDirectorType, + ILLMPurchaseHistoryUtils, + ILLMPurchaseHistoryUtilsType, + IPromptBuilderFactory, + IPromptBuilderFactoryType, + IAmazonNavigationUtils, + IAmazonNavigationUtilsType, + IWebpageClassifier, + IWebpageClassifierType, + IURLUtils, + IURLUtilsType, + IKeywordRepository, + IKeywordRepositoryType, + IKeywordUtils, + IKeywordUtilsType, + ILLMProductMetaUtils, + ILLMProductMetaUtilsType, + ILLMPurchaseValidatorType, + ILLMPurchaseValidator, +} from "@ai-scraper/interfaces/index.js"; + +export const scraperModule = new ContainerModule( + ( + bind: interfaces.Bind, + _unbind: interfaces.Unbind, + _isBound: interfaces.IsBound, + _rebind: interfaces.Rebind, + ) => { + // bind(IScraperConfigProviderType).toService( + // IConfigProviderType, + // ); // this one is to be bound by the core module as we don't have access to the IConfigProviderType here + + bind(IOpenAIUtilsType).to(OpenAIUtils).inSingletonScope(); + bind(IScraperServiceType) + .to(LLMScraperService) + .inSingletonScope(); + bind(ILLMRepositoryType) + .to(ChatGPTRepository) + .inSingletonScope(); + bind(IHTMLPreProcessorType) + .to(HTMLPreProcessor) + .inSingletonScope(); + bind(IPromptDirectorType) + .to(PromptDirector) + .inSingletonScope(); + + bind(IPromptBuilderFactoryType) + .to(PromptBuilderFactory) + .inSingletonScope(); + + bind(ILLMPurchaseHistoryUtilsType) + .to(LLMPurchaseHistoryUtilsChatGPT) + .inSingletonScope(); + bind(ILLMProductMetaUtilsType) + .to(LLMProductMetaUtilsChatGPT) + .inSingletonScope(); + + bind(ILLMPurchaseValidatorType) + .to(LLMPurchaseValidator) + .inSingletonScope(); + + bind(IWebpageClassifierType) + .to(WebpageClassifier) + .inSingletonScope(); + bind(IURLUtilsType).to(URLUtils).inSingletonScope(); + bind(IKeywordRepositoryType) + .to(KeywordRepository) + .inSingletonScope(); + bind(IKeywordUtilsType).to(KeywordUtils).inSingletonScope(); + + bind(IAmazonNavigationUtilsType) + .to(AmazonNavigationUtils) + .inSingletonScope(); + }, +); diff --git a/packages/ai-scraper/test/mocks/MockKeywordRepository.ts b/packages/ai-scraper/test/mocks/MockKeywordRepository.ts index 2cb4cfae76..19c5da07e3 100644 --- a/packages/ai-scraper/test/mocks/MockKeywordRepository.ts +++ b/packages/ai-scraper/test/mocks/MockKeywordRepository.ts @@ -1,14 +1,8 @@ -import { ELanguageCode } from "@snickerdoodlelabs/objects"; +import { ELanguageCode, ETask, Keyword } from "@snickerdoodlelabs/objects"; import * as td from "testdouble"; import { DefaultKeywords } from "@ai-scraper/data"; -import { - ETask, - IKeywordRepository, - IKeywordUtils, - Keyword, - Keywords, -} from "@ai-scraper/interfaces"; +import { IKeywordRepository, Keywords } from "@ai-scraper/interfaces"; export class MockKeywordRepository { public keywords = JSON.parse(DefaultKeywords) as Keywords; diff --git a/packages/ai-scraper/test/mocks/MockOpenAIClient.ts b/packages/ai-scraper/test/mocks/MockOpenAIClient.ts new file mode 100644 index 0000000000..3e8caab6df --- /dev/null +++ b/packages/ai-scraper/test/mocks/MockOpenAIClient.ts @@ -0,0 +1,2 @@ +import OpenAI from "openai"; +export class MockOpenAIClient extends OpenAI {} diff --git a/packages/ai-scraper/test/mocks/MockOpenAIUtils.ts b/packages/ai-scraper/test/mocks/MockOpenAIUtils.ts new file mode 100644 index 0000000000..7b40f828b7 --- /dev/null +++ b/packages/ai-scraper/test/mocks/MockOpenAIUtils.ts @@ -0,0 +1,21 @@ +import { LLMError, LLMResponse } from "@snickerdoodlelabs/objects"; +import { injectable } from "inversify"; +import { ResultAsync, errAsync, okAsync } from "neverthrow"; +import OpenAI from "openai"; +import { + ChatCompletion, + CompletionCreateParamsNonStreaming, +} from "openai/resources/chat"; + +import { IOpenAIUtils } from "@ai-scraper/interfaces"; +import { chatCompletion } from "@ai-scraper-test/mocks/testValues.js"; +// import { chatCompletion } from "@ai-scraper-test/mocks/testValues.js"; + +export class MockOpenAIUtils implements IOpenAIUtils { + public getLLMResponseNonStreaming( + client: OpenAI, + params: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming, + ): ResultAsync { + return okAsync(LLMResponse(chatCompletion.choices[0].message.content!)); + } +} diff --git a/packages/ai-scraper/test/mocks/MockPromptBuilder.ts b/packages/ai-scraper/test/mocks/MockPromptBuilder.ts new file mode 100644 index 0000000000..0ee2d9d776 --- /dev/null +++ b/packages/ai-scraper/test/mocks/MockPromptBuilder.ts @@ -0,0 +1,27 @@ +import { + Exemplar, + LLMAnswerStructure, + LLMData, + LLMError, + LLMQuestion, + LLMRole, + Prompt, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync, okAsync } from "neverthrow"; + +import { IPromptBuilder } from "@ai-scraper/interfaces"; + +export class MockPromptBuilder implements IPromptBuilder { + private promptToReturn: Prompt; + public constructor(promptToReturn: Prompt) { + this.promptToReturn = promptToReturn; + } + setExemplars(exemplars: Exemplar[]): void {} + setRole(role: LLMRole): void {} + setQuestion(question: LLMQuestion): void {} + setAnswerStructure(structure: LLMAnswerStructure): void {} + setData(data: LLMData): void {} + getPrompt(): ResultAsync { + return okAsync(this.promptToReturn); + } +} diff --git a/packages/ai-scraper/test/mocks/MockScraperConfigProvider.ts b/packages/ai-scraper/test/mocks/MockScraperConfigProvider.ts new file mode 100644 index 0000000000..8d14240ec2 --- /dev/null +++ b/packages/ai-scraper/test/mocks/MockScraperConfigProvider.ts @@ -0,0 +1,10 @@ +import { ResultAsync, okAsync } from "neverthrow"; + +import { IScraperConfig, IScraperConfigProvider } from "@ai-scraper/interfaces"; +import { scraperConfig } from "@ai-scraper-test/mocks/testValues.js"; + +export class MockScraperConfigProvider implements IScraperConfigProvider { + public getConfig(): ResultAsync { + return okAsync(scraperConfig); + } +} diff --git a/packages/ai-scraper/test/mocks/MockTimeUtils.ts b/packages/ai-scraper/test/mocks/MockTimeUtils.ts new file mode 100644 index 0000000000..3d1d635e57 --- /dev/null +++ b/packages/ai-scraper/test/mocks/MockTimeUtils.ts @@ -0,0 +1,43 @@ +import { ITimeUtils } from "@snickerdoodlelabs/common-utils"; +import { + UnixTimestamp, + MillisecondTimestamp, + Year, + Month, + ISO8601DateString, +} from "@snickerdoodlelabs/objects"; + +export class MockTimeUtils implements ITimeUtils { + public getUnixNow(): UnixTimestamp { + throw new Error("Method not implemented."); + } + public getMillisecondNow(): MillisecondTimestamp { + throw new Error("Method not implemented."); + } + public getISO8601TimeString(time: MillisecondTimestamp): string { + throw new Error("Method not implemented."); + } + public parseToDate(dateStr: string): Date | null { + throw new Error("Method not implemented."); + } + public parseToSDTimestamp(dateStr: string): UnixTimestamp | null { + throw new Error("Method not implemented."); + } + public getCurYear(): Year { + throw new Error("Method not implemented."); + } + public getCurMonth(): Month { + throw new Error("Method not implemented."); + } + public convertTimestampToISOString( + unixTimestamp: UnixTimestamp, + ): ISO8601DateString { + throw new Error("Method not implemented."); + } + public getUnixTodayStart(): UnixTimestamp { + throw new Error("Method not implemented."); + } + public getUnixTodayEnd(): UnixTimestamp { + throw new Error("Method not implemented."); + } +} diff --git a/packages/ai-scraper/test/mocks/index.ts b/packages/ai-scraper/test/mocks/index.ts index 7ff030f20b..7ede2dd4ee 100644 --- a/packages/ai-scraper/test/mocks/index.ts +++ b/packages/ai-scraper/test/mocks/index.ts @@ -1,2 +1,6 @@ export * from "@ai-scraper-test/mocks/MockKeywordRepository.js"; +export * from "@ai-scraper-test/mocks/MockOpenAIUtils.js"; +export * from "@ai-scraper-test/mocks/MockTimeUtils.js"; +export * from "@ai-scraper-test/mocks/testProductMetaData.js"; +export * from "@ai-scraper-test/mocks/testPurchaseHistoryData.js"; export * from "@ai-scraper-test/mocks/testValues.js"; diff --git a/packages/ai-scraper/test/mocks/testHTMLPreprocessorData.ts b/packages/ai-scraper/test/mocks/testHTMLPreprocessorData.ts new file mode 100644 index 0000000000..add9435d73 --- /dev/null +++ b/packages/ai-scraper/test/mocks/testHTMLPreprocessorData.ts @@ -0,0 +1,177 @@ +import { HTMLString } from "@snickerdoodlelabs/objects"; + +export const html1 = HTMLString("
Hello world
"); +export const text1 = "Hello world"; + +export const fullHtml = HTMLString(` + + + This is a title + + + + + + + + + + +

This is body

+ +

this is a paragraph

+ google + Product 1 + Product 2 +
+ + +
+
+
+ + + HE401 Filter Replacement for Shark Air Purifier 4 - Compatible with Shark Air Purifier 4 HE401 HE402 HE402AMZ HE405 HE400 Filter Replacement, H13 True HEPA Filter + + + +
+
+
+
+ + +
+ HE401 Filter Replacement for Shark Air Purifier 4 - Compatible with Shark Air Purifier 4 HE401 HE402 HE402AMZ HE405 HE400 Filter Replacement, H13 True HEPA Filter +
+ + +
+
+ + +
+ +
+ + + Return or replace items: Eligible through September 11, 2023 + + +
+ + +
+
+ + +
+
Buy it again
+ + + + + View your item + + +
+
+
+
+
+ +
+ +
+ +
+ + + `); +export const fullTextOnly = `THIS IS BODY + +this is a paragraph + +google Product 1 Product 2 +HE401 Filter Replacement for Shark Air Purifier 4 - Compatible with Shark Air +Purifier 4 HE401 HE402 HE402AMZ HE405 HE400 Filter Replacement, H13 True HEPA +Filter + +Return or replace items: Eligible through September 11, 2023 +Buy it again +View your item + * ←Previous + * 1 + * 2 + * 3 + * Next→`; + +export const fullTextWithImages = `THIS IS BODY + +this is a paragraph + +google Product 1 +[https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png] +Product 2 +[https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png] +HE401 Filter Replacement for Shark Air Purifier 4 - Compatible with Shark Air +Purifier 4 HE401 HE402 HE402AMZ HE405 HE400 Filter Replacement, H13 True HEPA +Filter [https://m.media-amazon.com/images/I/41pefwMK1VL._SS142_.jpg] +HE401 Filter Replacement for Shark Air Purifier 4 - Compatible with Shark Air +Purifier 4 HE401 HE402 HE402AMZ HE405 HE400 Filter Replacement, H13 True HEPA +Filter + +Return or replace items: Eligible through September 11, 2023 +Buy it again +View your item + * ←Previous + * 1 + * 2 + * 3 + * Next→`; + +export const fullTextAmazonPaginationOnly = ` * ←Previous + [/gp/your-account/order-history/ref=ppx_yo_dt_b_pagination_7_6?ie=UTF8&orderFilter=year-2022&search=&startIndex=50] + * 1 + [/gp/your-account/order-history/ref=ppx_yo_dt_b_pagination_7_1?ie=UTF8&orderFilter=year-2022&search=&startIndex=0] + * 2 + [/gp/your-account/order-history/ref=ppx_yo_dt_b_pagination_7_2?ie=UTF8&orderFilter=year-2022&search=&startIndex=10] + * 3 + [/gp/your-account/order-history/ref=ppx_yo_dt_b_pagination_7_3?ie=UTF8&orderFilter=year-2022&search=&startIndex=20] + * Next→ + [/gp/your-account/order-history/ref=ppx_yo_dt_b_pagination_7_8?ie=UTF8&orderFilter=year-2022&search=&startIndex=70]`; diff --git a/packages/ai-scraper/test/mocks/testProductMetaData.ts b/packages/ai-scraper/test/mocks/testProductMetaData.ts new file mode 100644 index 0000000000..7f63eb929f --- /dev/null +++ b/packages/ai-scraper/test/mocks/testProductMetaData.ts @@ -0,0 +1,53 @@ +import { LLMResponse } from "@snickerdoodlelabs/objects"; + +export const chatGPTProductMetaResponse = LLMResponse( + '[\n {\n "product_id": 1,\n "sub_category": "Hair Care",\n "category": "Beauty & Health",\n "keywords": ["Mielle Organics", "Rosemary Mint", "Scalp & Hair Strengthening Oil", "Biotin", "Essential Oils", "Nourishing Treatment", "Split Ends", "Dry Scalp", "All Hair Types", "2-Fluid Ounces"]\n },\n {\n "product_id": 2,\n "sub_category": "Outdoor Cooking",\n "category": "Outdoors",\n "keywords": ["Bonnlo", "3 Burner", "Outdoor Portable Propane Stove", "Gas Cooker", "Heavy Duty Iron Cast", "Patio Burner", "Detachable Stand Legs", "Camp Cooking", "225,000-BTU"]\n },\n {\n "product_id": 3,\n "sub_category": "Garage Door Opener",\n "category": "Home & Gardening",\n "keywords": ["Genie", "7155-TKV", "Smart Garage Door Opener", "StealthDrive Connect", "Ultra Quiet opener", "WiFi", "Battery Backup", "Works with Alexa", "Google Home"]\n },\n {\n "product_id": 4,\n "sub_category": "Car Phone Mount",\n "category": "Electronics",\n "keywords": ["VANMASS", "Universal Car Phone Mount", "Patent & Safety Certs", "Upgraded Handsfree Stand", "Phone Holder", "Car Dashboard Windshield Vent", "Compatible with iPhone", "Samsung Android", "Pickup Truck"]\n },\n {\n "product_id": 5,\n "sub_category": "Outdoor Cooking",\n "category": "Outdoors",\n "keywords": ["Bonnlo", "3 Burner", "Outdoor Portable Propane Stove", "Gas Cooker", "Heavy Duty Iron Cast", "Patio Burner", "Detachable Stand Legs", "Camp Cooking", "225,000-BTU"]\n },\n {\n "product_id": 6,\n "sub_category": "Garage Door Opener",\n "category": "Home & Gardening",\n "keywords": ["Genie", "7155-TKV", "Smart Garage Door Opener", "StealthDrive Connect", "Ultra Quiet opener", "WiFi", "Battery Backup", "Works with Alexa", "Google Home"]\n },\n {\n "product_id": 7,\n "sub_category": "Car Phone Mount",\n "category": "Electronics",\n "keywords": ["VANMASS", "Universal Car Phone Mount", "Patent & Safety Certs", "Upgraded Handsfree Stand", "Phone Holder", "Car Dashboard Windshield Vent", "Compatible with iPhone", "Samsung Android", "Pickup Truck"]\n },\n {\n "product_id": 9,\n "sub_category": "Outdoor Cooking",\n "category": "Outdoors",\n "keywords": ["Bonnlo", "3 Burner", "Outdoor Portable Propane Stove", "Gas Cooker", "Heavy Duty Iron Cast", "Patio Burner", "Detachable Stand Legs", "Camp Cooking", "225,000-BTU"]\n },\n {\n "product_id": 10,\n "sub_category": "Garage Door Opener",\n "category": "Home & Gardening",\n "keywords": ["Genie", "7155-TKV", "Smart Garage Door Opener", "StealthDrive Connect", "Ultra Quiet opener", "WiFi", "Battery Backup", "Works with Alexa", "Google Home"]\n },\n {\n "product_id": 11,\n "sub_category": "Car Phone Mount",\n "category": "Electronics",\n "keywords": ["VANMASS", "Universal Car Phone Mount", "Patent & Safety Certs", "Upgraded Handsfree Stand", "Phone Holder", "Car Dashboard Windshield Vent", "Compatible with iPhone", "Samsung Android", "Pickup Truck"]\n }\n]', +); + +export const chatGPTProductMetaResponseMissingProductId = LLMResponse( + '[\n {\n "product_id": null,\n "sub_category": "Hair Care",\n "category": "Beauty & Health",\n "keywords": ["Mielle Organics", "Rosemary Mint", "Scalp & Hair Strengthening Oil", "Biotin", "Essential Oils", "Nourishing Treatment", "Split Ends", "Dry Scalp", "All Hair Types", "2-Fluid Ounces"]\n },\n {\n "product_id": 2,\n "sub_category": "Outdoor Cooking",\n "category": "Outdoors",\n "keywords": ["Bonnlo", "3 Burner", "Outdoor Portable Propane Stove", "Gas Cooker", "Heavy Duty Iron Cast", "Patio Burner", "Detachable Stand Legs", "Camp Cooking", "225,000-BTU"]\n },\n {\n "product_id": 3,\n "sub_category": "Garage Door Opener",\n "category": "Home & Gardening",\n "keywords": ["Genie", "7155-TKV", "Smart Garage Door Opener", "StealthDrive Connect", "Ultra Quiet opener", "WiFi", "Battery Backup", "Works with Alexa", "Google Home"]\n },\n {\n "product_id": 4,\n "sub_category": "Car Phone Mount",\n "category": "Electronics",\n "keywords": ["VANMASS", "Universal Car Phone Mount", "Patent & Safety Certs", "Upgraded Handsfree Stand", "Phone Holder", "Car Dashboard Windshield Vent", "Compatible with iPhone", "Samsung Android", "Pickup Truck"]\n },\n {\n "product_id": 5,\n "sub_category": "Outdoor Cooking",\n "category": "Outdoors",\n "keywords": ["Bonnlo", "3 Burner", "Outdoor Portable Propane Stove", "Gas Cooker", "Heavy Duty Iron Cast", "Patio Burner", "Detachable Stand Legs", "Camp Cooking", "225,000-BTU"]\n },\n {\n "product_id": 6,\n "sub_category": "Garage Door Opener",\n "category": "Home & Gardening",\n "keywords": ["Genie", "7155-TKV", "Smart Garage Door Opener", "StealthDrive Connect", "Ultra Quiet opener", "WiFi", "Battery Backup", "Works with Alexa", "Google Home"]\n },\n {\n "product_id": 7,\n "sub_category": "Car Phone Mount",\n "category": "Electronics",\n "keywords": ["VANMASS", "Universal Car Phone Mount", "Patent & Safety Certs", "Upgraded Handsfree Stand", "Phone Holder", "Car Dashboard Windshield Vent", "Compatible with iPhone", "Samsung Android", "Pickup Truck"]\n },\n {\n "product_id": 9,\n "sub_category": "Outdoor Cooking",\n "category": "Outdoors",\n "keywords": ["Bonnlo", "3 Burner", "Outdoor Portable Propane Stove", "Gas Cooker", "Heavy Duty Iron Cast", "Patio Burner", "Detachable Stand Legs", "Camp Cooking", "225,000-BTU"]\n },\n {\n "product_id": 10,\n "sub_category": "Garage Door Opener",\n "category": "Home & Gardening",\n "keywords": ["Genie", "7155-TKV", "Smart Garage Door Opener", "StealthDrive Connect", "Ultra Quiet opener", "WiFi", "Battery Backup", "Works with Alexa", "Google Home"]\n },\n {\n "product_id": 11,\n "sub_category": "Car Phone Mount",\n "category": "Electronics",\n "keywords": ["VANMASS", "Universal Car Phone Mount", "Patent & Safety Certs", "Upgraded Handsfree Stand", "Phone Holder", "Car Dashboard Windshield Vent", "Compatible with iPhone", "Samsung Android", "Pickup Truck"]\n }\n]', +); + +export const chatGPTProductMetaResponseMissingCategory = LLMResponse( + '[\n {\n "product_id": 1,\n "sub_category": "Hair Care",\n "category": null,\n "keywords": ["Mielle Organics", "Rosemary Mint", "Scalp & Hair Strengthening Oil", "Biotin", "Essential Oils", "Nourishing Treatment", "Split Ends", "Dry Scalp", "All Hair Types", "2-Fluid Ounces"]\n },\n {\n "product_id": 2,\n "sub_category": "Outdoor Cooking",\n "category": "Outdoors",\n "keywords": ["Bonnlo", "3 Burner", "Outdoor Portable Propane Stove", "Gas Cooker", "Heavy Duty Iron Cast", "Patio Burner", "Detachable Stand Legs", "Camp Cooking", "225,000-BTU"]\n },\n {\n "product_id": 3,\n "sub_category": "Garage Door Opener",\n "category": "Home & Gardening",\n "keywords": ["Genie", "7155-TKV", "Smart Garage Door Opener", "StealthDrive Connect", "Ultra Quiet opener", "WiFi", "Battery Backup", "Works with Alexa", "Google Home"]\n },\n {\n "product_id": 4,\n "sub_category": "Car Phone Mount",\n "category": "Electronics",\n "keywords": ["VANMASS", "Universal Car Phone Mount", "Patent & Safety Certs", "Upgraded Handsfree Stand", "Phone Holder", "Car Dashboard Windshield Vent", "Compatible with iPhone", "Samsung Android", "Pickup Truck"]\n },\n {\n "product_id": 5,\n "sub_category": "Outdoor Cooking",\n "category": "Outdoors",\n "keywords": ["Bonnlo", "3 Burner", "Outdoor Portable Propane Stove", "Gas Cooker", "Heavy Duty Iron Cast", "Patio Burner", "Detachable Stand Legs", "Camp Cooking", "225,000-BTU"]\n },\n {\n "product_id": 6,\n "sub_category": "Garage Door Opener",\n "category": "Home & Gardening",\n "keywords": ["Genie", "7155-TKV", "Smart Garage Door Opener", "StealthDrive Connect", "Ultra Quiet opener", "WiFi", "Battery Backup", "Works with Alexa", "Google Home"]\n },\n {\n "product_id": 7,\n "sub_category": "Car Phone Mount",\n "category": "Electronics",\n "keywords": ["VANMASS", "Universal Car Phone Mount", "Patent & Safety Certs", "Upgraded Handsfree Stand", "Phone Holder", "Car Dashboard Windshield Vent", "Compatible with iPhone", "Samsung Android", "Pickup Truck"]\n },\n {\n "product_id": 9,\n "sub_category": "Outdoor Cooking",\n "category": "Outdoors",\n "keywords": ["Bonnlo", "3 Burner", "Outdoor Portable Propane Stove", "Gas Cooker", "Heavy Duty Iron Cast", "Patio Burner", "Detachable Stand Legs", "Camp Cooking", "225,000-BTU"]\n },\n {\n "product_id": 10,\n "sub_category": "Garage Door Opener",\n "category": "Home & Gardening",\n "keywords": ["Genie", "7155-TKV", "Smart Garage Door Opener", "StealthDrive Connect", "Ultra Quiet opener", "WiFi", "Battery Backup", "Works with Alexa", "Google Home"]\n },\n {\n "product_id": 11,\n "sub_category": "Car Phone Mount",\n "category": "Electronics",\n "keywords": ["VANMASS", "Universal Car Phone Mount", "Patent & Safety Certs", "Upgraded Handsfree Stand", "Phone Holder", "Car Dashboard Windshield Vent", "Compatible with iPhone", "Samsung Android", "Pickup Truck"]\n }\n]', +); + +export const emptyProductMetaResponse = LLMResponse("[]"); +export const emptyProductMetaResponseNonJSON = LLMResponse( + "There is no nothing.", +); + +export const firstProductMeta = { + product_id: 1, + sub_category: "Hair Care", + category: "Beauty & Health", + keywords: [ + "Mielle Organics", + "Rosemary Mint", + "Scalp & Hair Strengthening Oil", + "Biotin", + "Essential Oils", + "Nourishing Treatment", + "Split Ends", + "Dry Scalp", + "All Hair Types", + "2-Fluid Ounces", + ], +}; + +export const secondProductMeta = { + product_id: 2, + sub_category: "Outdoor Cooking", + category: "Outdoors", + keywords: [ + "Bonnlo", + "3 Burner", + "Outdoor Portable Propane Stove", + "Gas Cooker", + "Heavy Duty Iron Cast", + "Patio Burner", + "Detachable Stand Legs", + "Camp Cooking", + "225,000-BTU", + ], +}; diff --git a/packages/ai-scraper/test/mocks/testPurchaseHistoryData.ts b/packages/ai-scraper/test/mocks/testPurchaseHistoryData.ts new file mode 100644 index 0000000000..12b2e0c54f --- /dev/null +++ b/packages/ai-scraper/test/mocks/testPurchaseHistoryData.ts @@ -0,0 +1,558 @@ +import { + DomainName, + ELanguageCode, + LLMData, + LLMResponse, + Prompt, + PurchaseId, + PurchasedProduct, + UnixTimestamp, +} from "@snickerdoodlelabs/objects"; + +export const chatGPTPurchaseHistoryResponse = LLMResponse( + '[\n {\n "name": "VANMASS Universal Car Phone Mount",\n "brand": "VANMASS",\n "price": 23.8,\n "classification": "Electronics",\n "keywords": ["car phone mount", "handsfree stand", "phone holder", "dashboard", "windshield", "compatible", "iPhone", "Samsung", "Android", "pickup truck"],\n "date": "July 12, 2023"\n },\n {\n "name": "Genie 7155-TKV Smart Garage Door Opener",\n "brand": "Genie",\n "price": 246.8,\n "classification": "Home Improvement",\n "keywords": ["garage door opener", "smart", "WiFi", "battery backup", "Alexa", "Google Home"],\n "date": "July 12, 2023"\n },\n {\n "name": "Flexzilla Garden Hose",\n "brand": "Flexzilla",\n "price": 57.99,\n "classification": "Patio, Lawn & Garden",\n "keywords": ["garden hose", "heavy duty", "lightweight", "drinking water safe", "ZillaGreen"],\n "date": "July 11, 2023"\n },\n {\n "name": "Homall L Shaped Gaming Desk",\n "brand": "Homall",\n "price": 64.94,\n "classification": "Office Products",\n "keywords": ["gaming desk", "computer corner desk", "PC gaming desk", "monitor riser stand", "home office", "writing workstation"],\n "date": "July 9, 2023"\n },\n {\n "name": "ASURION 3 Year Furniture Protection Plan",\n "brand": "ASURION",\n "price": 15.14,\n "classification": "Home & Kitchen",\n "keywords": ["furniture protection plan"],\n "date": "July 9, 2023"\n },\n {\n "name": "Rust-Oleum 261845 EpoxyShield Garage Floor Coating",\n "brand": "Rust-Oleum",\n "price": 200.25,\n "classification": "Tools & Home Improvement",\n "keywords": ["garage floor coating", "2 gal", "gray"],\n "date": "July 3, 2023"\n },\n {\n "name": "Bonnlo 3 Burner Outdoor Portable Propane Stove Gas Cooker",\n "brand": "Bonnlo",\n "price": 115.92,\n "classification": "Sports & Outdoors",\n "keywords": ["outdoor portable propane stove", "gas cooker", "heavy duty", "iron cast", "patio burner", "camp cooking"],\n "date": "July 1, 2023"\n },\n {\n "name": "Purrfectzone Bidet Sprayer for Toilet",\n "brand": "Purrfectzone",\n "price": 0,\n "classification": "Tools & Home Improvement",\n "keywords": ["bidet sprayer", "handheld sprayer kit", "cloth diaper sprayer set"],\n "date": "June 27, 2023"\n },\n {\n "name": "Mielle Organics Rosemary Mint Scalp & Hair Strengthening Oil",\n "brand": "Mielle Organics",\n "price": 9.78,\n "classification": "Beauty & Personal Care",\n "keywords": ["hair strengthening oil", "biotin", "essential oils", "nourishing treatment", "split ends", "dry scalp"],\n "date": "June 26, 2023"\n },\n {\n "name": "Cantu Coconut Curling Cream with Shea Butter",\n "brand": "Cantu",\n "price": 25.64,\n "classification": "Beauty & Personal Care",\n "keywords": ["coconut curling cream", "shea butter", "natural hair"],\n "date": "June 26, 2023"\n },\n {\n "name": "Amazon Brand - Mama Bear Organic Kids Vitamin D3 25 mcg (1000 IU) Gummies",\n "brand": "Amazon Brand - Mama Bear",\n "price": 25.64,\n "classification": "Health & Household",\n "keywords": ["organic kids vitamin D3", "bone health", "immune health", "strawberry"],\n "date": "June 26, 2023"\n },\n {\n "name": "Garnier Fructis Sleek & Shine Moroccan Sleek Smoothing Oil",\n "brand": "Garnier Fructis",\n "price": 25.64,\n "classification": "Beauty & Personal Care",\n "keywords": ["sleek & shine oil", "frizzy hair", "dry hair", "argan oil"],\n "date": "June 26, 2023"\n }\n]', +); + +export const chatGPTPurchaseHistoryResponseFirstMissingDate = LLMResponse( + '[\n {\n "name": "VANMASS Universal Car Phone Mount",\n "brand": "VANMASS",\n "price": 23.8,\n "classification": "Electronics",\n "keywords": ["car phone mount", "handsfree stand", "phone holder", "dashboard", "windshield", "compatible", "iPhone", "Samsung", "Android", "pickup truck"],\n "date": null\n },\n {\n "name": "Genie 7155-TKV Smart Garage Door Opener",\n "brand": "Genie",\n "price": 246.8,\n "classification": "Home Improvement",\n "keywords": ["garage door opener", "smart", "WiFi", "battery backup", "Alexa", "Google Home"],\n "date": "July 12, 2023"\n },\n {\n "name": "Flexzilla Garden Hose",\n "brand": "Flexzilla",\n "price": 57.99,\n "classification": "Patio, Lawn & Garden",\n "keywords": ["garden hose", "heavy duty", "lightweight", "drinking water safe", "ZillaGreen"],\n "date": "July 11, 2023"\n },\n {\n "name": "Homall L Shaped Gaming Desk",\n "brand": "Homall",\n "price": 64.94,\n "classification": "Office Products",\n "keywords": ["gaming desk", "computer corner desk", "PC gaming desk", "monitor riser stand", "home office", "writing workstation"],\n "date": "July 9, 2023"\n },\n {\n "name": "ASURION 3 Year Furniture Protection Plan",\n "brand": "ASURION",\n "price": 15.14,\n "classification": "Home & Kitchen",\n "keywords": ["furniture protection plan"],\n "date": "July 9, 2023"\n },\n {\n "name": "Rust-Oleum 261845 EpoxyShield Garage Floor Coating",\n "brand": "Rust-Oleum",\n "price": 200.25,\n "classification": "Tools & Home Improvement",\n "keywords": ["garage floor coating", "2 gal", "gray"],\n "date": "July 3, 2023"\n },\n {\n "name": "Bonnlo 3 Burner Outdoor Portable Propane Stove Gas Cooker",\n "brand": "Bonnlo",\n "price": 115.92,\n "classification": "Sports & Outdoors",\n "keywords": ["outdoor portable propane stove", "gas cooker", "heavy duty", "iron cast", "patio burner", "camp cooking"],\n "date": "July 1, 2023"\n },\n {\n "name": "Purrfectzone Bidet Sprayer for Toilet",\n "brand": "Purrfectzone",\n "price": 0,\n "classification": "Tools & Home Improvement",\n "keywords": ["bidet sprayer", "handheld sprayer kit", "cloth diaper sprayer set"],\n "date": "June 27, 2023"\n },\n {\n "name": "Mielle Organics Rosemary Mint Scalp & Hair Strengthening Oil",\n "brand": "Mielle Organics",\n "price": 9.78,\n "classification": "Beauty & Personal Care",\n "keywords": ["hair strengthening oil", "biotin", "essential oils", "nourishing treatment", "split ends", "dry scalp"],\n "date": "June 26, 2023"\n },\n {\n "name": "Cantu Coconut Curling Cream with Shea Butter",\n "brand": "Cantu",\n "price": 25.64,\n "classification": "Beauty & Personal Care",\n "keywords": ["coconut curling cream", "shea butter", "natural hair"],\n "date": "June 26, 2023"\n },\n {\n "name": "Amazon Brand - Mama Bear Organic Kids Vitamin D3 25 mcg (1000 IU) Gummies",\n "brand": "Amazon Brand - Mama Bear",\n "price": 25.64,\n "classification": "Health & Household",\n "keywords": ["organic kids vitamin D3", "bone health", "immune health", "strawberry"],\n "date": "June 26, 2023"\n },\n {\n "name": "Garnier Fructis Sleek & Shine Moroccan Sleek Smoothing Oil",\n "brand": "Garnier Fructis",\n "price": 25.64,\n "classification": "Beauty & Personal Care",\n "keywords": ["sleek & shine oil", "frizzy hair", "dry hair", "argan oil"],\n "date": "June 26, 2023"\n }\n]', +); + +export const chatGPTPurchaseHistoryResponseMissingPrice = LLMResponse( + '[\n {\n "name": "VANMASS Universal Car Phone Mount",\n "brand": "VANMASS",\n "price": null,\n "classification": "Electronics",\n "keywords": ["car phone mount", "handsfree stand", "phone holder", "dashboard", "windshield", "compatible", "iPhone", "Samsung", "Android", "pickup truck"],\n "date": "July 12, 2023"\n },\n {\n "name": "Genie 7155-TKV Smart Garage Door Opener",\n "brand": "Genie",\n "price": 246.8,\n "classification": "Home Improvement",\n "keywords": ["garage door opener", "smart", "WiFi", "battery backup", "Alexa", "Google Home"],\n "date": "July 12, 2023"\n },\n {\n "name": "Flexzilla Garden Hose",\n "brand": "Flexzilla",\n "price": 57.99,\n "classification": "Patio, Lawn & Garden",\n "keywords": ["garden hose", "heavy duty", "lightweight", "drinking water safe", "ZillaGreen"],\n "date": "July 11, 2023"\n },\n {\n "name": "Homall L Shaped Gaming Desk",\n "brand": "Homall",\n "price": 64.94,\n "classification": "Office Products",\n "keywords": ["gaming desk", "computer corner desk", "PC gaming desk", "monitor riser stand", "home office", "writing workstation"],\n "date": "July 9, 2023"\n },\n {\n "name": "ASURION 3 Year Furniture Protection Plan",\n "brand": "ASURION",\n "price": 15.14,\n "classification": "Home & Kitchen",\n "keywords": ["furniture protection plan"],\n "date": "July 9, 2023"\n },\n {\n "name": "Rust-Oleum 261845 EpoxyShield Garage Floor Coating",\n "brand": "Rust-Oleum",\n "price": 200.25,\n "classification": "Tools & Home Improvement",\n "keywords": ["garage floor coating", "2 gal", "gray"],\n "date": "July 3, 2023"\n },\n {\n "name": "Bonnlo 3 Burner Outdoor Portable Propane Stove Gas Cooker",\n "brand": "Bonnlo",\n "price": 115.92,\n "classification": "Sports & Outdoors",\n "keywords": ["outdoor portable propane stove", "gas cooker", "heavy duty", "iron cast", "patio burner", "camp cooking"],\n "date": "July 1, 2023"\n },\n {\n "name": "Purrfectzone Bidet Sprayer for Toilet",\n "brand": "Purrfectzone",\n "price": 0,\n "classification": "Tools & Home Improvement",\n "keywords": ["bidet sprayer", "handheld sprayer kit", "cloth diaper sprayer set"],\n "date": "June 27, 2023"\n },\n {\n "name": "Mielle Organics Rosemary Mint Scalp & Hair Strengthening Oil",\n "brand": "Mielle Organics",\n "price": 9.78,\n "classification": "Beauty & Personal Care",\n "keywords": ["hair strengthening oil", "biotin", "essential oils", "nourishing treatment", "split ends", "dry scalp"],\n "date": "June 26, 2023"\n },\n {\n "name": "Cantu Coconut Curling Cream with Shea Butter",\n "brand": "Cantu",\n "price": 25.64,\n "classification": "Beauty & Personal Care",\n "keywords": ["coconut curling cream", "shea butter", "natural hair"],\n "date": "June 26, 2023"\n },\n {\n "name": "Amazon Brand - Mama Bear Organic Kids Vitamin D3 25 mcg (1000 IU) Gummies",\n "brand": "Amazon Brand - Mama Bear",\n "price": 25.64,\n "classification": "Health & Household",\n "keywords": ["organic kids vitamin D3", "bone health", "immune health", "strawberry"],\n "date": "June 26, 2023"\n },\n {\n "name": "Garnier Fructis Sleek & Shine Moroccan Sleek Smoothing Oil",\n "brand": "Garnier Fructis",\n "price": 25.64,\n "classification": "Beauty & Personal Care",\n "keywords": ["sleek & shine oil", "frizzy hair", "dry hair", "argan oil"],\n "date": "June 26, 2023"\n }\n]', +); + +export const emptyPurchaseHistoryList = LLMResponse("[]"); +export const emptyPurchaseHistoryNonJSON = LLMResponse("There is no history."); + +export const firstPurchase = { + name: "VANMASS Universal Car Phone Mount", + brand: "VANMASS", + price: 23.8, + classification: "Electronics", + keywords: [ + "car phone mount", + "handsfree stand", + "phone holder", + "dashboard", + "windshield", + "compatible", + "iPhone", + "Samsung", + "Android", + "pickup truck", + ], + date: "July 12, 2023", +}; + +export const nonHalucinatedPurchase = new PurchasedProduct( + DomainName("Amazon"), + ELanguageCode.English, + PurchaseId("114-1517274-7393830"), + "VANMASS Universal Car Phone Mount", + "VANMASS", + 23.8, + UnixTimestamp(1689145200), + UnixTimestamp(1689145200), + null, + null, + null, + "Electronics", + null, +); +export const halucinatedPurchase = new PurchasedProduct( + DomainName("Amazon"), + ELanguageCode.English, + PurchaseId("114-1517274-7393830"), + "My Product", + null, + 100, + UnixTimestamp(0), + UnixTimestamp(0), + null, + null, + null, + "Beauty", + null, +); + +export const multiplePurchasesInOneOrder = [ + { + name: "Cantu Coconut Curling Cream with Shea Butter", + brand: "Cantu", + price: 25.64, + classification: "Beauty & Personal Care", + keywords: ["coconut curling cream", "shea butter", "natural hair"], + date: "June 26, 2023", + }, + { + name: "Amazon Brand - Mama Bear Organic Kids Vitamin D3 25 mcg (1000 IU) Gummies", + brand: "Amazon Brand - Mama Bear", + price: 25.64, + classification: "Health & Household", + keywords: [ + "organic kids vitamin D3", + "bone health", + "immune health", + "strawberry", + ], + date: "June 26, 2023", + }, + { + name: "Garnier Fructis Sleek & Shine Moroccan Sleek Smoothing Oil", + brand: "Garnier Fructis", + price: 25.64, + classification: "Beauty & Personal Care", + keywords: ["sleek & shine oil", "frizzy hair", "dry hair", "argan oil"], + date: "June 26, 2023", + }, +]; + +export const purchaseHistoryData = LLMData( + ` * Your Account + * › + * Your Orders + + + YOUR ORDERS + + Search Orders + * Orders + * Buy Again + * Not Yet Shipped + * Digital Orders + * Local Store Orders + * Amazon Pay + * Cancelled Orders + + 23 orders placed in last 30 days past 3 months 2023 2022 2021 2020 2019 2018 + Archived Orders past 3 months + + AN ITEM YOU BOUGHT HAS BEEN RECALLED + + To ensure your safety, go to Your Recalls and Product Safety Alerts and see + recall information. + + ACTION IS REQUIRED ON ONE OR MORE OF YOUR ORDERS. + + Please see below. + + THERE'S A PROBLEM DISPLAYING YOUR ORDERS RIGHT NOW. + + + Order placed + July 12, 2023 + Total + $23.80 + Ship to + hidden name + hidden name + hidden info + hidden info, hidden info + hidden info + + Order # 114-1517274-7393830 + View order details + + View invoice + + Arriving Thursday + + Track package + VANMASS Universal Car Phone Mount,【Patent & Safety Certs】 Upgraded Handsfree + Stand, Phone Holder for Car Dashboard Windshield Vent, Compatible with iPhone 14 + 13 12 Samsung Android & Pickup Truck + + + Buy it again + View or edit order + Archive order + Order placed + July 12, 2023 + Total + $246.80 + Ship to + hidden name + hidden name + hidden info + hidden info, hidden info + hidden info + + Order # 114-2496144-2288268 + View order details + + View invoice + + Arriving Thursday + + Track package + Genie 7155-TKV Smart Garage Door Opener StealthDrive Connect - Ultra Quiet + opener, WiFi, Battery Backup - Works with Alexa & Google Home + + + Buy it again + View or edit order + Archive order + Order placed + July 11, 2023 + Total + $57.99 + Ship to + hidden name + hidden name + hidden info + hidden info, hidden info + hidden info + Order # 114-9708805-8079465 + View order details View invoice + Arriving today by 8 PM + Shipped + Flexzilla Garden Hose 5/8 in. x 100 ft., Heavy Duty, Lightweight, Drinking Water + Safe, ZillaGreen - HFZG5100YW-E + + Buy it again + Track package Get product support Return or replace items Share gift receipt + Write a product review + Archive order + Order placed + July 9, 2023 + Total + $64.94 + Ship to + hidden name + hidden name + hidden info + hidden info, hidden info + hidden info + Order # 114-6884294-0047419 + View order details View invoice + Delivered July 11 + + Homall L Shaped Gaming Desk Computer Corner Desk PC Gaming Desk Table with Large + Monitor Riser Stand for Home Office Sturdy Writing Workstation (Black, 51 Inch) + + Return items: Eligible through August 10, 2023 + Buy it again + View your item + Get product support Problem with order Track package Return items Share gift + receipt Write a product review + Archive order + Order placed + July 9, 2023 + Total + $15.14 + Policy sent to + adhocmaster@live.com + + Order # 114-3068656-9825042 + View order details + + View invoice + + Email delivery + + ASURION 3 Year Furniture Protection Plan ($60 - $69.99) + + + Buy it again + Problem with order Return items Share gift receipt Leave seller feedback Write a + product review + Archive order + Order placed + July 3, 2023 + Total + $200.25 + Ship to + hidden name + hidden name + hidden info + hidden info, hidden info + hidden info + Order # 114-1479831-3352260 + View order details View invoice + Delivered July 11 + Package was left in a parcel locker + Rust-Oleum 261845 EpoxyShield Garage Floor Coating , 2 gal, Gray + + Buy it again + View your item + Get product support Track package Return or replace items Write a product review + Archive order + Order placed + July 1, 2023 + Total + $115.92 + Ship to + hidden name + hidden name + hidden info + hidden info, hidden info + hidden info + Order # 113-0866043-5306630 + View order details View invoice + Delivered July 6 + + Bonnlo 3 Burner Outdoor Portable Propane Stove Gas Cooker, Heavy Duty Iron Cast + Patio Burner with Detachable Stand Legs for Camp Cooking (3-Burner 225,000-BTU) + + Buy it again + View your item + Problem with order Track package Return or replace items Write a product review + Archive order + Order placed + June 27, 2023 + Total + $0.00 + Ship to + hidden name + hidden name + hidden info + hidden info, hidden info + hidden info + Order # 113-7330109-7148238 + View order details View invoice + Delivered June 28 + Your package was left near the front door or porch. + Purrfectzone Bidet Sprayer for Toilet, Handheld Sprayer Kit , Cloth Diaper + Sprayer Set - Easy to Install - Stainless Steel + + Return or replace items: Eligible through July 28, 2023 + Buy it again + View your item + Get product support Track package Return or replace items Share gift receipt Get + help Write a product review + Archive order + Order placed + June 26, 2023 + Total + $9.78 + Ship to + hidden name + hidden name + hidden info + hidden info, hidden info + hidden info + Order # 113-4452204-9348259 + View order details View invoice + Delivered June 27 + Your package was left near the front door or porch. + Mielle Organics Rosemary Mint Scalp & Hair Strengthening Oil With Biotin & + Essential Oils, Nourishing Treatment for Split Ends and Dry Scalp for All Hair + Types, 2-Fluid Ounces + + Return or replace items: Eligible through July 27, 2023 + Buy it again + View your item + Track package Return or replace items Share gift receipt Get help Write a + product review + Archive order + Order placed + June 26, 2023 + Total + $25.64 + Ship to + hidden name + hidden name + hidden info + hidden info, hidden info + hidden info + Order # 113-4291615-9645819 + View order details View invoice + Delivered June 28 + Your package was left near the front door or porch. + Cantu Coconut Curling Cream with Shea Butter for Natural Hair, 12 oz (Packaging + May Vary) + + Return or replace items: Eligible through July 28, 2023 + Buy it again + View your item + Amazon Brand - Mama Bear Organic Kids Vitamin D3 25 mcg (1000 IU) Gummies per + serving, Bone and Immune Health, Strawberry, 80 Count + + Buy it again + View your item + Garnier Fructis Sleek & Shine Moroccan Sleek Smoothing Oil for Frizzy, Dry Hair, + Argan Oil, 3.75 Fl Oz, 1 Count (Packaging May Vary) + + Return or replace items: Eligible through July 28, 2023 + Buy it again + View your item + Get product support Track package Return or replace items Return or replace + items Share gift receipt Write a product review + Archive order + * ←Previous + * 1 + * 2 + * 3 + * Next→ + + `, +); + +export const purchaseHistoryPromptWithNoPurchases = + Prompt(`You are an expert in understanding e-commerce. I need to extract purchase information. I need all the output for each purchase in this format: +\n\nJSON format: \n + { + name: string, + brand: string, + price: number, + classification: string, + keywords: string[], + date: string + } + I need the purchase history from the following content. You need to follow theses rules: + A purchase history must have a product name, price, and date of purchase. It can also have brand, classification, keywords which are optional. + Classification denotes the category of the product and keywords describe the products using a few key words. + The purchase date and price cannot be null. + Do not include a purchase information in the output if the purchase date or price is missing. + Do not push yourself hard and do not generate imaginary purchases. + Give response in a JSON array in the preceding format. :\n\n + + Your Orders + Skip to main content + .us + + All + Select the department you want to search in All Departments Arts & Crafts + Automotive Baby Beauty & Personal Care Books Boys' Fashion Computers Deals + Digital Music Electronics Girls' Fashion Health & Household Home & Kitchen + Industrial & Scientific Kindle Store Luggage Men's Fashion Movies & TV Music, + CDs & Vinyl Pet Supplies Prime Video Software Sports & Outdoors Tools & Home + Improvement Toys & Games Video Games Women's Fashion + Search Amazon + + + EN + Hello, Orhun + Account & Lists Not Orhun? Sign Out Returns & Orders + 1 + Cart + + + All + Today's Deals Buy Again Customer Service Gift Cards Orhun's Amazon.com Browsing + History Sell Registry Disability Customer Support + + + + * Your Account + * › + * Your Orders + + THERE'S A PROBLEM DISPLAYING SOME OF YOUR ORDERS RIGHT NOW. + + If you don't see the order you're looking for, try refreshing this page, or + click "View order details" for that order. + + + YOUR ORDERS + + Search Your Orders: + + Search Orders + * + * Orders + * Buy Again + * Not Yet Shipped + * Digital Orders + * Local Store Orders + * Cancelled Orders + + 0 orders placed in + last 30 days past 3 months 2023 2022 2021 2020 2023 Go + You have not placed any orders in 2023. View orders in 2022 + + YOUR ATTENTION IS REQUIRED TO CONTINUE PROCESSING ONE OR MORE ORDERS ON THIS + PAGE. + + Please see below to address the issue. + + YOU HAVE AT LEAST ONE ORDER PENDING APPROVAL + + Unapproved orders expire after 48 hours + + + + + + + + Your recently viewed items and featured recommendations + › + View or edit your browsing history + After viewing product detail pages, look here to find an easy way to navigate + back to pages you are interested in. + Your recently viewed items and featured recommendations + › + View or edit your browsing history + After viewing product detail pages, look here to find an easy way to navigate + back to pages you are interested in. + + Back to top + Get to Know Us + * Careers + * Blog + * About Amazon + * Investor Relations + * Amazon Devices + * Amazon Science + + + Make Money with Us + * Sell products on Amazon + * Sell on Amazon Business + * Sell apps on Amazon + * Become an Affiliate + * Advertise Your Products + * Self-Publish with Us + * Host an Amazon Hub + * ›See More Make Money with Us + + + Amazon Payment Products + * Amazon Business Card + * Shop with Points + * Reload Your Balance + * Amazon Currency Converter + + + Let Us Help You + * Amazon and COVID-19 + * Your Account + * Your Orders + * Shipping Rates & Policies + * Returns & Replacements + * Manage Your Content and Devices + * Amazon Assistant + * Help + + + + English TRYTurkish Lira United States + + Amazon Music + Stream millions + of songs Amazon Ads + Reach customers + wherever they + spend their time 6pm + Score deals + on fashion brands AbeBooks + Books, art + & collectibles ACX + Audiobook Publishing + Made Easy Sell on Amazon + Start a Selling Account Amazon Business + Everything For + Your Business AmazonGlobal + Ship Orders + Internationally Home Services + Experienced Pros + Happiness Guarantee Amazon Web Services + Scalable Cloud + Computing Services Audible + Listen to Books & Original + Audio Performances Box Office Mojo + Find Movie + Box Office Data Goodreads + Book revi + `); diff --git a/packages/ai-scraper/test/mocks/testValues.ts b/packages/ai-scraper/test/mocks/testValues.ts index 81ed22a453..afc5c2a1f0 100644 --- a/packages/ai-scraper/test/mocks/testValues.ts +++ b/packages/ai-scraper/test/mocks/testValues.ts @@ -1,4 +1,7 @@ -import { URLString } from "@snickerdoodlelabs/objects"; +import { Exemplar, URLString } from "@snickerdoodlelabs/objects"; +import { ChatCompletion } from "openai/resources/chat"; + +import { IScraperConfig } from "@ai-scraper/interfaces"; export const AMAZON_URL = URLString( "https://www.amazon.com/gp/css/order-history?ref_=nav_orders_first", @@ -9,3 +12,30 @@ export const GOOGLE_URL = URLString("https://www.google.com"); export const INVALID_URL = URLString("invalidUrl"); export const AMAZON_HOST_NAME = "www.amazon.com"; + +// region exemplars + +export const Exemplars = [Exemplar("Q: 1 + 2 \n A: 3")]; + +export const scraperConfig: IScraperConfig = { + scraper: { + OPENAI_API_KEY: "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", // TODO: this is risky. What should we do + timeout: 5 * 60 * 1000, // 5 minutes + }, +}; + +export const chatCompletion: ChatCompletion = { + id: "cmpl-3QJ8Z5jX9J5X3", + // object: "text_completion", + object: 'chat.completion', + created: 1627770949, + model: "gpt-3.5-turbo", + choices: [ + { + finish_reason: "stop", + logprobs: null, + index: 0, + message: { content: "chatCompletion test content", role: "assistant" }, + }, + ], +}; diff --git a/packages/ai-scraper/test/unit/business/HTMLPreProcessor.test.ts b/packages/ai-scraper/test/unit/business/HTMLPreProcessor.test.ts new file mode 100644 index 0000000000..82b84af0c3 --- /dev/null +++ b/packages/ai-scraper/test/unit/business/HTMLPreProcessor.test.ts @@ -0,0 +1,62 @@ +import "reflect-metadata"; + +import { HTMLPreProcessor } from "@ai-scraper/implementations/utils/HTMLPreProcessor"; +import { IHTMLPreProcessor } from "@ai-scraper/interfaces"; +import { + fullHtml, + fullTextAmazonPaginationOnly, + fullTextOnly, + fullTextWithImages, + html1, + text1, +} from "@ai-scraper-test/mocks/testHTMLPreprocessorData"; + +class mocks { + public factory(): IHTMLPreProcessor { + return new HTMLPreProcessor(); + } +} +describe("HTMLPreProcessor", () => { + test("htmlToText hello world", async () => { + // Arrange + const processor = new mocks().factory(); + // Act + const result = await processor.htmlToText(html1, null); + // Assert + expect(result.isOk()).toBeTruthy(); + expect(result._unsafeUnwrap()).toEqual(text1); + }); + + test("htmlToText full text without images", async () => { + // Arrange + const processor = new mocks().factory(); + // Act + const result = await processor.htmlToText(fullHtml, null); + // Assert + expect(result.isOk()).toBeTruthy(); + expect(result._unsafeUnwrap()).toEqual(fullTextOnly); + }); + + test("htmlToText full text with images", async () => { + // Arrange + const processor = new mocks().factory(); + // Act + const result = await processor.htmlToTextWithImages(fullHtml); + // Assert + expect(result.isOk()).toBeTruthy(); + expect(result._unsafeUnwrap()).toEqual(fullTextWithImages); + }); + + test("htmlToText amazon pagination", async () => { + // Arrange + const processor = new mocks().factory(); + const amazonPagination = { + baseElements: { selectors: [".a-pagination"] }, + }; + // Act + const result = await processor.htmlToText(fullHtml, amazonPagination); + // Assert + expect(result.isOk()).toBeTruthy(); + expect(result._unsafeUnwrap()).toEqual(fullTextAmazonPaginationOnly); + }); +}); diff --git a/packages/ai-scraper/test/unit/business/PromptDirector.test.ts b/packages/ai-scraper/test/unit/business/PromptDirector.test.ts new file mode 100644 index 0000000000..a9fb0e6f0f --- /dev/null +++ b/packages/ai-scraper/test/unit/business/PromptDirector.test.ts @@ -0,0 +1,82 @@ +import "reflect-metadata"; +import { LLMData, Prompt } from "@snickerdoodlelabs/objects"; +import * as td from "testdouble"; + +import { PromptDirector } from "@ai-scraper/implementations"; +import { + ILLMProductMetaUtils, + ILLMPurchaseHistoryUtils, + IPromptBuilderFactory, + IPromptDirector, +} from "@ai-scraper/interfaces"; +import { MockPromptBuilder } from "@ai-scraper-test/mocks/MockPromptBuilder"; + +/** + * PromptDirector glues everything together. So, we test if it's able to glue the right things together. + */ + +const promptPH = Prompt("This is a purchase history prompt"); +const promptProductMeta = Prompt("This is a product meta prompt"); +class mocks { + private purchaseHistoryLLMUtils = td.object(); + private productMetaUtils = td.object(); + private promptBuilderFactory = td.object(); + private phBuilder = new MockPromptBuilder(promptPH); + private productMetaBuilder = new MockPromptBuilder(promptProductMeta); + public constructor() { + // td.when(this.purchaseHistoryLLMUtils.getRole()).thenReturn( + // LLMRole("user PH"), + // ); // Argument of type 'LLMRole' is not assignable to parameter of type 'never'. + // td.when(this.purchaseHistoryLLMUtils.getQuestion()).thenReturn( + // LLMQuestion("question PH"), + // ); + // td.when(this.purchaseHistoryLLMUtils.getAnswerStructure()).thenReturn( + // LLMAnswerStructure("answer PH"), + // ); + td.when(this.promptBuilderFactory.purchaseHistory()).thenReturn( + this.phBuilder, + ); + td.when(this.promptBuilderFactory.productMeta()).thenReturn( + this.productMetaBuilder, + ); + } + public factory(): IPromptDirector { + return new PromptDirector( + this.promptBuilderFactory, + this.purchaseHistoryLLMUtils, + this.productMetaUtils, + ); + } +} + +describe("PromptDirector", () => { + test("makePurchaseHistoryPrompt", async () => { + // Arrange + const m = new mocks(); + const director = m.factory(); + + // Act + const promptRes = await director.makePurchaseHistoryPrompt( + LLMData("anything"), + ); + + // Assert + expect(promptRes.isOk()).toBe(true); + const prompt = promptRes._unsafeUnwrap(); + expect(prompt).toEqual(promptPH); + }); + + test("makePromptBuilderPrompt", async () => { + // Arrange + const m = new mocks(); + const director = m.factory(); + + // Act + const promptRes = await director.makeProductMetaPrompt(LLMData("anything")); + + // Assert + expect(promptRes.isOk()).toBe(true); + const prompt = promptRes._unsafeUnwrap(); + expect(prompt).toEqual(promptProductMeta); + }); +}); diff --git a/packages/ai-scraper/test/unit/business/utils/LLMProductMetaUtilsChatGPT.test.ts b/packages/ai-scraper/test/unit/business/utils/LLMProductMetaUtilsChatGPT.test.ts new file mode 100644 index 0000000000..6440a3d85c --- /dev/null +++ b/packages/ai-scraper/test/unit/business/utils/LLMProductMetaUtilsChatGPT.test.ts @@ -0,0 +1,129 @@ +import "reflect-metadata"; + +import { TimeUtils, MockLogUtils } from "@snickerdoodlelabs/common-utils"; +import { DomainName, ELanguageCode } from "@snickerdoodlelabs/objects"; + +import { LLMProductMetaUtilsChatGPT } from "@ai-scraper/implementations/"; +import { + MockTimeUtils, + chatGPTProductMetaResponse, + chatGPTProductMetaResponseMissingCategory, + chatGPTProductMetaResponseMissingProductId, + emptyProductMetaResponse, + emptyProductMetaResponseNonJSON, + firstProductMeta, + secondProductMeta, +} from "@ai-scraper-test/mocks/index.js"; + +class Mocks { + public timeUtils = new MockTimeUtils(); + public logUtils = new MockLogUtils(); + public factory() { + return new LLMProductMetaUtilsChatGPT(this.timeUtils, this.logUtils); + } +} + +describe("LLMProductMetaUtilsChatGPT", () => { + test("parse product meta", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.parseMeta( + DomainName("amazon.com"), + ELanguageCode.English, + chatGPTProductMetaResponse, + ); + + // Assert + expect(result.isOk()).toBeTruthy(); + + const metas = result._unsafeUnwrap(); + expect(metas.length).toBe(10); + const gotFirst = metas[0]; + expect(gotFirst.productId).toBe(firstProductMeta.product_id.toString()); + expect(gotFirst.category).toBe(firstProductMeta.category); + expect(gotFirst.keywords).toEqual(firstProductMeta.keywords); + }); + test("parse product meta missing id", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.parseMeta( + DomainName("amazon.com"), + ELanguageCode.English, + chatGPTProductMetaResponseMissingProductId, + ); + + // Assert + expect(result.isOk()).toBeTruthy(); + + const metas = result._unsafeUnwrap(); + expect(metas.length).toBe(9); + const gotFirst = metas[0]; + expect(gotFirst.productId).toBe(secondProductMeta.product_id.toString()); + expect(gotFirst.category).toBe(secondProductMeta.category); + expect(gotFirst.keywords).toEqual(secondProductMeta.keywords); + }); + + test("parse product meta missing category", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.parseMeta( + DomainName("amazon.com"), + ELanguageCode.English, + chatGPTProductMetaResponseMissingCategory, + ); + + // Assert + expect(result.isOk()).toBeTruthy(); + + const metas = result._unsafeUnwrap(); + expect(metas.length).toBe(10); + const gotFirst = metas[0]; + expect(gotFirst.productId).toBe(firstProductMeta.product_id.toString()); + expect(gotFirst.category).toBeNull(); + expect(gotFirst.keywords).toEqual(firstProductMeta.keywords); + }); + + test("parse product meta empty", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.parseMeta( + DomainName("amazon.com"), + ELanguageCode.English, + emptyProductMetaResponse, + ); + + // Assert + expect(result.isOk()).toBeTruthy(); + + const metas = result._unsafeUnwrap(); + expect(metas.length).toBe(0); + }); + + test("parse product meta non JSON", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.parseMeta( + DomainName("amazon.com"), + ELanguageCode.English, + emptyProductMetaResponseNonJSON, + ); + + // Assert + expect(result.isErr()).toBeTruthy(); + }); +}); diff --git a/packages/ai-scraper/test/unit/business/utils/LLMPurchaseHistoryUtilsChatGPT.test.ts b/packages/ai-scraper/test/unit/business/utils/LLMPurchaseHistoryUtilsChatGPT.test.ts new file mode 100644 index 0000000000..9b484f70d6 --- /dev/null +++ b/packages/ai-scraper/test/unit/business/utils/LLMPurchaseHistoryUtilsChatGPT.test.ts @@ -0,0 +1,128 @@ +import "reflect-metadata"; +import { MockLogUtils, TimeUtils } from "@snickerdoodlelabs/common-utils"; +import { DomainName, ELanguageCode } from "@snickerdoodlelabs/objects"; + +import { LLMPurchaseHistoryUtilsChatGPT } from "@ai-scraper/implementations"; +import { + chatGPTPurchaseHistoryResponse, + chatGPTPurchaseHistoryResponseFirstMissingDate, + chatGPTPurchaseHistoryResponseMissingPrice, + emptyPurchaseHistoryList, + emptyPurchaseHistoryNonJSON, + firstPurchase, +} from "@ai-scraper-test/mocks/index.js"; + +class Mocks { + public timeUtils = new TimeUtils(); + public logUtils = new MockLogUtils(); + public factory() { + return new LLMPurchaseHistoryUtilsChatGPT(this.timeUtils, this.logUtils); + } +} + +describe("LLMPurchaseHistoryUtilsChatGPT", () => { + test("Parse purchase history of twelve products with one have 0 price", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.parsePurchases( + DomainName("amazon.com"), + ELanguageCode.English, + chatGPTPurchaseHistoryResponse, + ); + + // Assert + expect(result.isOk()).toBeTruthy(); + + const purchases = result._unsafeUnwrap(); + expect(purchases.length).toBe(11); // one of the 12 has price 0 + + const gotFirst = purchases[0]; + expect(gotFirst.name).toBe(firstPurchase.name); + expect(gotFirst.brand).toBe(firstPurchase.brand); + expect(gotFirst.price).toEqual(firstPurchase.price); + expect(gotFirst.category).toBe(firstPurchase.classification); + expect(gotFirst.keywords).toEqual(firstPurchase.keywords); + expect(gotFirst.datePurchased).toBe( + mocks.timeUtils.parseToSDTimestamp(firstPurchase.date), + ); + }); + + test("empty purchase history No JSON", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.parsePurchases( + DomainName("amazon.com"), + ELanguageCode.English, + emptyPurchaseHistoryNonJSON, + ); + + // Assert + expect(result.isErr()).toBeTruthy(); + }); + + test("empty purchase history", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.parsePurchases( + DomainName("amazon.com"), + ELanguageCode.English, + emptyPurchaseHistoryList, + ); + + // Assert + expect(result.isOk()).toBeTruthy(); + const purchases = result._unsafeUnwrap(); + expect(purchases.length).toBe(0); + }); + test("Parse purchase history VANMASS missing date and Purrfectzone price 0", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.parsePurchases( + DomainName("amazon.com"), + ELanguageCode.English, + chatGPTPurchaseHistoryResponseFirstMissingDate, + ); + + // Assert + expect(result.isOk()).toBeTruthy(); + + const purchases = result._unsafeUnwrap(); + expect(purchases.length).toBe(10); + + const gotFirst = purchases[0]; + expect(gotFirst.name).not.toBe(firstPurchase.name); + }); + + test("Parse purchase history VANMASS missing price and Purrfectzone price 0", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.parsePurchases( + DomainName("amazon.com"), + ELanguageCode.English, + chatGPTPurchaseHistoryResponseMissingPrice, + ); + + // Assert + expect(result.isOk()).toBeTruthy(); + + const purchases = result._unsafeUnwrap(); + expect(purchases.length).toBe(10); + const gotFirst = purchases[0]; + expect(gotFirst.name).not.toBe(firstPurchase.name); + }); +}); diff --git a/packages/ai-scraper/test/unit/business/utils/OpenAIUtils.test.ts b/packages/ai-scraper/test/unit/business/utils/OpenAIUtils.test.ts new file mode 100644 index 0000000000..d9645407a9 --- /dev/null +++ b/packages/ai-scraper/test/unit/business/utils/OpenAIUtils.test.ts @@ -0,0 +1,54 @@ +import "reflect-metadata"; +import OpenAI from "openai"; +import { + ChatCompletionCreateParamsNonStreaming, + ChatCompletionMessageParam, + CompletionCreateParamsNonStreaming, + CreateChatCompletionRequestMessage, +} from "openai/resources/chat/completions"; + +import { OpenAIUtils } from "@ai-scraper/implementations"; +import { scraperConfig } from "@ai-scraper-test/mocks/index.js"; + +/*** + * We will make actual api calls to OpenAI in this test. + */ + +function makeClient() { + const clientOptions = { + apiKey: scraperConfig.scraper.OPENAI_API_KEY, + timeout: scraperConfig.scraper.timeout, + }; + return new OpenAI(clientOptions); +} + +describe("OpenAIUtils", () => { + test("createChatCompletionNonStreaming", async () => { + // Arrange + const utils = new OpenAIUtils(); + const client = makeClient(); + + const message: ChatCompletionMessageParam[] = [ + { role: "system", content: "You are an helpful assistant." }, + { + role: "user", + content: + "How many months there are in an year? Respond with a number only.", + }, + ]; + const params: ChatCompletionCreateParamsNonStreaming = { + model: "gpt-3.5-turbo", + messages: message, + temperature: 0.1, + }; + + // Act + + const completion = await utils.getLLMResponseNonStreaming(client, params); + + // Assert + expect(completion.isOk()).toBe(true); + const response = completion._unsafeUnwrap(); + expect(response).toBe("12"); + }); +}); diff --git a/packages/ai-scraper/test/unit/business/utils/ProductMetaPromptBuilder.test.ts b/packages/ai-scraper/test/unit/business/utils/ProductMetaPromptBuilder.test.ts new file mode 100644 index 0000000000..e56d2dbca0 --- /dev/null +++ b/packages/ai-scraper/test/unit/business/utils/ProductMetaPromptBuilder.test.ts @@ -0,0 +1,153 @@ +import "reflect-metadata"; +import { + Exemplar, + LLMAnswerStructure, + LLMData, + LLMError, + LLMQuestion, + LLMRole, + Prompt, +} from "@snickerdoodlelabs/objects"; +import { Result, ResultAsync } from "neverthrow"; + +import { PurchaseHistoryPromptBuilder } from "@ai-scraper/implementations"; +import { ProductMetaPromptBuilder } from "@ai-scraper/implementations/business/utils/ProductMetaPromptBuilder"; +import { Exemplars } from "@ai-scraper-test/mocks"; + +class Mocks { + public factory(): ProductMetaPromptBuilder { + return new ProductMetaPromptBuilder(); + } +} + +function getTestPrompt( + exemplars: Exemplar[] | null, + question: LLMQuestion | null, + answerStructure: LLMAnswerStructure | null, + data: LLMData | null, + role: LLMRole | null, +): ResultAsync { + // Arrange + const mocks = new Mocks(); + const builder = mocks.factory(); + + // Act + if (exemplars != null) builder.setExemplars(Exemplars); + + if (question != null) builder.setQuestion(question); + if (answerStructure != null) builder.setAnswerStructure(answerStructure); + if (data != null) builder.setData(data); + if (role != null) builder.setRole(role); + return builder.getPrompt(); +} + +describe("ProductMetaPromptBuilder", () => { + test("no exemplars in prompt", async () => { + // Arrange + const question = LLMQuestion("What is 10 + 20?"); + const answerStructure = LLMAnswerStructure("json"); + const data = LLMData("na"); + const role = LLMRole("assistant"); + + // Act + const result = await getTestPrompt( + Exemplars, + question, + answerStructure, + data, + role, + ); + + // Assert + expect(result.isOk()).toBe(true); + const prompt = result._unsafeUnwrap(); + expect(prompt.includes(question)).toBe(true); + expect(prompt.includes(Exemplars[0])).toBe(false); + }); + + test("must have question", async () => { + // Arrange + const question = null; + const answerStructure = LLMAnswerStructure("json"); + const data = LLMData("na"); + const role = LLMRole("assistant"); + + // Act + const result = await getTestPrompt( + Exemplars, + question, + answerStructure, + data, + role, + ); + + // Assert + expect(result.isErr()).toBe(true); + const error = result._unsafeUnwrapErr(); + expect(error.message).toContain("question"); + }); + + test("must have answer structure", async () => { + // Arrange + const question = LLMQuestion("What is 10 + 20?"); + const answerStructure = null; + const data = LLMData("na"); + const role = LLMRole("assistant"); + + // Act + const result = await getTestPrompt( + Exemplars, + question, + answerStructure, + data, + role, + ); + + // Assert + expect(result.isErr()).toBe(true); + const error = result._unsafeUnwrapErr(); + expect(error.message).toContain("answerStructure"); + }); + + test("must have data", async () => { + // Arrange + const question = LLMQuestion("What is 10 + 20?"); + const answerStructure = LLMAnswerStructure("json"); + const data = null; + const role = LLMRole("assistant"); + + // Act + const result = await getTestPrompt( + Exemplars, + question, + answerStructure, + data, + role, + ); + + // Assert + expect(result.isErr()).toBe(true); + const error = result._unsafeUnwrapErr(); + expect(error.message).toContain("data"); + }); + + test("role is optional", async () => { + // Arrange + const question = LLMQuestion("What is 10 + 20?"); + const answerStructure = LLMAnswerStructure("json"); + const data = LLMData("na"); + const role = null; + + // Act + const result = await getTestPrompt( + Exemplars, + question, + answerStructure, + data, + role, + ); + + // Assert + expect(result.isOk()).toBe(true); + }); +}); diff --git a/packages/ai-scraper/test/unit/business/utils/PurchaseHistoryPromptBuilder.test.ts b/packages/ai-scraper/test/unit/business/utils/PurchaseHistoryPromptBuilder.test.ts new file mode 100644 index 0000000000..54343dadec --- /dev/null +++ b/packages/ai-scraper/test/unit/business/utils/PurchaseHistoryPromptBuilder.test.ts @@ -0,0 +1,152 @@ +import "reflect-metadata"; +import { + Exemplar, + LLMAnswerStructure, + LLMData, + LLMError, + LLMQuestion, + LLMRole, + Prompt, +} from "@snickerdoodlelabs/objects"; +import { Result, ResultAsync } from "neverthrow"; + +import { PurchaseHistoryPromptBuilder } from "@ai-scraper/implementations"; +import { Exemplars } from "@ai-scraper-test/mocks"; + +class PHPBuilderMocks { + public factory(): PurchaseHistoryPromptBuilder { + return new PurchaseHistoryPromptBuilder(); + } +} + +function getTestPrompt( + exemplars: Exemplar[] | null, + question: LLMQuestion | null, + answerStructure: LLMAnswerStructure | null, + data: LLMData | null, + role: LLMRole | null, +): ResultAsync { + // Arrange + const mocks = new PHPBuilderMocks(); + const builder = mocks.factory(); + + // Act + if (exemplars != null) builder.setExemplars(Exemplars); + + if (question != null) builder.setQuestion(question); + if (answerStructure != null) builder.setAnswerStructure(answerStructure); + if (data != null) builder.setData(data); + if (role != null) builder.setRole(role); + return builder.getPrompt(); +} + +describe("PurchaseHistoryPromptBuilder", () => { + test("no exemplars in prompt", async () => { + // Arrange + const question = LLMQuestion("What is 10 + 20?"); + const answerStructure = LLMAnswerStructure("json"); + const data = LLMData("na"); + const role = LLMRole("assistant"); + + // Act + const result = await getTestPrompt( + Exemplars, + question, + answerStructure, + data, + role, + ); + + // Assert + expect(result.isOk()).toBe(true); + const prompt = result._unsafeUnwrap(); + expect(prompt.includes(question)).toBe(true); + expect(prompt.includes(Exemplars[0])).toBe(false); + }); + + test("must have question", async () => { + // Arrange + const question = null; + const answerStructure = LLMAnswerStructure("json"); + const data = LLMData("na"); + const role = LLMRole("assistant"); + + // Act + const result = await getTestPrompt( + Exemplars, + question, + answerStructure, + data, + role, + ); + + // Assert + expect(result.isErr()).toBe(true); + const error = result._unsafeUnwrapErr(); + expect(error.message).toContain("question"); + }); + + test("must have answer structure", async () => { + // Arrange + const question = LLMQuestion("What is 10 + 20?"); + const answerStructure = null; + const data = LLMData("na"); + const role = LLMRole("assistant"); + + // Act + const result = await getTestPrompt( + Exemplars, + question, + answerStructure, + data, + role, + ); + + // Assert + expect(result.isErr()).toBe(true); + const error = result._unsafeUnwrapErr(); + expect(error.message).toContain("answerStructure"); + }); + + test("must have data", async () => { + // Arrange + const question = LLMQuestion("What is 10 + 20?"); + const answerStructure = LLMAnswerStructure("json"); + const data = null; + const role = LLMRole("assistant"); + + // Act + const result = await getTestPrompt( + Exemplars, + question, + answerStructure, + data, + role, + ); + + // Assert + expect(result.isErr()).toBe(true); + const error = result._unsafeUnwrapErr(); + expect(error.message).toContain("data"); + }); + + test("role is optional", async () => { + // Arrange + const question = LLMQuestion("What is 10 + 20?"); + const answerStructure = LLMAnswerStructure("json"); + const data = LLMData("na"); + const role = null; + + // Act + const result = await getTestPrompt( + Exemplars, + question, + answerStructure, + data, + role, + ); + + // Assert + expect(result.isOk()).toBe(true); + }); +}); diff --git a/packages/ai-scraper/test/unit/data/ChatGPTRepository.test.ts b/packages/ai-scraper/test/unit/data/ChatGPTRepository.test.ts new file mode 100644 index 0000000000..7924a36f77 --- /dev/null +++ b/packages/ai-scraper/test/unit/data/ChatGPTRepository.test.ts @@ -0,0 +1,104 @@ +import "reflect-metadata"; + +import { MockLogUtils, TimeUtils } from "@snickerdoodlelabs/common-utils"; +import { LLMError, LLMResponse, Prompt } from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +import { + ChatGPTRepository, + LLMProductMetaUtilsChatGPT, + LLMPurchaseHistoryUtilsChatGPT, + OpenAIUtils, + PromptDirector, +} from "@ai-scraper/implementations"; +import { PromptBuilderFactory } from "@ai-scraper/implementations/business/utils/PromptBuilderFactory"; +import { + MockOpenAIUtils, + chatCompletion, + chatGPTPurchaseHistoryResponse, + purchaseHistoryData, +} from "@ai-scraper-test/mocks/index.js"; +import { MockScraperConfigProvider } from "@ai-scraper-test/mocks/MockScraperConfigProvider"; + +class Mocks { + public logUtils = new MockLogUtils(); + public configProvider = new MockScraperConfigProvider(); + public mockOpenAiUtils = new MockOpenAIUtils(); // cannot use mock for all the tests as response structure is not strictconst openAIUtils = new OpenAIUtils(); + + public openAIUtils = new OpenAIUtils(); + public timeUtils = new TimeUtils(); + public purchaseHistoryLLMUtils = new LLMPurchaseHistoryUtilsChatGPT( + this.timeUtils, + this.logUtils, + ); + public productMetaLLMUtils = new LLMProductMetaUtilsChatGPT( + this.timeUtils, + this.logUtils, + ); + public promptBuilderFactory = new PromptBuilderFactory(); + public promptDirector = new PromptDirector( + this.promptBuilderFactory, + this.purchaseHistoryLLMUtils, + this.productMetaLLMUtils, + ); + + public factoryWithMockedClient(): ChatGPTRepository { + return new ChatGPTRepository( + this.configProvider, + this.logUtils, + this.mockOpenAiUtils, + ); + } + public factoryWithRealClient(): ChatGPTRepository { + return new ChatGPTRepository( + this.configProvider, + this.logUtils, + this.openAIUtils, + ); + } +} + +describe("ChatGPTRepository with Mock OpenAI api", () => { + test("executePrompt: Hello world", async () => { + // Arrange + const mocks = new Mocks(); + const provider = mocks.factoryWithMockedClient(); + const prompt = Prompt("Hello world"); + // Act + + const result = await provider.executePrompt(prompt); + + // Assert + expect(result.isOk()).toBe(true); + const response = result._unsafeUnwrap(); + expect(response).toBe( + LLMResponse(chatCompletion.choices[0].message.content!), + ); + }); +}); + +describe("ChatGPTRepository with Real OpenAI api", () => { + test.skip( + "executePrompt: ", + async () => { + // Arrange + const mocks = new Mocks(); + const provider = mocks.factoryWithRealClient(); + const promptDirector = mocks.promptDirector; + + // Act + const promptRes = await promptDirector.makePurchaseHistoryPrompt( + purchaseHistoryData, + ); + const result = await provider.executePrompt(promptRes._unsafeUnwrap()); + + // Assert + expect(result.isOk()).toBe(true); + const response = result._unsafeUnwrap(); + expect(response.length).toBeGreaterThan( + chatGPTPurchaseHistoryResponse.length / 2, + ); + }, + 60 * 1000, // 1 minute, but chat gpt can take as much as 20 minutes + ); +}); diff --git a/packages/ai-scraper/test/unit/utils/LLMPurchaseValidator.test.ts b/packages/ai-scraper/test/unit/utils/LLMPurchaseValidator.test.ts new file mode 100644 index 0000000000..4c48bae0a5 --- /dev/null +++ b/packages/ai-scraper/test/unit/utils/LLMPurchaseValidator.test.ts @@ -0,0 +1,109 @@ +import "reflect-metadata"; + +import { LLMResponse, Prompt } from "@snickerdoodlelabs/objects"; + +import { LLMPurchaseValidator } from "@ai-scraper/implementations/utils/LLMPurchaseValidator.js"; +import { + firstPurchase, + halucinatedPurchase, + nonHalucinatedPurchase, + purchaseHistoryData, + purchaseHistoryPromptWithNoPurchases, +} from "@ai-scraper-test/mocks"; + +async function malformedJSONTester(llmResponse, expected) { + // Arrange + const validator = new LLMPurchaseValidator(); + const llmResponseObj = LLMResponse(llmResponse); + + // Act + const res = await validator.fixMalformedJSONArrayResponse(llmResponseObj); + const got = res._unsafeUnwrap(); + + // Assert + expect(got).toEqual(expected); +} + +describe("LLMPurchaseValidator: trimHalucinatedPurchases", () => { + test("all fake in an empty history", async () => { + // Arrange + const validator = new LLMPurchaseValidator(); + const halucinatedPurchases = [nonHalucinatedPurchase, halucinatedPurchase]; + // Act + const validPurchases = await validator.trimHalucinatedPurchases( + purchaseHistoryPromptWithNoPurchases, + halucinatedPurchases, + ); + + // Assert + + expect(validPurchases.isOk()).toBeTruthy(); + expect(validPurchases._unsafeUnwrap()).toEqual([]); + }); + test("one fake in two purchases", async () => { + // Arrange + const validator = new LLMPurchaseValidator(); + const halucinatedPurchases = [nonHalucinatedPurchase, halucinatedPurchase]; + // Act + const validPurchases = await validator.trimHalucinatedPurchases( + Prompt(purchaseHistoryData as string), + halucinatedPurchases, + ); + + // Assert + + expect(validPurchases.isOk()).toBeTruthy(); + const got = validPurchases._unsafeUnwrap(); + expect(got.length).toBe(1); + expect(got[0].name).toEqual(nonHalucinatedPurchase.name); + }); +}); + + +// const llmResponses = [ +// "[]", +// 'The quick [brown [fox] jumps over the] lazy dog. It barked.', +// '[\n {\n "product_id": 1,\n "category": "Beauty & Health",\n "keywords": []\n },\n {\n "product_id": 2,\n ,\n "category": "Outdoors",\n "keywords": ["A", "B"]\n }, \n]', +// 'Bad bad[\n {\n "product_id": 1,\n "category": "Beauty & Health",\n "keywords": []\n },\n {\n "product_id": 2,\n ,\n "category": "Outdoors",\n "keywords": ["A", "B"]\n }, \n] \n This is a explanation', +// "I have not JSON", +// '{a: "I have a doc"}' +// ] + + + +describe("LLMPurchaseValidator: fixMalformedJSONArrayResponse", () => { + test("valid empty array", async () => { + await malformedJSONTester('[]', '[]'); + }); + + test("nested brackets", async () => { + // Arrange + const validator = new LLMPurchaseValidator(); + const llmResponse = LLMResponse('The quick [brown [fox] jumps over the] lazy dog. It barked.'); + const expected = LLMResponse('[brown [fox] jumps over the]'); + + // Test + await malformedJSONTester(llmResponse, expected); + }); + + test("missing brackets", async () => { + // Arrange + const validator = new LLMPurchaseValidator(); + const llmResponse = LLMResponse('{a: "I have a doc"}, {a: "I have a doc"}, and I have bad JSON'); + const expected = LLMResponse('[{a: "I have a doc"}, {a: "I have a doc"}]'); + + // Test + await malformedJSONTester(llmResponse, expected); + }); + + test("array with extra characters", async () => { + // Arrange + const validator = new LLMPurchaseValidator(); + const llmResponse = LLMResponse('`[\n {\n "product_id": 1,\n "category": "Beauty & Health",\n "keywords": []\n },\n {\n "product_id": 2,\n ,\n "category": "Outdoors",\n "keywords": ["A", "B"]\n }, \n] \n This is a explanation'); + const expected = LLMResponse('[\n {\n "product_id": 1,\n "category": "Beauty & Health",\n "keywords": []\n },\n {\n "product_id": 2,\n ,\n "category": "Outdoors",\n "keywords": ["A", "B"]\n }, \n]'); + + // Test + await malformedJSONTester(llmResponse, expected); + }); +}); + diff --git a/packages/ai-scraper/test/unit/utils/URLUtils.test.ts b/packages/ai-scraper/test/unit/utils/URLUtils.test.ts index 34eca7ec72..77200a3ad9 100644 --- a/packages/ai-scraper/test/unit/utils/URLUtils.test.ts +++ b/packages/ai-scraper/test/unit/utils/URLUtils.test.ts @@ -1,8 +1,11 @@ import "reflect-metadata"; -import { ELanguageCode } from "@snickerdoodlelabs/objects"; +import { + EKnownDomains, + ELanguageCode, + ETask, +} from "@snickerdoodlelabs/objects"; import { KeywordUtils, URLUtils } from "@ai-scraper/implementations"; -import { EKnownDomains, ETask } from "@ai-scraper/interfaces"; import { AMAZON_HOST_NAME, AMAZON_URL, diff --git a/packages/ai-scraper/test/unit/utils/WebpageClassifier.test.ts b/packages/ai-scraper/test/unit/utils/WebpageClassifier.test.ts index 3900e19849..e2e5ce1264 100644 --- a/packages/ai-scraper/test/unit/utils/WebpageClassifier.test.ts +++ b/packages/ai-scraper/test/unit/utils/WebpageClassifier.test.ts @@ -1,12 +1,15 @@ import "reflect-metadata"; -import { ELanguageCode } from "@snickerdoodlelabs/objects"; +import { + EKnownDomains, + ELanguageCode, + ETask, +} from "@snickerdoodlelabs/objects"; import { KeywordUtils, URLUtils, WebpageClassifier, } from "@ai-scraper/implementations/index.js"; -import { EKnownDomains, ETask } from "@ai-scraper/interfaces"; import { AMAZON_URL } from "@ai-scraper-test/mocks"; import { MockKeywordRepository } from "@ai-scraper-test/mocks/MockKeywordRepository.js"; diff --git a/packages/ai-scraper/tsconfig.json b/packages/ai-scraper/tsconfig.json index 0366f7600d..a2b008dc74 100644 --- a/packages/ai-scraper/tsconfig.json +++ b/packages/ai-scraper/tsconfig.json @@ -22,6 +22,12 @@ }, { "path": "../common-utils" + }, + { + "path": "../persistence" + }, + { + "path": "../shopping-data" } ] } \ No newline at end of file diff --git a/packages/browserExtension/src/background/configs.ts b/packages/browserExtension/src/background/configs.ts index e57d56fb77..05cdfa5d14 100644 --- a/packages/browserExtension/src/background/configs.ts +++ b/packages/browserExtension/src/background/configs.ts @@ -70,8 +70,30 @@ declare const __DROPBOX_APP_KEY__: string; declare const __DROPBOX_APP_SECRET__: string; declare const __DROPBOX_REDIRECT_URI__: string; +declare const __OPEN_API_KEY__: string; +declare const __SCRAPER_TIMEOUT__: string; + const ONE_MINUTE_MS = 60000; +const _buildScraperConfig = (): { + OPENAI_API_KEY: string; + timeout: number; +} => { + const scraperConfig = { + OPENAI_API_KEY: "", + timeout: 300000, + }; + + if (typeof __OPEN_API_KEY__ !== "undefined" && !!__OPEN_API_KEY__) { + scraperConfig["OPENAI_API_KEY"] = __OPEN_API_KEY__; + } + if (typeof __SCRAPER_TIMEOUT__ !== "undefined" && !!__SCRAPER_TIMEOUT__) { + scraperConfig["timeout"] = Number.parseInt(__SCRAPER_TIMEOUT__); + } + + return scraperConfig; +}; + const _buildDiscordConfig = (): Partial => { const oauthRedirectUrl = typeof __ONBOARDING_URL__ !== "undefined" && !!__ONBOARDING_URL__ @@ -362,6 +384,7 @@ export const config: IExtensionSdkConfigOverrides = { : false, discordOverrides: _buildDiscordConfig(), twitterOverrides: _buildTwitterConfig(), + scraper: _buildScraperConfig(), devChainProviderURL: typeof __DEV_CHAIN_PROVIDER_URL__ !== "undefined" && diff --git a/packages/browserExtension/src/popup/pages/Home/components/LinkCard/LinkCard.tsx b/packages/browserExtension/src/popup/pages/Home/components/LinkCard/LinkCard.tsx index cec2dd5d6f..348ebd57c4 100644 --- a/packages/browserExtension/src/popup/pages/Home/components/LinkCard/LinkCard.tsx +++ b/packages/browserExtension/src/popup/pages/Home/components/LinkCard/LinkCard.tsx @@ -1,8 +1,9 @@ -import { useAppContext } from "@browser-extension/popup/context"; -import { useStyles } from "@browser-extension/popup/pages/Home/components/LinkCard/LinkCard.style"; import { Box, Typography } from "@material-ui/core"; import React from "react"; +import { useAppContext } from "@browser-extension/popup/context"; +import { useStyles } from "@browser-extension/popup/pages/Home/components/LinkCard/LinkCard.style"; + interface ILinkCardProps { navigateTo: string; icon: string; diff --git a/packages/browserExtension/utils/envVars/demo-01-env.cjs b/packages/browserExtension/utils/envVars/demo-01-env.cjs index 47e95ee805..bde4754cd4 100644 --- a/packages/browserExtension/utils/envVars/demo-01-env.cjs +++ b/packages/browserExtension/utils/envVars/demo-01-env.cjs @@ -60,6 +60,8 @@ const envVars = { __DNS_SERVER_ADDRESS__: "", __GOOGLE_CLOUD_BUCKET__: "demo-01-fcszy-sdl-dw", __DEV_CHAIN_PROVIDER_URL__: "https://doodlechain.demo-01.snickerdoodle.dev", + __OPEN_API_KEY__: "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", + __SCRAPER_TIMEOUT__: 300000, }; for (const key in envVars) { diff --git a/packages/browserExtension/utils/envVars/demo-02-env.cjs b/packages/browserExtension/utils/envVars/demo-02-env.cjs index ce3aedd2a6..b41653b034 100644 --- a/packages/browserExtension/utils/envVars/demo-02-env.cjs +++ b/packages/browserExtension/utils/envVars/demo-02-env.cjs @@ -61,6 +61,8 @@ const envVars = { __DNS_SERVER_ADDRESS__: "", __GOOGLE_CLOUD_BUCKET__: "demo-02-hrbbt-sdl-dw", __DEV_CHAIN_PROVIDER_URL__: "https://doodlechain.demo-02.snickerdoodle.dev", + __OPEN_API_KEY__: "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", + __SCRAPER_TIMEOUT__: 300000, }; for (const key in envVars) { diff --git a/packages/browserExtension/utils/envVars/demo-03-env.cjs b/packages/browserExtension/utils/envVars/demo-03-env.cjs index 734c0af340..c0d1c743b1 100644 --- a/packages/browserExtension/utils/envVars/demo-03-env.cjs +++ b/packages/browserExtension/utils/envVars/demo-03-env.cjs @@ -61,6 +61,8 @@ const envVars = { __GOOGLE_CLOUD_BUCKET__: "demo-03-vufbw-sdl-dw", __BACKUP_POLLING_INTERVAL__: "", __DEV_CHAIN_PROVIDER_URL__: "https://doodlechain.demo-03.snickerdoodle.dev", + __OPEN_API_KEY__: "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", + __SCRAPER_TIMEOUT__: 300000, }; for (const key in envVars) { diff --git a/packages/browserExtension/utils/envVars/demo-04-env.cjs b/packages/browserExtension/utils/envVars/demo-04-env.cjs index 6d7596a8eb..605eb77043 100644 --- a/packages/browserExtension/utils/envVars/demo-04-env.cjs +++ b/packages/browserExtension/utils/envVars/demo-04-env.cjs @@ -61,6 +61,8 @@ const envVars = { __DNS_SERVER_ADDRESS__: "", __GOOGLE_CLOUD_BUCKET__: "demo-04-xpcqe-sdl-dw", __DEV_CHAIN_PROVIDER_URL__: "https://doodlechain.demo-04.snickerdoodle.dev", + __OPEN_API_KEY__: "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", + __SCRAPER_TIMEOUT__: 300000, }; for (const key in envVars) { diff --git a/packages/browserExtension/utils/envVars/demo-05-env.cjs b/packages/browserExtension/utils/envVars/demo-05-env.cjs index a6a7db9c95..bd16055a28 100644 --- a/packages/browserExtension/utils/envVars/demo-05-env.cjs +++ b/packages/browserExtension/utils/envVars/demo-05-env.cjs @@ -62,6 +62,8 @@ const envVars = { __DNS_SERVER_ADDRESS__: "", __GOOGLE_CLOUD_BUCKET__: "demo-05-eksbc-sdl-dw", __DEV_CHAIN_PROVIDER_URL__: "https://doodlechain.demo-05.snickerdoodle.dev", + __OPEN_API_KEY__: "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", + __SCRAPER_TIMEOUT__: 300000, }; for (const key in envVars) { diff --git a/packages/browserExtension/utils/envVars/dev-env.cjs b/packages/browserExtension/utils/envVars/dev-env.cjs index 4bed89d845..b5f534441b 100644 --- a/packages/browserExtension/utils/envVars/dev-env.cjs +++ b/packages/browserExtension/utils/envVars/dev-env.cjs @@ -61,6 +61,8 @@ const envVars = { __DNS_SERVER_ADDRESS__: "", __GOOGLE_CLOUD_BUCKET__: "dev-zitrz-sdl-dw", __DEV_CHAIN_PROVIDER_URL__: "https://doodlechain.dev.snickerdoodle.dev", + __OPEN_API_KEY__: "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", + __SCRAPER_TIMEOUT__: 300000, }; for (const key in envVars) { diff --git a/packages/browserExtension/utils/envVars/dev-webpack-env.cjs b/packages/browserExtension/utils/envVars/dev-webpack-env.cjs index 54542c386f..f527c0d16f 100644 --- a/packages/browserExtension/utils/envVars/dev-webpack-env.cjs +++ b/packages/browserExtension/utils/envVars/dev-webpack-env.cjs @@ -61,6 +61,8 @@ const envVars = { __DNS_SERVER_ADDRESS__: "", __GOOGLE_CLOUD_BUCKET__: "dev-zitrz-sdl-dw", __DEV_CHAIN_PROVIDER_URL__: "https://doodlechain.dev.snickerdoodle.dev", + __OPEN_API_KEY__: "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", + __SCRAPER_TIMEOUT__: 300000, }; for (const key in envVars) { diff --git a/packages/browserExtension/utils/envVars/local-env.cjs b/packages/browserExtension/utils/envVars/local-env.cjs index cde0b4ff1b..a5563861ac 100644 --- a/packages/browserExtension/utils/envVars/local-env.cjs +++ b/packages/browserExtension/utils/envVars/local-env.cjs @@ -49,6 +49,8 @@ const envVars = { __TRANSACTION_POLLING_INTERVAL__: "", __BACKUP_POLLING_INTERVAL__: "", __DEV_CHAIN_PROVIDER_URL__: "http://127.0.0.1:8545", + __OPEN_API_KEY__: "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", + __SCRAPER_TIMEOUT__: 300000, }; for (const key in envVars) { diff --git a/packages/browserExtension/utils/envVars/prod-env.cjs b/packages/browserExtension/utils/envVars/prod-env.cjs index f83955790e..feaf2fc705 100644 --- a/packages/browserExtension/utils/envVars/prod-env.cjs +++ b/packages/browserExtension/utils/envVars/prod-env.cjs @@ -60,6 +60,8 @@ const envVars = { __DNS_SERVER_ADDRESS__: "", __GOOGLE_CLOUD_BUCKET__: "prod-qkppf-sdl-dw", __DEV_CHAIN_PROVIDER_URL__: "", + __OPEN_API_KEY__: "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", + __SCRAPER_TIMEOUT__: 300000, }; for (const key in envVars) { diff --git a/packages/browserExtension/utils/webserver.cjs b/packages/browserExtension/utils/webserver.cjs index 7d1d5661bc..549eac42cc 100644 --- a/packages/browserExtension/utils/webserver.cjs +++ b/packages/browserExtension/utils/webserver.cjs @@ -63,6 +63,9 @@ process.env.__TWITTER_CONSUMER_SECRET__ = "y4FOFgQnuRo7vvnRuKqFhBbM3sYWuSZyg5RqHlRIc3DZ4N7Hnx"; process.env.__TWITTER_POLL_INTERVAL__ = "86400000"; +process.env.__OPEN_API_KEY__ = ""; +process.env.__SCRAPER_TIMEOUT__ = ""; + var WebpackDevServer = require("webpack-dev-server"), webpack = require("webpack"), config = require("../webpack.config.cjs"), @@ -70,6 +73,16 @@ var WebpackDevServer = require("webpack-dev-server"), path = require("path"); + + + + + + + + + + config.plugins = [new webpack.HotModuleReplacementPlugin()].concat( config.plugins || [], ); diff --git a/packages/browserExtension/webpack.config.cjs b/packages/browserExtension/webpack.config.cjs index 1ffd8794aa..f4e524a062 100644 --- a/packages/browserExtension/webpack.config.cjs +++ b/packages/browserExtension/webpack.config.cjs @@ -274,6 +274,10 @@ var options = { __DEV_CHAIN_PROVIDER_URL__: JSON.stringify( process.env.__DEV_CHAIN_PROVIDER_URL__, ), + /* SCRAPER KEY */ + __OPEN_API_KEY__: JSON.stringify(process.env.__OPEN_API_KEY__), + __SCRAPER_TIMEOUT__: JSON.stringify(process.env.__SCRAPER_TIMEOUT__), + /* */ }), new CopyWebpackPlugin({ diff --git a/packages/common-utils/src/implementations/MockLogUtils.ts b/packages/common-utils/src/implementations/MockLogUtils.ts new file mode 100644 index 0000000000..f9206c45cc --- /dev/null +++ b/packages/common-utils/src/implementations/MockLogUtils.ts @@ -0,0 +1,28 @@ +import { ILogUtils } from "@common-utils/interfaces/index.js"; +import { pino } from "pino"; + +export class MockLogUtils implements ILogUtils { + debug(message?: string, ...optionalParams: unknown[]): void { + console.log(message); + console.log(optionalParams); + } + info(message?: string, ...optionalParams: unknown[]): void { + console.log(message); + console.log(optionalParams); + } + log(message?: string, ...optionalParams: unknown[]): void { + console.log(message); + console.log(optionalParams); + } + warning(message?: string, ...optionalParams: unknown[]): void { + console.log(message); + console.log(optionalParams); + } + error(message?: string, ...optionalParams: unknown[]): void { + console.log(message); + console.log(optionalParams); + } + getPino(): pino.Logger { + throw new Error("Method not implemented."); + } +} diff --git a/packages/common-utils/src/implementations/ObjectUtils.ts b/packages/common-utils/src/implementations/ObjectUtils.ts index 41cb36b55c..1fb3c1f407 100644 --- a/packages/common-utils/src/implementations/ObjectUtils.ts +++ b/packages/common-utils/src/implementations/ObjectUtils.ts @@ -4,6 +4,7 @@ import { PagingRequest, JSONString, InvalidParametersError, + PageNumber, BigNumberString, } from "@snickerdoodlelabs/objects"; import { errAsync, okAsync, ResultAsync } from "neverthrow"; @@ -112,7 +113,7 @@ export class ObjectUtils { serialize = false, ): ResultAsync { // Create the initial paging request - const pagingRequest = new PagingRequest(pageNumber, pageSize); + const pagingRequest = new PagingRequest(PageNumber(pageNumber), pageSize); // Read the page return readFunc(pagingRequest).andThen((objectPage) => { @@ -271,7 +272,7 @@ export class ObjectUtils { return okAsync(undefined); }; - return readFunc(new PagingRequest(1, 1)) + return readFunc(new PagingRequest(PageNumber(1), 1)) .andThen((firstPage) => { const pageSize = firstPage.totalResults; diff --git a/packages/common-utils/src/implementations/TimeUtils.ts b/packages/common-utils/src/implementations/TimeUtils.ts index abb23fe5c1..22b61ed130 100644 --- a/packages/common-utils/src/implementations/TimeUtils.ts +++ b/packages/common-utils/src/implementations/TimeUtils.ts @@ -1,7 +1,9 @@ import { ISO8601DateString, MillisecondTimestamp, + Month, UnixTimestamp, + Year, } from "@snickerdoodlelabs/objects"; import { injectable } from "inversify"; @@ -27,6 +29,30 @@ export class TimeUtils implements ITimeUtils { return ISO8601DateString(new Date(time).toISOString()); } + public parseToDate(dateStr: string): Date | null { + // we cannot create a date object directly as it creates an unknown object incase of invalid input + const time = this.parseToSDTimestamp(dateStr); + if (time == null) { + return null; + } + return new Date(time); + } + + public parseToSDTimestamp(dateStr: string): UnixTimestamp | null { + const time = Date.parse(dateStr); + if (isNaN(time)) { + return null; + } + return UnixTimestamp(Math.floor(time / 1000)); + } + + public getCurYear(): Year { + return Year(new Date().getFullYear()); + } + public getCurMonth(): Month { + return Month(new Date().getMonth()); + } + public convertTimestampToISOString( unixTimestamp: UnixTimestamp, ): ISO8601DateString { diff --git a/packages/common-utils/src/implementations/index.ts b/packages/common-utils/src/implementations/index.ts index 58818cbb76..85b7cce109 100644 --- a/packages/common-utils/src/implementations/index.ts +++ b/packages/common-utils/src/implementations/index.ts @@ -2,6 +2,7 @@ export * from "@common-utils/implementations/AxiosAjaxUtils.js"; export * from "@common-utils/implementations/BigNumberUtils.js"; export * from "@common-utils/implementations/JsonUtils.js"; export * from "@common-utils/implementations/LogUtils.js"; +export * from "@common-utils/implementations/MockLogUtils.js"; export * from "@common-utils/implementations/NftMetadataParseUtils.js"; export * from "@common-utils/implementations/ObjectUtils.js"; export * from "@common-utils/implementations/TimeUtils.js"; diff --git a/packages/common-utils/src/interfaces/ITimeUtils.ts b/packages/common-utils/src/interfaces/ITimeUtils.ts index 93790d1e8b..472a8b871c 100644 --- a/packages/common-utils/src/interfaces/ITimeUtils.ts +++ b/packages/common-utils/src/interfaces/ITimeUtils.ts @@ -1,13 +1,20 @@ import { ISO8601DateString, MillisecondTimestamp, + Month, UnixTimestamp, + Year, } from "@snickerdoodlelabs/objects"; export interface ITimeUtils { getUnixNow(): UnixTimestamp; getMillisecondNow(): MillisecondTimestamp; getISO8601TimeString(time: MillisecondTimestamp): string; + parseToDate(dateStr: string): Date | null; + parseToSDTimestamp(dateStr: string): UnixTimestamp | null; + + getCurYear(): Year; + getCurMonth(): Month; convertTimestampToISOString(unixTimestamp: UnixTimestamp): ISO8601DateString; getUnixTodayStart(): UnixTimestamp; getUnixTodayEnd(): UnixTimestamp; diff --git a/packages/common-utils/test/unit/ObjectUtils.test.ts b/packages/common-utils/test/unit/ObjectUtils.test.ts index ac2517e134..239aabebba 100644 --- a/packages/common-utils/test/unit/ObjectUtils.test.ts +++ b/packages/common-utils/test/unit/ObjectUtils.test.ts @@ -4,6 +4,7 @@ import { PagedResponse, PagingRequest, BigNumberString, + PageNumber, } from "@snickerdoodlelabs/objects"; import { errAsync, okAsync, ResultAsync } from "neverthrow"; @@ -138,13 +139,13 @@ describe("ObjectUtils tests", () => { const readFunc = (pagingRequest: PagingRequest) => { if (pagingRequest.page == 1) { - return okAsync(new PagedResponse([1, 2, 3], 1, 3, 9)); + return okAsync(new PagedResponse([1, 2, 3], PageNumber(1), 3, 9)); } if (pagingRequest.page == 2) { - return okAsync(new PagedResponse([4, 5, 6], 2, 3, 9)); + return okAsync(new PagedResponse([4, 5, 6], PageNumber(2), 3, 9)); } if (pagingRequest.page == 3) { - return okAsync(new PagedResponse([7, 8, 9], 3, 3, 9)); + return okAsync(new PagedResponse([7, 8, 9], PageNumber(3), 3, 9)); } // If it asks for page 4 return errAsync(new Error("Asked for pages beyond totalResults!")); @@ -171,13 +172,13 @@ describe("ObjectUtils tests", () => { const readFunc = (pagingRequest: PagingRequest) => { if (pagingRequest.page == 1) { - return okAsync(new PagedResponse([1, 2, 3], 1, 3, 9)); + return okAsync(new PagedResponse([1, 2, 3], PageNumber(1), 3, 9)); } if (pagingRequest.page == 2) { return errAsync(new Error("Read failure for page 2!")); } if (pagingRequest.page == 3) { - return okAsync(new PagedResponse([7, 8, 9], 3, 3, 9)); + return okAsync(new PagedResponse([7, 8, 9], PageNumber(3), 3, 9)); } // If it asks for page 4 return errAsync(new Error("Asked for pages beyond totalResults!")); @@ -204,13 +205,13 @@ describe("ObjectUtils tests", () => { const readFunc = (pagingRequest: PagingRequest) => { if (pagingRequest.page == 1) { - return okAsync(new PagedResponse([1, 2, 3], 1, 3, 9)); + return okAsync(new PagedResponse([1, 2, 3], PageNumber(1), 3, 9)); } if (pagingRequest.page == 2) { - return okAsync(new PagedResponse([4, 5, 6], 2, 3, 9)); + return okAsync(new PagedResponse([4, 5, 6], PageNumber(2), 3, 9)); } if (pagingRequest.page == 3) { - return okAsync(new PagedResponse([7, 8, 9], 3, 3, 9)); + return okAsync(new PagedResponse([7, 8, 9], PageNumber(3), 3, 9)); } // If it asks for page 4 return errAsync(new Error("Asked for pages beyond totalResults!")); @@ -283,10 +284,10 @@ describe("ObjectUtils tests", () => { // Arrange const readFunc = (pagingRequest: PagingRequest) => { if (pagingRequest.page == 1 && pagingRequest.pageSize == 1) { - return okAsync(new PagedResponse([1], 1, 1, 9)); + return okAsync(new PagedResponse([1], PageNumber(1), 1, 9)); } if (pagingRequest.page == 1 && pagingRequest.pageSize == 9) { - return okAsync(new PagedResponse([1, 2, 3, 4, 5, 6, 7, 8, 9], 1, 9, 9)); + return okAsync(new PagedResponse([1, 2, 3, 4, 5, 6, 7, 8, 9], PageNumber(1), 9, 9)); } // If it asks for page 2 return errAsync(new Error("Asked for pages beyond totalResults!")); diff --git a/packages/core/package.json b/packages/core/package.json index 54ca3d9f9b..9d63a30e29 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -37,14 +37,17 @@ "type": "module", "types": "dist/index.d.ts", "dependencies": { + "@snickerdoodlelabs/ai-scraper": "workspace:^", "@snickerdoodlelabs/common-utils": "workspace:^", "@snickerdoodlelabs/contracts-sdk": "workspace:^", "@snickerdoodlelabs/indexers": "workspace:^", "@snickerdoodlelabs/insight-platform-api": "workspace:^", + "@snickerdoodlelabs/nlp": "workspace:^", "@snickerdoodlelabs/node-utils": "workspace:^", "@snickerdoodlelabs/objects": "workspace:^", "@snickerdoodlelabs/persistence": "workspace:^", "@snickerdoodlelabs/query-parser": "workspace:^", + "@snickerdoodlelabs/shopping-data": "workspace:^", "@snickerdoodlelabs/signature-verification": "workspace:^", "ethers": "^6.10.0", "inversify": "^6.0.2", diff --git a/packages/core/src/implementations/SnickerdoodleCore.module.ts b/packages/core/src/implementations/SnickerdoodleCore.module.ts index c78fd48e74..2b5ad8cd9e 100644 --- a/packages/core/src/implementations/SnickerdoodleCore.module.ts +++ b/packages/core/src/implementations/SnickerdoodleCore.module.ts @@ -1,3 +1,31 @@ +import { + AmazonNavigationUtils, + ChatGPTRepository, + HTMLPreProcessor, + IAmazonNavigationUtils, + IAmazonNavigationUtilsType, + IHTMLPreProcessor, + IHTMLPreProcessorType, + ILLMRepository, + ILLMRepositoryType, + ILLMPurchaseHistoryUtils, + ILLMPurchaseHistoryUtilsType, + IOpenAIUtils, + IOpenAIUtilsType, + IPromptBuilderFactory, + IPromptBuilderFactoryType, + IPromptDirector, + IPromptDirectorType, + IScraperConfigProvider, + IScraperConfigProviderType, + IScraperService, + IScraperServiceType, + LLMPurchaseHistoryUtilsChatGPT, + LLMScraperService, + OpenAIUtils, + PromptBuilderFactory, + PromptDirector, +} from "@snickerdoodlelabs/ai-scraper"; import { AxiosAjaxUtils, BigNumberUtils, @@ -13,10 +41,34 @@ import { TimeUtils, } from "@snickerdoodlelabs/common-utils"; import { + AlchemyIndexer, + AnkrIndexer, + CovalentEVMTransactionRepository, + EtherscanIndexer, + IAlchemyIndexerType, + IAnkrIndexerType, + ICovalentEVMTransactionRepositoryType, + IEVMIndexer, + IEtherscanIndexerType, IIndexerConfigProvider, IIndexerConfigProviderType, IIndexerContextProvider, IIndexerContextProviderType, + IMoralisEVMPortfolioRepositoryType, + INftScanEVMPortfolioRepositoryType, + IOklinkIndexerType, + IPoapRepositoryType, + IPolygonIndexerType, + ISimulatorEVMTransactionRepositoryType, + ISolanaIndexer, + ISolanaIndexerType, + MoralisEVMPortfolioRepository, + NftScanEVMPortfolioRepository, + OklinkIndexer, + PoapRepository, + PolygonIndexer, + SimulatorEVMTransactionRepository, + SolanaIndexer, } from "@snickerdoodlelabs/indexers"; import { IInsightPlatformRepository, @@ -80,6 +132,10 @@ import { QueryFactories, SDQLQueryWrapperFactory, } from "@snickerdoodlelabs/query-parser"; +import { + IPurchaseRepository, + IPurchaseRepositoryType, +} from "@snickerdoodlelabs/shopping-data"; import { ContainerModule, interfaces } from "inversify"; import { @@ -103,6 +159,7 @@ import { QueryService, TwitterService, CloudStorageService, + PurchaseService, CachingService, QuestionnaireService, } from "@core/implementations/business/index.js"; @@ -140,6 +197,7 @@ import { TransactionHistoryRepository, TwitterRepository, AuthenticatedStorageRepository, + PurchaseRepository, NftRepository, QuestionnaireRepository, } from "@core/implementations/data/index.js"; @@ -189,6 +247,8 @@ import { IQuestionnaireServiceType, ITwitterService, ITwitterServiceType, + IPurchaseService, + IPurchaseServiceType, } from "@core/interfaces/business/index.js"; import { IBalanceQueryEvaluator, @@ -423,7 +483,9 @@ export const snickerdoodleCoreModule = new ContainerModule( bind(ITwitterRepositoryType) .to(TwitterRepository) .inSingletonScope(); - bind(ISocialRepositoryType).to(SocialRepository); + bind(ISocialRepositoryType) + .to(SocialRepository) + .inSingletonScope(); bind(IBackupUtilsType).to(BackupUtils).inSingletonScope(); bind(IVolatileStorageSchemaProviderType) @@ -543,6 +605,22 @@ export const snickerdoodleCoreModule = new ContainerModule( .to(DropboxCloudStorage) .inSingletonScope(); + // region shopping data + bind(IPurchaseRepositoryType) + .to(PurchaseRepository) + .inSingletonScope(); + // endregion + + // region scraper + bind(IScraperConfigProviderType).toService( + IConfigProviderType, + ); + // endregion + // region purchase + bind(IPurchaseServiceType) + .to(PurchaseService) + .inSingletonScope(); + // endregion /** * Binding of Modules With Extra Capabilities. * diff --git a/packages/core/src/implementations/SnickerdoodleCore.ts b/packages/core/src/implementations/SnickerdoodleCore.ts index c863e3abfe..410b95734c 100644 --- a/packages/core/src/implementations/SnickerdoodleCore.ts +++ b/packages/core/src/implementations/SnickerdoodleCore.ts @@ -4,11 +4,19 @@ * Regardless of form factor, you need to instantiate an instance * of SnickerdoodleCore. */ +import { + IAmazonNavigationUtils, + IAmazonNavigationUtilsType, + IScraperService, + IScraperServiceType, + scraperModule, +} from "@snickerdoodlelabs/ai-scraper"; import { IMasterIndexer, IMasterIndexerType, indexersModule, } from "@snickerdoodlelabs/indexers"; +import { nlpModule } from "@snickerdoodlelabs/nlp"; import { AccountAddress, AccountIndexingError, @@ -88,8 +96,21 @@ import { BlockNumber, RefreshToken, SiteVisitsMap, + IScraperMethods, + DomainTask, + ELanguageCode, + HTMLString, + ScraperError, + IScraperNavigationMethods, + PageNumber, + Year, + IPurchaseMethods, TransactionFlowInsight, URLString, + ShoppingDataConnectionStatus, + PurchasedProduct, + InvalidURLError, + LLMError, INftMethods, IQuestionnaireMethods, NewQuestionnaireAnswer, @@ -103,6 +124,7 @@ import { ICloudStorageManager, ICloudStorageManagerType, } from "@snickerdoodlelabs/persistence"; +import { shoppingDataModule } from "@snickerdoodlelabs/shopping-data"; import { IStorageUtils, IStorageUtilsType, @@ -144,6 +166,8 @@ import { IMetricsServiceType, IProfileService, IProfileServiceType, + IPurchaseService, + IPurchaseServiceType, IQueryService, IQueryServiceType, IQuestionnaireService, @@ -187,6 +211,10 @@ export class SnickerdoodleCore implements ISnickerdoodleCore { public nft: INftMethods; public questionnaire: IQuestionnaireMethods; + public purchase: IPurchaseMethods; + public scraper: IScraperMethods; + public scraperNavigation: IScraperNavigationMethods; + public constructor( configOverrides?: IConfigOverrides, storageUtils?: IStorageUtils, @@ -195,7 +223,15 @@ export class SnickerdoodleCore implements ISnickerdoodleCore { this.iocContainer = new Container(); // Elaborate syntax to demonstrate that we can use multiple modules - this.iocContainer.load(...[snickerdoodleCoreModule, indexersModule]); + this.iocContainer.load( + ...[ + snickerdoodleCoreModule, + indexersModule, + scraperModule, + nlpModule, + shoppingDataModule, + ], + ); // If persistence is provided, we need to hook it up. If it is not, we will use the default // persistence. @@ -704,6 +740,116 @@ export class SnickerdoodleCore implements ISnickerdoodleCore { ); }, }; + // Scraper Methods --------------------------------------------------------------------------- + this.scraper = { + scrape: ( + url: URLString, + html: HTMLString, + suggestedDomainTask: DomainTask, + ): ResultAsync => { + const scraperService = + this.iocContainer.get(IScraperServiceType); + return scraperService.scrape(url, html, suggestedDomainTask); + }, + classifyURL: ( + url: URLString, + language: ELanguageCode, + ): ResultAsync => { + const scraperService = + this.iocContainer.get(IScraperServiceType); + return scraperService.classifyURL(url, language); + }, + }; + + this.purchase = { + getPurchasedProducts: (): ResultAsync< + PurchasedProduct[], + PersistenceError + > => { + const purchaseService = + this.iocContainer.get(IPurchaseServiceType); + return purchaseService.getPurchasedProducts(); + }, + getByMarketplace: ( + marketPlace: DomainName, + ): ResultAsync => { + const purchaseService = + this.iocContainer.get(IPurchaseServiceType); + return purchaseService.getByMarketplace(marketPlace); + }, + getByMarketplaceAndDate: ( + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ): ResultAsync => { + const purchaseService = + this.iocContainer.get(IPurchaseServiceType); + return purchaseService.getByMarketplaceAndDate( + marketPlace, + datePurchased, + ); + }, + getShoppingDataConnectionStatus: (): ResultAsync< + ShoppingDataConnectionStatus[], + PersistenceError + > => { + const purchaseService = + this.iocContainer.get(IPurchaseServiceType); + return purchaseService.getShoppingDataConnectionStatus(); + }, + setShoppingDataConnectionStatus: ( + ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ): ResultAsync => { + const purchaseService = + this.iocContainer.get(IPurchaseServiceType); + return purchaseService.setShoppingDataConnectionStatus( + ShoppingDataConnectionStatus, + ); + }, + }; + + this.scraperNavigation = { + amazon: { + getOrderHistoryPage: ( + lang: ELanguageCode, + page: PageNumber, + ): URLString => { + const amazonNavigationUtils = + this.iocContainer.get( + IAmazonNavigationUtilsType, + ); + return amazonNavigationUtils.getOrderHistoryPage(lang, page); + }, + getYears: (html: HTMLString): Year[] => { + const amazonNavigationUtils = + this.iocContainer.get( + IAmazonNavigationUtilsType, + ); + return amazonNavigationUtils.getYears(html); + }, + getOrderHistoryPageByYear: ( + lang: ELanguageCode, + year: Year, + page: PageNumber, + ): URLString => { + const amazonNavigationUtils = + this.iocContainer.get( + IAmazonNavigationUtilsType, + ); + return amazonNavigationUtils.getOrderHistoryPageByYear( + lang, + year, + page, + ); + }, + getPageCount: (html: HTMLString, year: Year): number => { + const amazonNavigationUtils = + this.iocContainer.get( + IAmazonNavigationUtilsType, + ); + return amazonNavigationUtils.getPageCount(html, year); + }, + }, + }; // Nft Methods --------------------------------------------------------------------------- this.nft = { getNfts: ( diff --git a/packages/core/src/implementations/business/PurchaseService.ts b/packages/core/src/implementations/business/PurchaseService.ts new file mode 100644 index 0000000000..a8144f8400 --- /dev/null +++ b/packages/core/src/implementations/business/PurchaseService.ts @@ -0,0 +1,51 @@ +import { + DomainName, + PersistenceError, + ShoppingDataConnectionStatus, + UnixTimestamp, + PurchasedProduct, +} from "@snickerdoodlelabs/objects"; +import { + IPurchaseRepository, + IPurchaseRepositoryType, +} from "@snickerdoodlelabs/shopping-data"; +import { inject, injectable } from "inversify"; +import { ResultAsync } from "neverthrow"; + +import { IPurchaseService } from "@core/interfaces/business/IPurchaseService"; +@injectable() +export class PurchaseService implements IPurchaseService { + public constructor( + @inject(IPurchaseRepositoryType) public purchaseRepo: IPurchaseRepository, + ) {} + getPurchasedProducts(): ResultAsync { + return this.purchaseRepo.getPurchasedProducts(); + } + getByMarketplace( + marketPlace: DomainName, + ): ResultAsync { + return this.purchaseRepo.getByMarketplace(marketPlace); + } + getByMarketplaceAndDate( + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ): ResultAsync { + return this.purchaseRepo.getByMarketplaceAndDate( + marketPlace, + datePurchased, + ); + } + getShoppingDataConnectionStatus(): ResultAsync< + ShoppingDataConnectionStatus[], + PersistenceError + > { + return this.purchaseRepo.getShoppingDataConnectionStatus(); + } + setShoppingDataConnectionStatus( + ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ): ResultAsync { + return this.purchaseRepo.setShoppingDataConnectionStatus( + ShoppingDataConnectionStatus, + ); + } +} diff --git a/packages/core/src/implementations/business/index.ts b/packages/core/src/implementations/business/index.ts index 92aecd10b8..63b73ef7bd 100644 --- a/packages/core/src/implementations/business/index.ts +++ b/packages/core/src/implementations/business/index.ts @@ -9,6 +9,7 @@ export * from "@core/implementations/business/MarketplaceService.js"; export * from "@core/implementations/business/MetricsService.js"; export * from "@core/implementations/business/MonitoringService.js"; export * from "@core/implementations/business/ProfileService.js"; +export * from "@core/implementations/business/PurchaseService.js"; export * from "@core/implementations/business/QueryService.js"; export * from "@core/implementations/business/QuestionnaireService.js"; export * from "@core/implementations/business/TwitterService.js"; diff --git a/packages/core/src/implementations/data/PurchaseRepository.ts b/packages/core/src/implementations/data/PurchaseRepository.ts new file mode 100644 index 0000000000..c673e6522b --- /dev/null +++ b/packages/core/src/implementations/data/PurchaseRepository.ts @@ -0,0 +1,94 @@ +import { + PersistenceError, + DomainName, + ERecordKey, + UnixTimestamp, + ShoppingDataConnectionStatus, + PurchasedProduct, +} from "@snickerdoodlelabs/objects"; +import { + IPurchaseRepository, + IPurchaseUtils, + IPurchaseUtilsType, +} from "@snickerdoodlelabs/shopping-data"; +import { inject, injectable } from "inversify"; +import { ResultAsync, okAsync } from "neverthrow"; + +import { + IDataWalletPersistence, + IDataWalletPersistenceType, +} from "@core/interfaces/data/index.js"; + +@injectable() +export class PurchaseRepository implements IPurchaseRepository { + public constructor( + @inject(IDataWalletPersistenceType) + protected persistence: IDataWalletPersistence, + @inject(IPurchaseUtilsType) + protected purchaseUtils: IPurchaseUtils, + ) {} + public add(purchase: PurchasedProduct): ResultAsync { + return this.getByMarketplaceAndDate( + purchase.marketPlace, + purchase.datePurchased, + ).andThen((existingPurchases) => { + return this.purchaseUtils + .contains(existingPurchases, purchase) + .andThen((contains) => { + if (contains) { + return okAsync(undefined); + } + return this.persistence.updateRecord( + ERecordKey.PURCHASED_PRODUCT, + purchase, + ); + }); + }); + } + + public getPurchasedProducts(): ResultAsync< + PurchasedProduct[], + PersistenceError + > { + return this.persistence.getAll( + ERecordKey.PURCHASED_PRODUCT, + ); + } + + public getByMarketplace( + marketPlace: DomainName, + ): ResultAsync { + return this.persistence.getAllByIndex( + ERecordKey.PURCHASED_PRODUCT, + "marketPlace", + marketPlace, + ); + } + public getByMarketplaceAndDate( + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ): ResultAsync { + return this.persistence.getAllByMultiIndex( + ERecordKey.PURCHASED_PRODUCT, + ["marketPlace", "datePurchased"], + [marketPlace, datePurchased], + ); + } + public getShoppingDataConnectionStatus(): ResultAsync< + ShoppingDataConnectionStatus[], + PersistenceError + > { + return this.persistence.getAll( + ERecordKey.SHOPPING_DATA_CONNECTION_STATUS, + ); + } + + public setShoppingDataConnectionStatus( + ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ): ResultAsync { + return this.persistence.updateRecord( + ERecordKey.SHOPPING_DATA_CONNECTION_STATUS, + ShoppingDataConnectionStatus, + ); + } +} diff --git a/packages/core/src/implementations/data/index.ts b/packages/core/src/implementations/data/index.ts index 41a476b6d9..0481227147 100644 --- a/packages/core/src/implementations/data/index.ts +++ b/packages/core/src/implementations/data/index.ts @@ -17,6 +17,7 @@ export * from "@core/implementations/data/NftRepository.js"; export * from "@core/implementations/data/MetricsRepository.js"; export * from "@core/implementations/data/PermissionRepository.js"; export * from "@core/implementations/data/PortfolioBalanceRepository.js"; +export * from "@core/implementations/data/PurchaseRepository.js"; export * from "@core/implementations/data/QuestionnaireRepository.js"; export * from "@core/implementations/data/SDQLQueryRepository.js"; export * from "@core/implementations/data/TransactionHistoryRepository.js"; diff --git a/packages/core/src/implementations/data/utilities/DataWalletPersistence.ts b/packages/core/src/implementations/data/utilities/DataWalletPersistence.ts index 7ce7450809..bafa95516c 100644 --- a/packages/core/src/implementations/data/utilities/DataWalletPersistence.ts +++ b/packages/core/src/implementations/data/utilities/DataWalletPersistence.ts @@ -274,6 +274,17 @@ export class DataWalletPersistence implements IDataWalletPersistence { return values.map((x) => x.data); }); } + public getAllByMultiIndex( + recordKey: ERecordKey, + indices: string[], + values: IDBValidKey | IDBKeyRange, + ): ResultAsync { + return this.volatileStorage + .getAllByIndex(recordKey, indices, values) + .map((values) => { + return values.map((x) => x.data); + }); + } // TODO: Fix this- it should return keys, not T! public getAllKeys( diff --git a/packages/core/src/implementations/utilities/ConfigProvider.ts b/packages/core/src/implementations/utilities/ConfigProvider.ts index f1de51f0c2..d1630c6c2a 100644 --- a/packages/core/src/implementations/utilities/ConfigProvider.ts +++ b/packages/core/src/implementations/utilities/ConfigProvider.ts @@ -1,3 +1,4 @@ +import { IScraperConfig } from "@snickerdoodlelabs/ai-scraper"; import { IIndexerConfigProvider } from "@snickerdoodlelabs/indexers"; import { chainConfig, @@ -87,6 +88,13 @@ export class ConfigProvider pollInterval: 1 * 24 * 3600 * 1000, } as TwitterConfig; + const scraperConfig = { + scraper: { + OPENAI_API_KEY: "", + timeout: 5 * 60 * 1000, // 5 minutes + }, + } as IScraperConfig; + // All the default config below is for testing on local, using the test-harness package this.config = new CoreConfig( controlChainId, // controlChainId @@ -183,8 +191,9 @@ export class ConfigProvider ), null, // devChainProviderURL, Defaults to null but will be set if the control chain is Doodlechain 60 * 60 * 6, // maxStatsRetentionSeconds 6 hours - LanguageCode("en"), // passwordLanguageCode + LanguageCode("en"), // passwordLanguageCode, 100, // sets the size for query performance events, e.g. how many errors and durations will be stored, main metric object will not be effected + scraperConfig.scraper, ); } @@ -372,6 +381,11 @@ export class ConfigProvider }; this.config.heartbeatIntervalMS = overrides.heartbeatIntervalMS ?? this.config.heartbeatIntervalMS; + + this.config.scraper = { + ...this.config.scraper, + ...overrides.scraper, + }; this.config.queryPerformanceMetricsLimit = overrides.queryPerformanceMetricsLimit ?? this.config.queryPerformanceMetricsLimit; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 1914e433f3..53bf58d916 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,3 +2,5 @@ import "reflect-metadata"; export * from "@core/implementations/SnickerdoodleCore.js"; export * from "@core/implementations/utilities/ConfigProvider.js"; + +export * from "@core/interfaces/utilities/index.js"; diff --git a/packages/core/src/interfaces/business/IPurchaseService.ts b/packages/core/src/interfaces/business/IPurchaseService.ts new file mode 100644 index 0000000000..8b46fbbf16 --- /dev/null +++ b/packages/core/src/interfaces/business/IPurchaseService.ts @@ -0,0 +1,28 @@ +import { + DomainName, + PersistenceError, + PurchasedProduct, + ShoppingDataConnectionStatus, + UnixTimestamp, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface IPurchaseService { + getPurchasedProducts(): ResultAsync; + getByMarketplace( + marketPlace: DomainName, + ): ResultAsync; + getByMarketplaceAndDate( + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ): ResultAsync; + getShoppingDataConnectionStatus(): ResultAsync< + ShoppingDataConnectionStatus[], + PersistenceError + >; + setShoppingDataConnectionStatus( + ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ): ResultAsync; +} + +export const IPurchaseServiceType = Symbol.for("IPurchaseService"); diff --git a/packages/core/src/interfaces/business/index.ts b/packages/core/src/interfaces/business/index.ts index 6ed9ee6d5b..8c7e958603 100644 --- a/packages/core/src/interfaces/business/index.ts +++ b/packages/core/src/interfaces/business/index.ts @@ -10,6 +10,7 @@ export * from "@core/interfaces/business/IMarketplaceService.js"; export * from "@core/interfaces/business/IMetricsService.js"; export * from "@core/interfaces/business/IMonitoringService.js"; export * from "@core/interfaces/business/IProfileService.js"; +export * from "@core/interfaces/business/IPurchaseService.js"; export * from "@core/interfaces/business/IQueryService.js"; export * from "@core/interfaces/business/IQuestionnaireService.js"; export * from "@core/interfaces/business/ITwitterService.js"; diff --git a/packages/core/src/interfaces/data/utilities/IDataWalletPersistence.ts b/packages/core/src/interfaces/data/utilities/IDataWalletPersistence.ts index e3ce8c248b..d8da2f51ec 100644 --- a/packages/core/src/interfaces/data/utilities/IDataWalletPersistence.ts +++ b/packages/core/src/interfaces/data/utilities/IDataWalletPersistence.ts @@ -52,6 +52,13 @@ export interface IDataWalletPersistence { query: IDBValidKey | IDBKeyRange, priority?: EBackupPriority, ): ResultAsync; + + getAllByMultiIndex( + recordKey: ERecordKey, + indices: string[], + values: IDBValidKey | IDBKeyRange, + ): ResultAsync; + getAllKeys( recordKey: ERecordKey, indexName?: string, @@ -174,4 +181,4 @@ export interface IDataWalletPersistence { // #endregion } -export const IDataWalletPersistenceType = Symbol.for("IDataWalletPersistence"); +export const IDataWalletPersistenceType = Symbol.for("IDataWalletPersistence"); \ No newline at end of file diff --git a/packages/core/src/interfaces/objects/CoreConfig.ts b/packages/core/src/interfaces/objects/CoreConfig.ts index 5d67e8f8a4..ae125b0e4b 100644 --- a/packages/core/src/interfaces/objects/CoreConfig.ts +++ b/packages/core/src/interfaces/objects/CoreConfig.ts @@ -1,3 +1,4 @@ +import { IScraperConfig } from "@snickerdoodlelabs/ai-scraper"; import { IIndexerConfig } from "@snickerdoodlelabs/indexers"; import { ControlChainInformation, @@ -14,7 +15,9 @@ import { IPersistenceConfig } from "@snickerdoodlelabs/persistence"; import { MetatransactionGasAmounts } from "@core/interfaces/objects/MetatransactionGasAmounts.js"; -export class CoreConfig implements IIndexerConfig, IPersistenceConfig { +export class CoreConfig + implements IIndexerConfig, IPersistenceConfig, IScraperConfig +{ public constructor( public controlChainId: EChain, public controlChainInformation: ControlChainInformation, @@ -51,5 +54,9 @@ export class CoreConfig implements IIndexerConfig, IPersistenceConfig { public maxStatsRetentionSeconds: number, public passwordLanguageCode: LanguageCode, public queryPerformanceMetricsLimit: number, + public scraper: { + OPENAI_API_KEY: string; + timeout: number; + }, ) {} } diff --git a/packages/core/test/mock/mocks/commonValues.ts b/packages/core/test/mock/mocks/commonValues.ts index 17a7d2bc46..4b87b23d06 100644 --- a/packages/core/test/mock/mocks/commonValues.ts +++ b/packages/core/test/mock/mocks/commonValues.ts @@ -220,8 +220,12 @@ export const testCoreConfig = new CoreConfig( ), // metatransactionGasAmounts ProviderUrl("devChainProviderURL"), // devChainProviderURL 60, // maxStatsRetentionSeconds - LanguageCode("en-pw"), // passwordLanguageCode + LanguageCode("en-pw"), // passwordLanguageCode, 100, + { + OPENAI_API_KEY: process.env.OPENAI_API_KEY ?? "", + timeout: 5 * 60 * 1000, // 5 minutes + }, ); const adContent1: AdContent = new AdContent( diff --git a/packages/core/test/unit/data/PurchaseRepository.test.ts b/packages/core/test/unit/data/PurchaseRepository.test.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/core/test/unit/data/QuestionnaireRepository.test.ts b/packages/core/test/unit/data/QuestionnaireRepository.test.ts index b8f08827d1..206ffc6ebc 100644 --- a/packages/core/test/unit/data/QuestionnaireRepository.test.ts +++ b/packages/core/test/unit/data/QuestionnaireRepository.test.ts @@ -5,6 +5,7 @@ import { EQuestionnaireStatus, ERecordKey, InvalidParametersError, + PageNumber, PagedResponse, PagingRequest, QuestionnaireHistory, @@ -36,7 +37,7 @@ import { import { AjaxUtilsMock, ConfigProviderMock } from "@core-tests/mock/utilities"; import "fake-indexeddb/auto"; const currentTime = UnixTimestamp(1701779736); -const pagingRequest = new PagingRequest(1, 10); +const pagingRequest = new PagingRequest(PageNumber(1), 10); class QuestionnaireRepositoryMocks { public persistence: IDataWalletPersistence; @@ -380,7 +381,7 @@ describe("QuestionnaireRepository tests", () => { const expectedPagedResponse = new PagedResponse( [mockQuestionnaireWithAnswer, mockQuestionnaire2], - 1, + PageNumber(1), 10, 2, ); @@ -403,7 +404,7 @@ describe("QuestionnaireRepository tests", () => { const expectedPagedResponse = new PagedResponse( [mockQuestionnaire2, mockQuestionnaireWithAnswer], - 1, + PageNumber(1), 10, 2, ); @@ -427,7 +428,7 @@ describe("QuestionnaireRepository tests", () => { const expectedPagedResponse = new PagedResponse( [mockQuestionnaire2], - 1, + PageNumber(1), 10, 1, ); @@ -447,7 +448,7 @@ describe("QuestionnaireRepository tests", () => { const expectedPagedResponse = new PagedResponse( [mockQuestionnaire2], - 1, + PageNumber(1), 10, 1, ); @@ -467,7 +468,7 @@ describe("QuestionnaireRepository tests", () => { const expectedPagedResponse = new PagedResponse( [mockQuestionnaireWithAnswer], - 1, + PageNumber(1), 10, 1, ); @@ -487,7 +488,7 @@ describe("QuestionnaireRepository tests", () => { const expectedPagedResponse = new PagedResponse( [mockQuestionnaireWithAnswer, mockQuestionnaire2], - 1, + PageNumber(1), 10, 2, ); diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index fc372a943e..21ba8b4047 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -14,6 +14,9 @@ } }, "references": [ + { + "path": "../ai-scraper" + }, { "path": "../common-utils" }, @@ -26,6 +29,9 @@ { "path": "../insight-platform-api" }, + { + "path": "../nlp" + }, { "path": "../node-utils" }, @@ -40,6 +46,9 @@ }, { "path": "../signatureVerification" + }, + { + "path": "../shopping-data" } ], "include": [ diff --git a/packages/docker/Dockerfile b/packages/docker/Dockerfile index 43958a8aed..c5967d5739 100644 --- a/packages/docker/Dockerfile +++ b/packages/docker/Dockerfile @@ -7,6 +7,7 @@ WORKDIR /build # Copy all the stuff needed to run and cache yarn COPY .yarn .yarn COPY package.json yarn.lock .yarnrc.yml ./ +COPY packages/ai-scraper/package.json /build/packages/ai-scraper/package.json COPY packages/browserExtension/package.json /build/packages/browserExtension/package.json COPY packages/common-utils/package.json /build/packages/common-utils/package.json COPY packages/contracts/package.json /build/packages/contracts/package.json @@ -18,11 +19,13 @@ COPY packages/iframe/package.json /build/packages/iframe/package.json COPY packages/indexers/package.json /build/packages/indexers/package.json COPY packages/insight-platform-api/package.json /build/packages/insight-platform-api/package.json COPY packages/mobile/package.json /build/packages/mobile/package.json +COPY packages/nlp/package.json /build/packages/nlp/package.json COPY packages/node-utils/package.json /build/packages/node-utils/package.json COPY packages/objects/package.json /build/packages/objects/package.json COPY packages/persistence/package.json /build/packages/persistence/package.json COPY packages/query-parser/package.json /build/packages/query-parser/package.json COPY packages/shared-components/package.json /build/packages/shared-components/package.json +COPY packages/shopping-data/package.json /build/packages/shopping-data/package.json COPY packages/signatureVerification/package.json /build/packages/signatureVerification/package.json COPY packages/static-web-integration/package.json /build/packages/static-web-integration/package.json COPY packages/synamint-extension-sdk/package.json /build/packages/synamint-extension-sdk/package.json diff --git a/packages/extension-onboarding/package.json b/packages/extension-onboarding/package.json index ddaf74fe8f..45611bee83 100644 --- a/packages/extension-onboarding/package.json +++ b/packages/extension-onboarding/package.json @@ -51,6 +51,7 @@ "react-chartjs-2": "^4.3.1", "react-dom": "^17.0.2", "react-ga": "^3.3.1", + "react-google-charts": "^4.0.1", "react-google-login": "^5.2.2", "react-hotjar": "^5.2.0", "react-intersection-observer": "^9.4.3", diff --git a/packages/extension-onboarding/src/containers/Router/Router.paths.ts b/packages/extension-onboarding/src/containers/Router/Router.paths.ts index 324691beff..8af2740c29 100644 --- a/packages/extension-onboarding/src/containers/Router/Router.paths.ts +++ b/packages/extension-onboarding/src/containers/Router/Router.paths.ts @@ -34,5 +34,6 @@ export enum EPaths { BROWSER_ACTIVITY = "/data-dashboard/browser-activity", SOCIAL_MEDIA_DATA = "/data-dashboard/social-media-data", // PERSONAL_INFO = "/data-dashboard/personal-info", + SHOPPING_DATA = "/data-dashboard/shopping-data", NFT_DETAIL = "/data-dashboard/nfts/detail", } diff --git a/packages/extension-onboarding/src/containers/Router/Router.pathsV2.ts b/packages/extension-onboarding/src/containers/Router/Router.pathsV2.ts index 064afbb8b9..55ea2828fc 100644 --- a/packages/extension-onboarding/src/containers/Router/Router.pathsV2.ts +++ b/packages/extension-onboarding/src/containers/Router/Router.pathsV2.ts @@ -14,4 +14,5 @@ export enum EPathsV2 { BROWSER_ACTIVITY = "/data-dashboard/browser-activity", SOCIAL_MEDIA_DATA = "/data-dashboard/social-media-data", TRANSACTION_HISTORY = "/data-dashboard/transaction-history", + SHOPPING_DATA = "/data-dashboard/shopping-data", } diff --git a/packages/extension-onboarding/src/containers/Router/Router.routes.tsx b/packages/extension-onboarding/src/containers/Router/Router.routes.tsx index cebef4026b..aa8d13ef02 100644 --- a/packages/extension-onboarding/src/containers/Router/Router.routes.tsx +++ b/packages/extension-onboarding/src/containers/Router/Router.routes.tsx @@ -34,6 +34,9 @@ const LazySocialMediaInfo = lazy( const LazySettings = lazy( () => import("@extension-onboarding/pages/V2/Settings"), ); +const LazyShoppingData = lazy( + () => import("@extension-onboarding/pages/V2/ShoppingData"), +); export const OnboardingRoutes = ( @@ -122,6 +125,22 @@ export const AuthFlowRoutes = ( } /> + + + + } + /> + + + + } + /> } /> diff --git a/packages/extension-onboarding/src/context/AccountLinkingContext.tsx b/packages/extension-onboarding/src/context/AccountLinkingContext.tsx index 323b1fd898..524c723ec8 100644 --- a/packages/extension-onboarding/src/context/AccountLinkingContext.tsx +++ b/packages/extension-onboarding/src/context/AccountLinkingContext.tsx @@ -1,23 +1,3 @@ -import AccountLinkingIndicator from "@extension-onboarding/components/loadingIndicators/AccountLinking"; -import { EModalSelectors } from "@extension-onboarding/components/Modals/"; -import LinkAccountModal from "@extension-onboarding/components/Modals/V2/LinkAccountModal"; -import { EWalletProviderKeys } from "@extension-onboarding/constants"; -import { useAppContext } from "@extension-onboarding/context/App"; -import { useDataWalletContext } from "@extension-onboarding/context/DataWalletContext"; -import { - ELoadingIndicatorType, - useLayoutContext, -} from "@extension-onboarding/context/LayoutContext"; -import useIsMobile from "@extension-onboarding/hooks/useIsMobile"; -import { IProvider } from "@extension-onboarding/services/blockChainWalletProviders"; -import { - DiscordProvider, - TwitterProvider, -} from "@extension-onboarding/services/socialMediaProviders/implementations"; -import { - IDiscordProvider, - ITwitterProvider, -} from "@extension-onboarding/services/socialMediaProviders/interfaces"; import { defaultLanguageCode, EChain, @@ -42,6 +22,27 @@ import React, { } from "react"; import { useAccount, useDisconnect, useSignMessage, useConnect } from "wagmi"; +import AccountLinkingIndicator from "@extension-onboarding/components/loadingIndicators/AccountLinking"; +import { EModalSelectors } from "@extension-onboarding/components/Modals/"; +import LinkAccountModal from "@extension-onboarding/components/Modals/V2/LinkAccountModal"; +import { EWalletProviderKeys } from "@extension-onboarding/constants"; +import { useAppContext } from "@extension-onboarding/context/App"; +import { useDataWalletContext } from "@extension-onboarding/context/DataWalletContext"; +import { + ELoadingIndicatorType, + useLayoutContext, +} from "@extension-onboarding/context/LayoutContext"; +import useIsMobile from "@extension-onboarding/hooks/useIsMobile"; +import { IProvider } from "@extension-onboarding/services/blockChainWalletProviders"; +import { + DiscordProvider, + TwitterProvider, +} from "@extension-onboarding/services/socialMediaProviders/implementations"; +import { + IDiscordProvider, + ITwitterProvider, +} from "@extension-onboarding/services/socialMediaProviders/interfaces"; + export enum EWalletProviderKit { SUI = "SUI", WEB3_MODAL = "WEB3_MODAL", @@ -96,6 +97,7 @@ export const AccountLinkingContextProvider: FC = memo(({ children }) => { isLinkerModalOpen, setLinkerModalClose, socialMediaProviderList, + shoppingDataProviderList, } = useAppContext(); const { setModal, setLoadingStatus } = useLayoutContext(); const [isSuiOpen, setIsSuiOpen] = useState(false); diff --git a/packages/extension-onboarding/src/context/App.tsx b/packages/extension-onboarding/src/context/App.tsx index 174001ac01..c4ef71c9d1 100644 --- a/packages/extension-onboarding/src/context/App.tsx +++ b/packages/extension-onboarding/src/context/App.tsx @@ -1,23 +1,7 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ - -import { EAlertSeverity } from "@extension-onboarding/components/CustomizedAlert"; -import { ALERT_MESSAGES } from "@extension-onboarding/constants"; -import { useDataWalletContext } from "@extension-onboarding/context/DataWalletContext"; -import { useNotificationContext } from "@extension-onboarding/context/NotificationContext"; import { EOnboardingState, - IUIState, } from "@extension-onboarding/objects/interfaces/IUState"; -import { - getProviderList as getChainProviderList, - IProvider, -} from "@extension-onboarding/services/blockChainWalletProviders"; -import { ApiGateway } from "@extension-onboarding/services/implementations/ApiGateway"; -import { DataWalletGateway } from "@extension-onboarding/services/implementations/DataWalletGateway"; -import { - getProviderList as getSocialMediaProviderList, - ISocialMediaWrapper, -} from "@extension-onboarding/services/socialMediaProviders"; import { BigNumberString, EarnedReward, @@ -39,9 +23,29 @@ import React, { } from "react"; import { Subscription } from "rxjs"; import Loading from "@extension-onboarding/setupScreens/Loading"; -import { okAsync } from "neverthrow"; import { UIStateUtils } from "@extension-onboarding/utils/UIStateUtils"; +import { EAlertSeverity } from "@extension-onboarding/components/CustomizedAlert"; +import { + ALERT_MESSAGES, +} from "@extension-onboarding/constants"; +import { useDataWalletContext } from "@extension-onboarding/context/DataWalletContext"; +import { useNotificationContext } from "@extension-onboarding/context/NotificationContext"; +import { + getProviderList as getChainProviderList, + IProvider, +} from "@extension-onboarding/services/blockChainWalletProviders"; +import { ApiGateway } from "@extension-onboarding/services/implementations/ApiGateway"; +import { DataWalletGateway } from "@extension-onboarding/services/implementations/DataWalletGateway"; +import { + getProviderList as getShoppingDataProviderList, + IShoppingDataWrapper, +} from "@extension-onboarding/services/shoppingDataProvider"; +import { + getProviderList as getSocialMediaProviderList, + ISocialMediaWrapper, +} from "@extension-onboarding/services/socialMediaProviders"; + export interface IInvitationInfo { consentAddress: EVMContractAddress | undefined; tokenId: BigNumberString | undefined; @@ -58,6 +62,7 @@ export interface IAppContext { earnedRewards: EarnedReward[] | undefined; optedInContracts: Map | undefined; socialMediaProviderList: ISocialMediaWrapper[]; + shoppingDataProviderList: IShoppingDataWrapper[]; addAccount(account: LinkedAccount): void; invitationInfo: IInvitationInfo; setInvitationInfo: (invitationInfo: IInvitationInfo) => void; @@ -286,6 +291,7 @@ export const AppContextProvider: FC = ({ children }) => { dataWalletGateway: new DataWalletGateway(sdlDataWallet), providerList: chainProviderList, socialMediaProviderList: getSocialMediaProviderList(sdlDataWallet), + shoppingDataProviderList: getShoppingDataProviderList(), linkedAccounts, earnedRewards, addAccount, diff --git a/packages/extension-onboarding/src/layouts/DataDashboardLayout.tsx b/packages/extension-onboarding/src/layouts/DataDashboardLayout.tsx index 26d644d83d..336ebefedd 100644 --- a/packages/extension-onboarding/src/layouts/DataDashboardLayout.tsx +++ b/packages/extension-onboarding/src/layouts/DataDashboardLayout.tsx @@ -1,13 +1,16 @@ -import Container from "@extension-onboarding/components/v2/Container"; -import DashboardTitle from "@extension-onboarding/components/v2/DashboardTitle"; -import { EPathsV2 as EPaths } from "@extension-onboarding/containers/Router/Router.pathsV2"; -import { DashboardContextProvider } from "@extension-onboarding/context/DashboardContext"; import { Box } from "@material-ui/core"; import { makeStyles } from "@material-ui/core/styles"; +import { ECoreProxyType } from "@snickerdoodlelabs/objects"; import { SDTypography } from "@snickerdoodlelabs/shared-components"; import React from "react"; import { Outlet, useLocation, useNavigate } from "react-router-dom"; +import Container from "@extension-onboarding/components/v2/Container"; +import DashboardTitle from "@extension-onboarding/components/v2/DashboardTitle"; +import { EPathsV2 as EPaths } from "@extension-onboarding/containers/Router/Router.pathsV2"; +import { DashboardContextProvider } from "@extension-onboarding/context/DashboardContext"; +import { useDataWalletContext } from "@extension-onboarding/context/DataWalletContext"; + const useStyles = makeStyles((theme) => ({ link: { textAlign: "center", @@ -79,6 +82,12 @@ const LINKS: ILink[] = [ subtitle: "Share what kinds of Discord channels you are subscribed to. No one will ever know your discord handle.", }, + { + path: EPaths.SHOPPING_DATA, + title: "Shopping Data", + /* subtitle: + "Share what kinds of Discord channels you are subscribed to. No one will ever know your discord handle.", */ + }, ]; const DataDashboardLayout = () => { @@ -86,6 +95,8 @@ const DataDashboardLayout = () => { const navigate = useNavigate(); const classes = useStyles(); + const { sdlDataWallet } = useDataWalletContext(); + const navContainerRef = React.useRef(null); React.useEffect(() => { @@ -144,15 +155,20 @@ const DataDashboardLayout = () => { - link.path === location.pathname)?.title ?? "" - } - description={ - LINKS.find((link) => link.path === location.pathname)?.subtitle ?? - "" - } - /> + {!( + sdlDataWallet.proxyType === ECoreProxyType.IFRAME_INJECTED && + location.pathname === EPaths.SHOPPING_DATA + ) && ( + link.path === location.pathname)?.title ?? "" + } + description={ + LINKS.find((link) => link.path === location.pathname)?.subtitle ?? + "" + } + /> + )} diff --git a/packages/extension-onboarding/src/pages/V2/CookieVault/Sections/Questionnaires/Questionnaires.tsx b/packages/extension-onboarding/src/pages/V2/CookieVault/Sections/Questionnaires/Questionnaires.tsx index ae6e135be7..abb892f1b0 100644 --- a/packages/extension-onboarding/src/pages/V2/CookieVault/Sections/Questionnaires/Questionnaires.tsx +++ b/packages/extension-onboarding/src/pages/V2/CookieVault/Sections/Questionnaires/Questionnaires.tsx @@ -13,6 +13,7 @@ import { QuestionnaireWithAnswers, URLString, PagingRequest, + PageNumber, } from "@snickerdoodlelabs/objects"; import { useResponsiveValue } from "@snickerdoodlelabs/shared-components"; import { okAsync } from "neverthrow"; @@ -37,7 +38,7 @@ const Questionnaries = () => { const getQuestionnaires = () => { sdlDataWallet.questionnaire - .getAllQuestionnaires(new PagingRequest(1, 50)) + .getAllQuestionnaires(new PagingRequest(PageNumber(1), 50)) .map((res) => { setQuestionnaires(res.response); }) diff --git a/packages/extension-onboarding/src/pages/V2/ShoppingData/Components/IFrameComponent.tsx b/packages/extension-onboarding/src/pages/V2/ShoppingData/Components/IFrameComponent.tsx new file mode 100644 index 0000000000..da96f892a3 --- /dev/null +++ b/packages/extension-onboarding/src/pages/V2/ShoppingData/Components/IFrameComponent.tsx @@ -0,0 +1,44 @@ +import { Box } from "@material-ui/core"; +import { SDButton, SDTypography } from "@snickerdoodlelabs/shared-components"; +import React from "react"; + +import { DOWNLOAD_URL } from "@extension-onboarding/constants"; +import useIsMobile from "@extension-onboarding/hooks/useIsMobile"; + +export const IFrameComponent = () => { + const isMobile = useIsMobile(); + + return ( + + + + + + + + {isMobile + ? "This feature is not usable on mobile." + : "For this feature, you will need a SnickerDoodle Data Wallet extension."} + + + + {!isMobile && ( + + { + window.open(DOWNLOAD_URL, "_blank"); + }} + > + Get Extension + + + )} + + ); +}; diff --git a/packages/extension-onboarding/src/pages/V2/ShoppingData/Components/index.ts b/packages/extension-onboarding/src/pages/V2/ShoppingData/Components/index.ts new file mode 100644 index 0000000000..b46e2ff8f8 --- /dev/null +++ b/packages/extension-onboarding/src/pages/V2/ShoppingData/Components/index.ts @@ -0,0 +1 @@ +export * from "@extension-onboarding/pages/V2/ShoppingData/Components/IFrameComponent"; diff --git a/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Amazon.tsx b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Amazon.tsx new file mode 100644 index 0000000000..49ee0ecbe7 --- /dev/null +++ b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Amazon.tsx @@ -0,0 +1,183 @@ +import { Box } from "@material-ui/core"; +import { + EKnownDomains, + PurchasedProduct, + ShoppingDataConnectionStatus, +} from "@snickerdoodlelabs/objects"; +import { + SCRAPING_INDEX, + SCRAPING_URLS, + SDTypography, +} from "@snickerdoodlelabs/shared-components"; +import React, { FC, memo, useEffect, useState } from "react"; + +import Table, { IColumn } from "@extension-onboarding/components/v2/Table"; +import { useDataWalletContext } from "@extension-onboarding/context/DataWalletContext"; +import { + AmazonDisConnectItem, + AmazonConnectItem, + AmazonDataItem, +} from "@extension-onboarding/pages/V2/ShoppingData/Platforms/Amazon/Items"; +import { IShoppingDataPlatformProps } from "@extension-onboarding/pages/V2/ShoppingData/Platforms/types"; + +export const Amazon: FC = memo( + ({ name, icon }: IShoppingDataPlatformProps) => { + const [product, setProduct] = useState([]); + const [isConnected, setIsConnected] = useState(); + const { sdlDataWallet } = useDataWalletContext(); + + const AMAZONINDEX: string | undefined = SCRAPING_INDEX.get( + EKnownDomains.Amazon, + ); + + useEffect(() => { + getProducts(); + getConnectionStatus(); + }, []); + + useEffect(() => { + console.log(product); + }, [product.length]); + + const getProducts = () => { + return sdlDataWallet.purchase.getPurchasedProducts().map((products) => { + setProduct(products); + }); + }; + + const getConnectionStatus = () => { + return sdlDataWallet.purchase + .getShoppingDataConnectionStatus() + .map((ShoppingDataConnectionStatus) => { + ShoppingDataConnectionStatus.map((status) => { + if ( + status.type === EKnownDomains.Amazon && + status.isConnected === true + ) { + setIsConnected(true); + } else { + setIsConnected(false); + } + }); + }); + }; + + const handleConnectClick = () => { + SCRAPING_URLS.filter( + (provider) => provider.key === EKnownDomains.Amazon, + ).map((provider) => { + window.location.href = `${provider.url}&SDLStep=${AMAZONINDEX}`; + }); + }; + + const handleDisconnectClick = () => { + const shoppingDataConnectionStatus: ShoppingDataConnectionStatus = + new ShoppingDataConnectionStatus(EKnownDomains.Amazon, false); + sdlDataWallet.purchase.setShoppingDataConnectionStatus( + shoppingDataConnectionStatus, + ); + window.location.reload(); + }; + + const columns: IColumn[] = [ + { + sortKey: "datePurchased", + label: "Product Name", + render: (row: PurchasedProduct) => ( + + {row.name.length > 20 ? row.name.slice(0, 20) + "..." : row.name} + + ), + }, + { + label: "Brand", + render: (row: PurchasedProduct) => ( + + {!row.brand && "N/A"} + {row.brand} + + ), + }, + { + label: "Category", + render: (row: PurchasedProduct) => ( + + {!row.category && "N/A"} + {row.category} + + ), + }, + { + sortKey: "price", + label: "Price", + render: (row: PurchasedProduct) => ( + + ${row.price} + + ), + align: "right" as const, + }, + ]; + + return ( + <> + + {isConnected ? ( + + ) : ( + + )} + + {isConnected && ( + <> + + + + + + + + )} + + ); + }, +); diff --git a/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Items/AmazonConnectItem.tsx b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Items/AmazonConnectItem.tsx new file mode 100644 index 0000000000..e9c5919a74 --- /dev/null +++ b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Items/AmazonConnectItem.tsx @@ -0,0 +1,31 @@ +import { Box } from "@material-ui/core"; +import { SDButton, SDTypography } from "@snickerdoodlelabs/shared-components"; +import React, { FC, memo } from "react"; + +interface IAmazonConnectionItemProps { + icon: string; + providerName: string; + handleConnectClick: () => void; +} + +export const AmazonConnectItem: FC = memo( + ({ icon, providerName, handleConnectClick }: IAmazonConnectionItemProps) => { + return ( + <> + + + + + + + {providerName} + + + + + Connect + + + ); + }, +); diff --git a/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Items/AmazonDataItem.tsx b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Items/AmazonDataItem.tsx new file mode 100644 index 0000000000..662505dc3c --- /dev/null +++ b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Items/AmazonDataItem.tsx @@ -0,0 +1,267 @@ +import { Box, Grid } from "@material-ui/core"; +import { PurchasedProduct } from "@snickerdoodlelabs/objects"; +import { SDTypography } from "@snickerdoodlelabs/shared-components"; +import { ChartData } from "chart.js"; +import React, { FC, memo, useMemo } from "react"; +import { Chart } from "react-google-charts"; + +interface IAmazonDataItemProps { + product: PurchasedProduct[]; +} + +export const AmazonDataItem: FC = memo( + ({ product }: IAmazonDataItemProps) => { + const calculateTotalPrices = (products) => { + const totalPrice = parseFloat( + products + .reduce((total, product) => total + product.price, 0) + .toFixed(3), + ); + + const getCategoryTotalPrice = (category, products) => { + const categoryProducts = products.filter( + (product) => product.category === category, + ); + return parseFloat( + categoryProducts + .reduce((total, product) => total + product.price, 0) + .toFixed(3), + ); + }; + + const clothesTotalPrice = getCategoryTotalPrice("Clothing", products); + const electronicsTotalPrice = getCategoryTotalPrice( + "Electronics", + products, + ); + const gameTotalPrice = getCategoryTotalPrice("Game", products); + + const otherProducts = products.filter( + (product) => + !( + product.category === "Clothing" || + product.category === "Electronics" || + product.category === "Game" + ), + ); + const otherTotalPrice = parseFloat( + otherProducts + .reduce((total, product) => total + product.price, 0) + .toFixed(3), + ); + + return { + totalPrice, + clothesTotalPrice, + electronicsTotalPrice, + gameTotalPrice, + otherTotalPrice, + }; + }; + + const data: number[] = [ + calculateTotalPrices(product).clothesTotalPrice, + calculateTotalPrices(product).electronicsTotalPrice, + calculateTotalPrices(product).gameTotalPrice, + calculateTotalPrices(product).otherTotalPrice, + ]; + const labels: string[] = ["Clothes ", "Electronic ", "Game", "Other"]; + const pieChartData: ChartData<"pie", number[], any> = { + labels: labels, + datasets: [ + { + data: data, + backgroundColor: ["#292648", "#6E62A6", "#D2CEE3", "#AFAADB"], + }, + ], + }; + + const chartData = [ + ["Categories", "Price"], + ["Clothes", calculateTotalPrices(product).clothesTotalPrice], + ["Electronic", calculateTotalPrices(product).electronicsTotalPrice], + ["Game", calculateTotalPrices(product).gameTotalPrice], + ["Other", calculateTotalPrices(product).otherTotalPrice], + ]; + + const options = { + legend: "none", + pieSliceText: "label", + pieStartAngle: 100, + colors: ["#292648", "#6E62A6", "#D2CEE3", "#AFAADB"], + }; + + const chartComponent = useMemo(() => { + if (product.length !== 0) { + return ( + <> + + + + + + Total Spending + + + + + ${calculateTotalPrices(product).totalPrice} + + + + + + + + + Number Of Purchased Product + + + + + {product.length} + + + + + + + + + + + Most Selected Categories Breakdown + + + + + + + + {pieChartData.labels?.map((label, index) => ( + + + + + {label} + + + {pieChartData.datasets[0].data[index]} + + + + ))} + + + + + + + ); + } else { + return ; + } + }, [product]); + + return ( + <> + {chartComponent} + + ); + }, +); diff --git a/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Items/AmazonDisconnectItem.tsx b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Items/AmazonDisconnectItem.tsx new file mode 100644 index 0000000000..e30802054f --- /dev/null +++ b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Items/AmazonDisconnectItem.tsx @@ -0,0 +1,87 @@ +import { Box, Button, Typography } from "@material-ui/core"; +import { PurchasedProduct } from "@snickerdoodlelabs/objects"; +import { SDButton, SDTypography } from "@snickerdoodlelabs/shared-components"; +import React, { FC, memo } from "react"; + +interface IAmazonDisConnectItemProps { + icon: string; + providerName: string; + handleDisconnectClick: () => void; + product: PurchasedProduct[]; +} + +export const AmazonDisConnectItem: FC = memo( + ({ + icon, + providerName, + handleDisconnectClick, + product, + }: IAmazonDisConnectItemProps) => { + const dateCreatedArray = product.map((product) => { + const unixTimestamp = product.dateCreated; + const date = new Date(unixTimestamp * 1000); + const options: Intl.DateTimeFormatOptions = { + day: "numeric", + month: "long", + hour: "2-digit", + minute: "2-digit", + }; + return date.toLocaleDateString("en-US", options); + }); + + const lastUpdate = dateCreatedArray.reduce((oldestDate, currentDate) => { + const currentDateTimestamp = new Date(currentDate).getTime(); + const oldestDateTimestamp = new Date(oldestDate).getTime(); + return currentDateTimestamp < oldestDateTimestamp + ? currentDate + : oldestDate; + }, dateCreatedArray[0]); + + return ( + <> + + + + + + + {providerName} + + + + + + + + Connected + + + + + + Last Updated on {lastUpdate} + + + + Disconnect + + + ); + }, +); diff --git a/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Items/index.ts b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Items/index.ts new file mode 100644 index 0000000000..c14312aa76 --- /dev/null +++ b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/Items/index.ts @@ -0,0 +1,3 @@ +export * from "@extension-onboarding/pages/V2/ShoppingData/Platforms/Amazon/Items/AmazonConnectItem"; +export * from "@extension-onboarding/pages/V2/ShoppingData/Platforms/Amazon/Items/AmazonDataItem"; +export * from "@extension-onboarding/pages/V2/ShoppingData/Platforms/Amazon/Items/AmazonDisconnectItem"; diff --git a/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/index.ts b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/index.ts new file mode 100644 index 0000000000..0c3dd465f4 --- /dev/null +++ b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/Amazon/index.ts @@ -0,0 +1 @@ +export * from "@extension-onboarding/pages/V2/ShoppingData/Platforms/Amazon/Amazon"; diff --git a/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/index.ts b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/index.ts new file mode 100644 index 0000000000..8f0ec30d8e --- /dev/null +++ b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/index.ts @@ -0,0 +1 @@ +export * from "@extension-onboarding/pages/V2/ShoppingData/Platforms/Amazon"; diff --git a/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/types.ts b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/types.ts new file mode 100644 index 0000000000..97a53d2bda --- /dev/null +++ b/packages/extension-onboarding/src/pages/V2/ShoppingData/Platforms/types.ts @@ -0,0 +1,4 @@ +export interface IShoppingDataPlatformProps { + name: string; + icon: string; +} diff --git a/packages/extension-onboarding/src/pages/V2/ShoppingData/ShoppingDataDashBoard.tsx b/packages/extension-onboarding/src/pages/V2/ShoppingData/ShoppingDataDashBoard.tsx new file mode 100644 index 0000000000..d59ca72a1e --- /dev/null +++ b/packages/extension-onboarding/src/pages/V2/ShoppingData/ShoppingDataDashBoard.tsx @@ -0,0 +1,48 @@ +import { Box } from "@material-ui/core"; +import { ECoreProxyType, EKnownDomains } from "@snickerdoodlelabs/objects"; +import React from "react"; + +import { useAppContext } from "@extension-onboarding/context/App"; +import { useDataWalletContext } from "@extension-onboarding/context/DataWalletContext"; +import { IFrameComponent } from "@extension-onboarding/pages/V2/ShoppingData/Components"; +import { Amazon } from "@extension-onboarding/pages/V2/ShoppingData/Platforms/Amazon/Amazon"; + +interface IShoppingDataProps { + name: string; + icon: string; + key: EKnownDomains; +} + +export default () => { + const { shoppingDataProviderList } = useAppContext(); + + const { sdlDataWallet } = useDataWalletContext(); + + const getShoppingDataComponentGivenProps = ({ + name, + icon, + key, + }: IShoppingDataProps) => { + switch (key) { + case EKnownDomains.Amazon: + return ; + + default: + return null; + } + }; + + return ( + + {sdlDataWallet.proxyType === ECoreProxyType.IFRAME_INJECTED ? ( + + ) : ( + shoppingDataProviderList.map(({ icon, name, key }) => ( + + {getShoppingDataComponentGivenProps({ icon, name, key })} + + )) + )} + + ); +}; diff --git a/packages/extension-onboarding/src/pages/V2/ShoppingData/index.ts b/packages/extension-onboarding/src/pages/V2/ShoppingData/index.ts new file mode 100644 index 0000000000..047843fdc3 --- /dev/null +++ b/packages/extension-onboarding/src/pages/V2/ShoppingData/index.ts @@ -0,0 +1 @@ +export { default } from "@extension-onboarding/pages/V2/ShoppingData/ShoppingDataDashBoard"; diff --git a/packages/extension-onboarding/src/services/shoppingDataProvider/index.tsx b/packages/extension-onboarding/src/services/shoppingDataProvider/index.tsx new file mode 100644 index 0000000000..bafbc95774 --- /dev/null +++ b/packages/extension-onboarding/src/services/shoppingDataProvider/index.tsx @@ -0,0 +1,15 @@ +import { EKnownDomains } from "@snickerdoodlelabs/objects"; + +export interface IShoppingDataWrapper { + icon: string; + name: string; + key: EKnownDomains; +} + +export const getProviderList = (): IShoppingDataWrapper[] => [ + { + icon: "https://storage.googleapis.com/dw-assets/spa/images/amazon-logo.png", + name: "Amazon", + key: EKnownDomains.Amazon, + }, +]; diff --git a/packages/iframe/src/app/ProxyBridge.ts b/packages/iframe/src/app/ProxyBridge.ts index 145f8cff49..e3643ce945 100644 --- a/packages/iframe/src/app/ProxyBridge.ts +++ b/packages/iframe/src/app/ProxyBridge.ts @@ -30,6 +30,7 @@ import { IProxyDiscordMethods, IProxyIntegrationMethods, IProxyMetricsMethods, + IProxyPurchaseMethods, IProxyQuestionnaireMethods, IProxyStorageMethods, IProxyTwitterMethods, @@ -54,8 +55,10 @@ import { PagingRequest, PersistenceError, ProxyError, + PurchasedProduct, QueryStatus, RuntimeMetrics, + ShoppingDataConnectionStatus, Signature, SiteVisit, TokenAddress, @@ -78,6 +81,7 @@ export class ProxyBridge implements ISdlDataWallet { public discord: IProxyDiscordMethods; public integration: IProxyIntegrationMethods; public metrics: IProxyMetricsMethods; + public purchase: IProxyPurchaseMethods; public storage: IProxyStorageMethods; public twitter: IProxyTwitterMethods = {} as IProxyTwitterMethods; public nft: INftProxyMethods; @@ -177,6 +181,43 @@ export class ProxyBridge implements ISdlDataWallet { }, }; + this.purchase = { + getPurchasedProducts: (): ResultAsync => { + return this.call(this.core.purchase.getPurchasedProducts()); + }, + getByMarketplace: ( + marketPlace: DomainName, + ): ResultAsync => { + return this.call(this.core.purchase.getByMarketplace(marketPlace)); + }, + getByMarketplaceAndDate: ( + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ): ResultAsync => { + return this.call( + this.core.purchase.getByMarketplaceAndDate( + marketPlace, + datePurchased, + ), + ); + }, + getShoppingDataConnectionStatus: (): ResultAsync< + ShoppingDataConnectionStatus[], + ProxyError + > => { + return this.call(this.core.purchase.getShoppingDataConnectionStatus()); + }, + setShoppingDataConnectionStatus: ( + ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ): ResultAsync => { + return this.call( + this.core.purchase.setShoppingDataConnectionStatus( + ShoppingDataConnectionStatus, + ), + ); + }, + }; + this.storage = { setAuthenticatedStorage: ( type: ECloudStorageType, diff --git a/packages/nlp/jest.config.ts b/packages/nlp/jest.config.ts new file mode 100644 index 0000000000..b58d7a0c9c --- /dev/null +++ b/packages/nlp/jest.config.ts @@ -0,0 +1,22 @@ +import type { Config } from "@jest/types"; + +const config: Config.InitialOptions = { + testEnvironment: "node", + testMatch: ["/dist/test/**/*.test.js"], + + // This does not seem to support blacklisting any folder which means we can't enable parent directory and disable child + // We should be using peer directories for coverage and non-coverage tests. + collectCoverageFrom: [ + // Enabling following means we can't disable src/tests from coverage report + // "/src/**/*.ts", + + // Add other allowed folders to the list below. + "/dist/implementations/**/*.js", + "!/src/implementations/**/index.ts", + + // Disabled because we don't want it to end up in coverage report, + // "/src/tests/**/*.ts", + ], +}; + +export default config; diff --git a/packages/nlp/package.json b/packages/nlp/package.json new file mode 100644 index 0000000000..a48ffa2ac6 --- /dev/null +++ b/packages/nlp/package.json @@ -0,0 +1,52 @@ +{ + "name": "@snickerdoodlelabs/nlp", + "version": "0.0.17", + "description": "Shopping data library for Data Wallet extension", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/SnickerdoodleLabs/protocol.git" + }, + "bugs": { + "url": "https://github.com/SnickerdoodleLabs/protocol/issues" + }, + "homepage": "https://github.com/SnickerdoodleLabs/protocol/tree/master/documentation/nlp", + "author": "Golam Muktadir ", + "keywords": [ + "Snickerdoodle", + "SDQL" + ], + "main": "dist/index.js", + "files": [ + "dist", + "!dist/test", + "!dist/tsconfig.tsbuildinfo", + "!test", + "!src", + "!tsconfig.json" + ], + "scripts": { + "alias": "tsc-alias && tsc-alias -p test/tsconfig.json", + "alias-with-copyfiles": "yarn copy-files && tsc-alias", + "build": "yarn clean && yarn compile", + "clean": "npx rimraf dist tsconfig.tsbuildinfo", + "compile": "npx tsc --build && cd ../.. && yarn alias", + "copy-files": "copyfiles -u 1 src/**/*.d.ts dist/", + "prepare": "yarn build", + "prepublish": "yarn build", + "test": "NODE_OPTIONS=--experimental-vm-modules npx jest --maxWorkers=50% --coverage --passWithNoTests" + }, + "type": "module", + "types": "dist/index.d.ts", + "dependencies": { + "@nlpjs/lang-en": "^4.26.1", + "@snickerdoodlelabs/common-utils": "workspace:^", + "@snickerdoodlelabs/objects": "workspace:^", + "ethers": "^5.6.6", + "html-to-text": "^9.0.5", + "inversify": "^6.0.1", + "neverthrow": "^5.1.0", + "neverthrow-result-utils": "^2.0.2", + "openai": "^4.0.1" + } +} diff --git a/packages/nlp/src/implementations/StemmerService.ts b/packages/nlp/src/implementations/StemmerService.ts new file mode 100644 index 0000000000..60b59a42bb --- /dev/null +++ b/packages/nlp/src/implementations/StemmerService.ts @@ -0,0 +1,52 @@ +import { BaseStemmer } from "@nlpjs/core"; +import { StemmerEn, StopwordsEn } from "@nlpjs/lang-en"; +import { ELanguageCode, NLPError } from "@snickerdoodlelabs/objects"; +import { injectable } from "inversify"; +import { ResultAsync, errAsync, okAsync } from "neverthrow"; + +import { IStemmerService } from "@nlp/interfaces/index.js"; +import { NLPSupportedLanguages } from "@nlp/objects/index.js"; + +@injectable() +export class StemmerService implements IStemmerService { + public tokenizeSync(language: ELanguageCode, text: string): string[] { + if (!NLPSupportedLanguages.includes(language)) { + return text.split(" "); + } + try { + return this.getStemmer(language).tokenizeAndStem(text, false); // does normalization by default and, false means "dont keep stopwords" + } catch (error) { + throw new NLPError((error as Error).message, error); + } + } + + public tokenize( + language: ELanguageCode, + text: string, + ): ResultAsync { + try { + return okAsync(this.tokenizeSync(language, text)); + } catch (error) { + return errAsync(error as NLPError); // guranteed to be NLPError + } + } + + /** + * + * @param language + * @returns returns english stemmer by default + */ + private getStemmer(language: ELanguageCode): BaseStemmer { + switch (language) { + case ELanguageCode.English: + return this.getStemmerEn(); + } + return this.getStemmerEn(); + } + + private getStemmerEn(): StemmerEn { + const stemmer = new StemmerEn(); + stemmer.stopwords = new StopwordsEn(); + return stemmer; + } +} diff --git a/packages/nlp/src/implementations/index.ts b/packages/nlp/src/implementations/index.ts new file mode 100644 index 0000000000..fcb54dd144 --- /dev/null +++ b/packages/nlp/src/implementations/index.ts @@ -0,0 +1 @@ +export * from "@nlp/implementations/StemmerService.js"; diff --git a/packages/nlp/src/index.ts b/packages/nlp/src/index.ts new file mode 100644 index 0000000000..78f706eb17 --- /dev/null +++ b/packages/nlp/src/index.ts @@ -0,0 +1,4 @@ +export * from "@nlp/objects/index.js"; +export * from "@nlp/interfaces/index.js"; +export * from "@nlp/implementations/index.js"; +export * from "@nlp/nlp.module.js"; diff --git a/packages/nlp/src/interfaces/IStemmerService.ts b/packages/nlp/src/interfaces/IStemmerService.ts new file mode 100644 index 0000000000..711225f889 --- /dev/null +++ b/packages/nlp/src/interfaces/IStemmerService.ts @@ -0,0 +1,20 @@ +import { ELanguageCode, NLPError } from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface IStemmerService { + tokenize( + language: ELanguageCode, + text: string, + ): ResultAsync; + + /** + * This version throws NLPError. Needed for optimization + * @param language + * @param text + * @returns tokens + * @throws NLPError + */ + tokenizeSync(language: ELanguageCode, text: string): string[]; +} + +export const IStemmerServiceType = Symbol.for("IStemmerService"); diff --git a/packages/nlp/src/interfaces/index.ts b/packages/nlp/src/interfaces/index.ts new file mode 100644 index 0000000000..6246a20320 --- /dev/null +++ b/packages/nlp/src/interfaces/index.ts @@ -0,0 +1 @@ +export * from "@nlp/interfaces/IStemmerService.js"; diff --git a/packages/nlp/src/nlp.module.ts b/packages/nlp/src/nlp.module.ts new file mode 100644 index 0000000000..717782cb59 --- /dev/null +++ b/packages/nlp/src/nlp.module.ts @@ -0,0 +1,16 @@ +import { ContainerModule, interfaces } from "inversify"; + +import { StemmerService } from "@nlp/implementations/index.js"; +import { IStemmerService, IStemmerServiceType } from "@nlp/interfaces/index.js"; +export const nlpModule = new ContainerModule( + ( + bind: interfaces.Bind, + _unbind: interfaces.Unbind, + _isBound: interfaces.IsBound, + _rebind: interfaces.Rebind, + ) => { + bind(IStemmerServiceType) + .to(StemmerService) + .inSingletonScope(); + }, +); diff --git a/packages/nlp/src/objects/NLPSupportedLanguages.ts b/packages/nlp/src/objects/NLPSupportedLanguages.ts new file mode 100644 index 0000000000..0cb891eb4f --- /dev/null +++ b/packages/nlp/src/objects/NLPSupportedLanguages.ts @@ -0,0 +1,3 @@ +import { ELanguageCode } from "@snickerdoodlelabs/objects"; + +export const NLPSupportedLanguages = [ELanguageCode.English]; diff --git a/packages/nlp/src/objects/index.ts b/packages/nlp/src/objects/index.ts new file mode 100644 index 0000000000..8ef5c2288a --- /dev/null +++ b/packages/nlp/src/objects/index.ts @@ -0,0 +1 @@ +export * from "@nlp/objects/NLPSupportedLanguages.js"; diff --git a/packages/nlp/test/mocks/index.ts b/packages/nlp/test/mocks/index.ts new file mode 100644 index 0000000000..be177baecf --- /dev/null +++ b/packages/nlp/test/mocks/index.ts @@ -0,0 +1 @@ +export * from "@nlp-test/mocks/testData.js"; diff --git a/packages/nlp/test/mocks/testData.ts b/packages/nlp/test/mocks/testData.ts new file mode 100644 index 0000000000..09a7c5e917 --- /dev/null +++ b/packages/nlp/test/mocks/testData.ts @@ -0,0 +1,2 @@ +export const englishText = `is In Chrome, back in the @#$ 11 ,][] ye-old days, the sorting algorithm wasn't as good as today. +One of its previous implementations included insertion sort O(n2)`; diff --git a/packages/nlp/test/tsconfig.json b/packages/nlp/test/tsconfig.json new file mode 100644 index 0000000000..1b2f7e64a3 --- /dev/null +++ b/packages/nlp/test/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "outDir": "../dist/test", + "rootDir": "..", + "module": "ES2022", + }, + "references": [ + { + "path": ".." + } + ], + "include": ["../src/**/*.ts", "../test/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/packages/nlp/test/unit/StemmerService.test.ts b/packages/nlp/test/unit/StemmerService.test.ts new file mode 100644 index 0000000000..feeafe909b --- /dev/null +++ b/packages/nlp/test/unit/StemmerService.test.ts @@ -0,0 +1,37 @@ +import "reflect-metadata"; +import { ELanguageCode } from "@snickerdoodlelabs/objects"; + +import { StemmerService } from "@nlp/implementations"; +import { IStemmerService } from "@nlp/interfaces"; +import { englishText } from "@nlp-test/mocks/index.js"; + +class Mocks { + public factory(): IStemmerService { + return new StemmerService(); + } +} +describe("StemmerService", () => { + test("english stop words", async () => { + // Arange + const mocks = new Mocks(); + const service = mocks.factory(); + + // Act + const result = await service.tokenize(ELanguageCode.English, englishText); + + // Assert + expect(result.isOk()).toBeTruthy(); + const tokens = result._unsafeUnwrap(); + // console.log(tokens); + expect(tokens.includes("the")).toBeFalsy(); + expect(tokens.includes("a")).toBeFalsy(); + expect(tokens.includes("an")).toBeFalsy(); + expect(tokens.includes("and")).toBeFalsy(); + expect(tokens.includes("or")).toBeFalsy(); + expect(tokens.includes("it")).toBeFalsy(); + expect(tokens.includes(",")).toBeFalsy(); + expect(tokens.includes(";")).toBeFalsy(); + expect(tokens.includes("[")).toBeFalsy(); + expect(tokens.includes("]")).toBeFalsy(); + }); +}); diff --git a/packages/nlp/tsconfig.json b/packages/nlp/tsconfig.json new file mode 100644 index 0000000000..f71d6150a5 --- /dev/null +++ b/packages/nlp/tsconfig.json @@ -0,0 +1,27 @@ +{ + "extends": "../../tsconfig.build.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src", + "module": "ES2022", + "paths": { + "@nlp/*": ["./packages/nlp/src/*"], + "@nlp-test/*": ["./packages/nlp/test/*"], + }, + }, + "include": [ + "./src/**/*.ts" + ], + "exclude": [ + "node_modules", + "dist" + ], + "references": [ + { + "path": "../objects", + }, + { + "path": "../common-utils" + } + ] +} \ No newline at end of file diff --git a/packages/objects/src/businessObjects/PagedResponse.ts b/packages/objects/src/businessObjects/PagedResponse.ts index ad3ff60370..1cd818e21f 100644 --- a/packages/objects/src/businessObjects/PagedResponse.ts +++ b/packages/objects/src/businessObjects/PagedResponse.ts @@ -1,7 +1,9 @@ +import { PageNumber } from "@objects/primitives/PageNumber.js"; + export class PagedResponse { public constructor( public response: T[], - public page: number, + public page: PageNumber, public pageSize: number, public totalResults: number, ) {} diff --git a/packages/objects/src/businessObjects/PagingRequest.ts b/packages/objects/src/businessObjects/PagingRequest.ts index fadfb69c6f..036a6f88a8 100644 --- a/packages/objects/src/businessObjects/PagingRequest.ts +++ b/packages/objects/src/businessObjects/PagingRequest.ts @@ -1,5 +1,6 @@ +import { PageNumber } from "@objects/primitives/PageNumber.js"; export class PagingRequest { - public constructor(public page: number, public pageSize: number) { + public constructor(public page: PageNumber, public pageSize: number) { this.page = page ?? 1; this.pageSize = pageSize ?? 25; } diff --git a/packages/objects/src/businessObjects/index.ts b/packages/objects/src/businessObjects/index.ts index 434ea589ab..2152c27ffb 100644 --- a/packages/objects/src/businessObjects/index.ts +++ b/packages/objects/src/businessObjects/index.ts @@ -61,5 +61,8 @@ export * from "@objects/businessObjects/WalletNFT.js"; export * from "@objects/businessObjects/events/index.js"; export * from "@objects/businessObjects/rewards/index.js"; export * from "@objects/businessObjects/oauth/index.js"; +export * from "@objects/businessObjects/scraper/index.js"; export * from "@objects/businessObjects/queryResponse/index.js"; export * from "@objects/businessObjects/versioned/index.js"; + +export * from "@objects/businessObjects/shoppingData/index.js"; diff --git a/packages/objects/src/businessObjects/scraper/DomainTask.ts b/packages/objects/src/businessObjects/scraper/DomainTask.ts new file mode 100644 index 0000000000..513bf87250 --- /dev/null +++ b/packages/objects/src/businessObjects/scraper/DomainTask.ts @@ -0,0 +1,6 @@ +import { ETask } from "@objects/enum/scraper/ETask.js"; +import { DomainName } from "@objects/primitives/DomainName.js"; + +export class DomainTask { + constructor(readonly domain: DomainName, readonly taskType: ETask) {} +} diff --git a/packages/objects/src/businessObjects/scraper/IScraperConfig.ts b/packages/objects/src/businessObjects/scraper/IScraperConfig.ts new file mode 100644 index 0000000000..1d62b4dd63 --- /dev/null +++ b/packages/objects/src/businessObjects/scraper/IScraperConfig.ts @@ -0,0 +1,6 @@ +export interface IScraperConfig { + scraper: { + OPENAI_API_KEY: string; + timeout: number; + }; +} diff --git a/packages/objects/src/businessObjects/scraper/ScraperJob.ts b/packages/objects/src/businessObjects/scraper/ScraperJob.ts new file mode 100644 index 0000000000..f24ebcdf88 --- /dev/null +++ b/packages/objects/src/businessObjects/scraper/ScraperJob.ts @@ -0,0 +1,64 @@ +import { DomainTask } from "@objects/businessObjects/scraper/DomainTask.js"; +import { + VersionedObject, + VersionedObjectMigrator, +} from "@objects/businessObjects/versioned/VersionedObject.js"; +import { ETask } from "@objects/enum/scraper/ETask.js"; +import { + DomainName, + HTMLString, + URLString, + UnixTimestamp, + WebPageText, +} from "@objects/primitives/index.js"; + +export class ScraperJob extends VersionedObject { + public static CURRENT_VERSION = 1; + + public constructor( + readonly url: URLString, + readonly html: HTMLString, + readonly domainTask: DomainTask, + readonly startTime: UnixTimestamp, + public endTime: UnixTimestamp | null, + public text: WebPageText | null, + ) { + super(); + } + + public getVersion(): number { + return ScraperJob.CURRENT_VERSION; + } +} + +export class ScraperJobMigrator extends VersionedObjectMigrator { + public getCurrentVersion(): number { + return ScraperJob.CURRENT_VERSION; + } + + protected factory(data: Record): ScraperJob { + const domainTaskSerialized = data["domainTask"] as { + domain: string; + taskType: string; + }; + const domainTask = new DomainTask( + DomainName(domainTaskSerialized["domain"]), + ETask[domainTaskSerialized["taskType"]], + ); + return new ScraperJob( + data["url"] as URLString, + data["html"] as HTMLString, + domainTask, + data["startTime"] as UnixTimestamp, + data["endTime"] as UnixTimestamp, + data["text"] as WebPageText, + ); + } + + protected getUpgradeFunctions(): Map< + number, + (data: Record, version: number) => Record + > { + return new Map(); + } +} diff --git a/packages/objects/src/businessObjects/scraper/index.ts b/packages/objects/src/businessObjects/scraper/index.ts new file mode 100644 index 0000000000..7f80ec8753 --- /dev/null +++ b/packages/objects/src/businessObjects/scraper/index.ts @@ -0,0 +1,3 @@ +export * from "@objects/businessObjects/scraper/DomainTask.js"; +export * from "@objects/businessObjects/scraper/IScraperConfig"; +export * from "@objects/businessObjects/scraper/ScraperJob.js"; diff --git a/packages/objects/src/businessObjects/shoppingData/Product.ts b/packages/objects/src/businessObjects/shoppingData/Product.ts new file mode 100644 index 0000000000..1bbb3ed37f --- /dev/null +++ b/packages/objects/src/businessObjects/shoppingData/Product.ts @@ -0,0 +1,29 @@ +import { VersionedObject } from "@objects/businessObjects/versioned/VersionedObject.js"; +import { + URLString, + UnixTimestamp, + ProductKeyword, +} from "@objects/primitives/index.js"; +export class Product extends VersionedObject { + public static CURRENT_VERSION = 1; + /** + * Brands in later cycles + */ + constructor( + readonly id: string, + readonly name: string, + readonly brand: string, + readonly price: number, + readonly description: string, + readonly image: URLString, + readonly url: string, + readonly category: string, + readonly keywords: ProductKeyword[], + readonly dateCreated: UnixTimestamp, + ) { + super(); + } + public getVersion(): number { + return Product.CURRENT_VERSION; + } +} diff --git a/packages/objects/src/businessObjects/shoppingData/ProductCategories.ts b/packages/objects/src/businessObjects/shoppingData/ProductCategories.ts new file mode 100644 index 0000000000..ffc3480866 --- /dev/null +++ b/packages/objects/src/businessObjects/shoppingData/ProductCategories.ts @@ -0,0 +1,21 @@ +export const UnknownProductCategory = "unknown"; + +export const ProductCategories = [ + UnknownProductCategory, + "Home Improvement", + "Electronics", + "Outdoors", + "Hair Care", + "Automotive Accessories", + "Phone Accessories", + "Fashion", + "Books", + "Sports", + "Food & Grocery", + "Health & Personal Care", + "Toys & Games", + "Office Products", + "Baby", + "Pet Supplies", + "Computers", +]; diff --git a/packages/objects/src/businessObjects/shoppingData/ProductMeta.ts b/packages/objects/src/businessObjects/shoppingData/ProductMeta.ts new file mode 100644 index 0000000000..dafef65745 --- /dev/null +++ b/packages/objects/src/businessObjects/shoppingData/ProductMeta.ts @@ -0,0 +1,9 @@ +import { ProductId, ProductKeyword } from "@objects/primitives/index.js"; + +export class ProductMeta { + constructor( + readonly productId: ProductId, + readonly category: string | null, + readonly keywords: ProductKeyword[], + ) {} +} diff --git a/packages/objects/src/businessObjects/shoppingData/Purchase.ts b/packages/objects/src/businessObjects/shoppingData/Purchase.ts new file mode 100644 index 0000000000..9ef070d619 --- /dev/null +++ b/packages/objects/src/businessObjects/shoppingData/Purchase.ts @@ -0,0 +1,17 @@ +import { VersionedObject } from "@objects/businessObjects/versioned/VersionedObject.js"; +import { UnixTimestamp } from "@objects/primitives/index.js"; +export class Purchase extends VersionedObject { + public static CURRENT_VERSION = 1; + /** + * Brands in later cycles + */ + public constructor( + readonly price: number, + readonly datePurchased: UnixTimestamp, + ) { + super(); + } + public getVersion(): number { + return Purchase.CURRENT_VERSION; + } +} diff --git a/packages/objects/src/businessObjects/shoppingData/PurchasedProduct.ts b/packages/objects/src/businessObjects/shoppingData/PurchasedProduct.ts new file mode 100644 index 0000000000..1f2283760e --- /dev/null +++ b/packages/objects/src/businessObjects/shoppingData/PurchasedProduct.ts @@ -0,0 +1,73 @@ +import { + VersionedObject, + VersionedObjectMigrator, +} from "@objects/businessObjects/versioned/VersionedObject.js"; +import { ELanguageCode } from "@objects/enum/ELanguageCode.js"; +import { + DomainName, + URLString, + UnixTimestamp, + PurchaseId, + ProductKeyword, +} from "@objects/primitives/index.js"; + +/** + * We will use this class for now to reduce development complexity and also store this in the persistence for now. Later we will decompose this into a Product and a Purchase + */ +export class PurchasedProduct extends VersionedObject { + public static CURRENT_VERSION = 1; + /** + * Brands in later cycles + */ + constructor( + readonly marketPlace: DomainName, + readonly language: ELanguageCode, + readonly id: PurchaseId, + readonly name: string, + public brand: string | null, + readonly price: number, + public datePurchased: UnixTimestamp, + + readonly dateCreated: UnixTimestamp, + public description: string | null, + public image: URLString | null, + public url: URLString | null, + public category: string, + public keywords: ProductKeyword[] | null, + ) { + super(); + } + public getVersion(): number { + return PurchasedProduct.CURRENT_VERSION; + } +} + +export class PurchasedProductMigrator extends VersionedObjectMigrator { + public getCurrentVersion(): number { + return PurchasedProduct.CURRENT_VERSION; + } + protected factory(data: Record): PurchasedProduct { + return new PurchasedProduct( + DomainName(data["marketPlace"] as string), + ELanguageCode[data["languageCode"] as string], + PurchaseId(data["id"] as string), + data["name"] as string, + data["brand"] as string, + data["price"] as number, + data["datePurchased"] as UnixTimestamp, + + data["dateCreated"] as UnixTimestamp, + data["description"] as string, + data["image"] as URLString, + data["url"] as URLString, + data["category"] as string, + data["keywords"] as ProductKeyword[], + ); + } + protected getUpgradeFunctions(): Map< + number, + (data: Record, version: number) => Record + > { + return new Map(); + } +} diff --git a/packages/objects/src/businessObjects/shoppingData/index.ts b/packages/objects/src/businessObjects/shoppingData/index.ts new file mode 100644 index 0000000000..a265eb2e96 --- /dev/null +++ b/packages/objects/src/businessObjects/shoppingData/index.ts @@ -0,0 +1,5 @@ +export * from "@objects/businessObjects/shoppingData/Product.js"; +export * from "@objects/businessObjects/shoppingData/ProductCategories.js"; +export * from "@objects/businessObjects/shoppingData/ProductMeta.js"; +export * from "@objects/businessObjects/shoppingData/Purchase.js"; +export * from "@objects/businessObjects/shoppingData/PurchasedProduct.js"; diff --git a/packages/objects/src/businessObjects/versioned/ShoppingDataConnectionStatus.ts b/packages/objects/src/businessObjects/versioned/ShoppingDataConnectionStatus.ts new file mode 100644 index 0000000000..c7b28a4681 --- /dev/null +++ b/packages/objects/src/businessObjects/versioned/ShoppingDataConnectionStatus.ts @@ -0,0 +1,57 @@ +import { + VersionedObject, + VersionedObjectMigrator, +} from "@objects/businessObjects/versioned/VersionedObject.js"; +import { EKnownDomains } from "@objects/enum"; + +export class ShoppingDataConnectionStatus extends VersionedObject { + public static CURRENT_VERSION = 1; + public constructor(public type: EKnownDomains, public isConnected: boolean) { + super(); + } + public getVersion(): number { + return ShoppingDataConnectionStatus.CURRENT_VERSION; + } +} + +export class ShoppingDataConnectionStatusMigrator extends VersionedObjectMigrator { + public getCurrentVersion(): number { + return ShoppingDataConnectionStatus.CURRENT_VERSION; + } + + protected amazonMigrator: AmazonMigrator; + + public constructor() { + super(); + this.amazonMigrator = new AmazonMigrator(); + } + + protected factory( + data: Record, + ): ShoppingDataConnectionStatus { + switch (data["type"]) { + case EKnownDomains.Amazon: + return this.amazonMigrator.factory(data); + } + return this.amazonMigrator.factory(data); + } + + protected getUpgradeFunctions(): Map< + number, + (data: Record, version: number) => Record + > { + return new Map(); + } +} + +export class Amazon extends ShoppingDataConnectionStatus { + public constructor(public isConnected: boolean) { + super(EKnownDomains.Amazon, isConnected); + } +} + +export class AmazonMigrator { + public factory(data: Record): Amazon { + return new Amazon(data["isConnected"] as boolean); + } +} diff --git a/packages/objects/src/businessObjects/versioned/index.ts b/packages/objects/src/businessObjects/versioned/index.ts index dc41d45bca..cbe93145d4 100644 --- a/packages/objects/src/businessObjects/versioned/index.ts +++ b/packages/objects/src/businessObjects/versioned/index.ts @@ -15,6 +15,7 @@ export * from "@objects/businessObjects/versioned/QuestionnaireMigrator.js"; export * from "@objects/businessObjects/versioned/ReceivingAccount.js"; export * from "@objects/businessObjects/versioned/RejectedInvitation.js"; export * from "@objects/businessObjects/versioned/RestoredBackup.js"; +export * from "@objects/businessObjects/versioned/ShoppingDataConnectionStatus"; export * from "@objects/businessObjects/versioned/SiteVisit.js"; export * from "@objects/businessObjects/versioned/SocialGroupProfile.js"; export * from "@objects/businessObjects/versioned/SocialProfile.js"; diff --git a/packages/objects/src/enum/StorageKey.ts b/packages/objects/src/enum/StorageKey.ts index e8dbf31072..3ce47ec291 100644 --- a/packages/objects/src/enum/StorageKey.ts +++ b/packages/objects/src/enum/StorageKey.ts @@ -21,6 +21,8 @@ export enum ERecordKey { QUERY_STATUS = "SD_QueryStatus", DOMAIN_CREDENTIALS = "SD_DomainCredentials", REJECTED_INVITATIONS = "SD_RejectedInvitations", + PURCHASED_PRODUCT = "SD_PurchasedProduct", + SHOPPING_DATA_CONNECTION_STATUS = "SD_ShoppingDataConnectionStatus", OPTED_IN_INVITATIONS = "SD_OptedInInvitations", QUESTIONNAIRES = "SD_Questionnaires", diff --git a/packages/objects/src/enum/index.ts b/packages/objects/src/enum/index.ts index c7cfb5bd5d..be65372ead 100644 --- a/packages/objects/src/enum/index.ts +++ b/packages/objects/src/enum/index.ts @@ -40,3 +40,5 @@ export * from "@objects/enum/ETag.js"; export * from "@objects/enum/ETimePeriods.js"; export * from "@objects/enum/EWalletDataType.js"; export * from "@objects/enum/StorageKey.js"; + +export * from "@objects/enum/scraper/index.js"; diff --git a/packages/ai-scraper/src/interfaces/enums/EKnownDomains.ts b/packages/objects/src/enum/scraper/EKnownDomains.ts similarity index 100% rename from packages/ai-scraper/src/interfaces/enums/EKnownDomains.ts rename to packages/objects/src/enum/scraper/EKnownDomains.ts diff --git a/packages/objects/src/enum/scraper/EScraperJobStatus.ts b/packages/objects/src/enum/scraper/EScraperJobStatus.ts new file mode 100644 index 0000000000..37e6bcfc3d --- /dev/null +++ b/packages/objects/src/enum/scraper/EScraperJobStatus.ts @@ -0,0 +1,7 @@ +export enum EScraperJobStatus { + Initiated = 1, + PreProcessed = 2, + PreProcessFailed = 3, + Extracted = 4, + ExtractionFailed = 5, +} diff --git a/packages/ai-scraper/src/interfaces/enums/ETask.ts b/packages/objects/src/enum/scraper/ETask.ts similarity index 100% rename from packages/ai-scraper/src/interfaces/enums/ETask.ts rename to packages/objects/src/enum/scraper/ETask.ts diff --git a/packages/objects/src/enum/scraper/index.ts b/packages/objects/src/enum/scraper/index.ts new file mode 100644 index 0000000000..1997f09c6f --- /dev/null +++ b/packages/objects/src/enum/scraper/index.ts @@ -0,0 +1,3 @@ +export * from "@objects/enum/scraper/EKnownDomains.js"; +export * from "@objects/enum/scraper/ETask.js"; +export * from "@objects/enum/scraper/EScraperJobStatus.js"; diff --git a/packages/objects/src/errors/InvalidURLError.ts b/packages/objects/src/errors/InvalidURLError.ts new file mode 100644 index 0000000000..a999d8f7cc --- /dev/null +++ b/packages/objects/src/errors/InvalidURLError.ts @@ -0,0 +1,9 @@ +import { BaseError } from "@objects/errors/BaseError.js"; +import errorCodes from "@objects/errors/errorCodes.js"; + +export class InvalidURLError extends BaseError { + protected errorCode: string = errorCodes[InvalidURLError.name]; + constructor(message: string, public src?: unknown) { + super(message, 500, errorCodes[InvalidURLError.name], src, false); + } +} diff --git a/packages/objects/src/errors/LLMError.ts b/packages/objects/src/errors/LLMError.ts new file mode 100644 index 0000000000..72e074c4ea --- /dev/null +++ b/packages/objects/src/errors/LLMError.ts @@ -0,0 +1,9 @@ +import { BaseError } from "@objects/errors/BaseError.js"; +import errorCodes from "@objects/errors/errorCodes.js"; + +export class LLMError extends BaseError { + protected errorCode: string = errorCodes[LLMError.name]; + constructor(message: string, public src?: unknown) { + super(message, 500, errorCodes[LLMError.name], src, false); + } +} diff --git a/packages/objects/src/errors/LLMMaxTokensExceededError.ts b/packages/objects/src/errors/LLMMaxTokensExceededError.ts new file mode 100644 index 0000000000..d1b79ebc0c --- /dev/null +++ b/packages/objects/src/errors/LLMMaxTokensExceededError.ts @@ -0,0 +1,9 @@ +import { BaseError } from "@objects/errors/BaseError.js"; +import errorCodes from "@objects/errors/errorCodes.js"; + +export class LLMMaxTokensExceededError extends BaseError { + protected errorCode: string = errorCodes[LLMMaxTokensExceededError.name]; + constructor(message: string, public src?: unknown) { + super(message, 500, errorCodes[LLMMaxTokensExceededError.name], src, false); + } +} diff --git a/packages/objects/src/errors/NLPError.ts b/packages/objects/src/errors/NLPError.ts new file mode 100644 index 0000000000..2902b99fe8 --- /dev/null +++ b/packages/objects/src/errors/NLPError.ts @@ -0,0 +1,9 @@ +import { BaseError } from "@objects/errors/BaseError.js"; +import errorCodes from "@objects/errors/errorCodes.js"; + +export class NLPError extends BaseError { + protected errorCode: string = errorCodes[NLPError.name]; + constructor(message: string, public src?: unknown) { + super(message, 500, errorCodes[NLPError.name], src, false); + } +} diff --git a/packages/objects/src/errors/ScraperError.ts b/packages/objects/src/errors/ScraperError.ts new file mode 100644 index 0000000000..ed1191de9e --- /dev/null +++ b/packages/objects/src/errors/ScraperError.ts @@ -0,0 +1,9 @@ +import { BaseError } from "@objects/errors/BaseError.js"; +import errorCodes from "@objects/errors/errorCodes.js"; + +export class ScraperError extends BaseError { + protected errorCode: string = errorCodes[ScraperError.name]; + constructor(message: string, public src?: unknown) { + super(message, 500, errorCodes[ScraperError.name], src, false); + } +} diff --git a/packages/objects/src/errors/errorCodes.ts b/packages/objects/src/errors/errorCodes.ts index a08cd9b55e..42ca3f8f74 100644 --- a/packages/objects/src/errors/errorCodes.ts +++ b/packages/objects/src/errors/errorCodes.ts @@ -13,6 +13,7 @@ const errorCodes = { DataWalletLockedError: "ERR_DATA_WALLET_LOCKED", InvalidParametersError: "INVALID_PARAMETERS_ERROR", InvalidSignatureError: "ERR_INVALID_SIGNATURE", + InvalidURLError: "ERR_INVALID_URL", IPFSError: "ERR_IPFS", KeyGenerationError: "ERR_CRYPTO_KEY_GENERATION", MinimalForwarderContractError: "ERR_MINIMAL_FORWARDER_CONTRACT", @@ -46,6 +47,10 @@ const errorCodes = { InvalidAddressError: "ERR_INVALID_ADDRESS", ExecutionRevertedError: "ERR_EXECUTION_REVERTED", ProofError: "ERR_PROOF", + ScraperError: "ERR_SCRAPER", + NLPError: "ERR_NLP", + LLMError: "ERR_LLM", + LLMMaxTokensExceededError: "ERR_LLM_MAX_TOKENS_EXCEEDED", SingerUnavailableError: "ERR_SIGNER_UNAVAILABLE", //SDQL errors OperandTypeError: "ER_SDQL_OPERAND_TYPE", diff --git a/packages/objects/src/errors/index.ts b/packages/objects/src/errors/index.ts index 5852979587..5b357473cb 100644 --- a/packages/objects/src/errors/index.ts +++ b/packages/objects/src/errors/index.ts @@ -12,11 +12,14 @@ export * from "@objects/errors/GasPriceError.js"; export * from "@objects/errors/IBlockchainError.js"; export * from "@objects/errors/InvalidParametersError.js"; export * from "@objects/errors/InvalidSignatureError.js"; +export * from "@objects/errors/InvalidURLError.js"; export * from "@objects/errors/IPFSError.js"; export * from "@objects/errors/KeyGenerationError.js"; +export * from "@objects/errors/LLMError.js"; export * from "@objects/errors/MethodSupportError.js"; export * from "@objects/errors/MissingASTError.js"; export * from "@objects/errors/MissingWalletDataTypeError.js"; +export * from "@objects/errors/NLPError.js"; export * from "@objects/errors/OAuthError.js"; export * from "@objects/errors/ParserTypeNotImplementedError.js"; export * from "@objects/errors/PermissionError.js"; @@ -26,12 +29,12 @@ export * from "@objects/errors/ProviderRpcError.js"; export * from "@objects/errors/ProxyError.js"; export * from "@objects/errors/QueryExpiredError.js"; export * from "@objects/errors/QueryFormatError.js"; +export * from "@objects/errors/ScraperError.js"; export * from "@objects/errors/ServerRewardError.js"; export * from "@objects/errors/TransactionResponseError"; export * from "@objects/errors/TwitterError.js"; export * from "@objects/errors/UnauthorizedError.js"; export * from "@objects/errors/UninitializedError.js"; export * from "@objects/errors/UnsupportedLanguageError.js"; - export * from "@objects/errors/blockchain/index.js"; export * from "@objects/errors/sdqlErrors/index.js"; diff --git a/packages/objects/src/interfaces/IConfigOverrides.ts b/packages/objects/src/interfaces/IConfigOverrides.ts index b0d768499c..511c6bef8c 100644 --- a/packages/objects/src/interfaces/IConfigOverrides.ts +++ b/packages/objects/src/interfaces/IConfigOverrides.ts @@ -69,4 +69,8 @@ export interface IConfigOverrides { iframeURL?: URLString; debug?: boolean; queryPerformanceMetricsLimit?: number; + scraper?: { + OPENAI_API_KEY: string; + timeout: number; + }; } diff --git a/packages/objects/src/interfaces/ISdlDataWallet.ts b/packages/objects/src/interfaces/ISdlDataWallet.ts index 504ce95123..15f771dbcb 100644 --- a/packages/objects/src/interfaces/ISdlDataWallet.ts +++ b/packages/objects/src/interfaces/ISdlDataWallet.ts @@ -30,6 +30,8 @@ import { ICoreIntegrationMethods, ICoreTwitterMethods, IMetricsMethods, + IPurchaseMethods, + IScraperNavigationMethods, IStorageMethods, INftMethods, IQuestionnaireMethods, @@ -192,6 +194,30 @@ export type IProxyStorageMethods = { >; }; +export type IProxyScraperNavigationMethods = { + [key in Exclude< + FunctionKeys, + | "getYears" + | "getPageCount" + | "getOrderHistoryPageByYear" + | "getOrderHistoryPage" + >]: ( + ...args: [...Parameters] + ) => ResultAsync< + ReturnType, + ProxyError + >; +}; + +export type IProxyPurchaseMethods = { + [key in FunctionKeys]: ( + ...args: [...Parameters] + ) => ResultAsync< + GetResultAsyncValueType>, + ProxyError + >; +}; + // This stuff is left in for reference- I'm still working on improving these // methods and // type test = Parameters; @@ -360,6 +386,7 @@ export interface ISdlDataWallet { storage: IProxyStorageMethods; nft: INftProxyMethods; events: ISnickerdoodleCoreEvents; + purchase: IProxyPurchaseMethods; questionnaire: IProxyQuestionnaireMethods; } diff --git a/packages/objects/src/interfaces/ISnickerdoodleCore.ts b/packages/objects/src/interfaces/ISnickerdoodleCore.ts index c44f1f5eb1..3fec3757db 100644 --- a/packages/objects/src/interfaces/ISnickerdoodleCore.ts +++ b/packages/objects/src/interfaces/ISnickerdoodleCore.ts @@ -35,6 +35,9 @@ import { SiteVisitsMap, TransactionFlowInsight, OptInInfo, + DomainTask, + PurchasedProduct, + ShoppingDataConnectionStatus, NftRepositoryCache, WalletNFTData, WalletNFTHistory, @@ -49,6 +52,7 @@ import { ECloudStorageType, EDataWalletPermission, EInvitationStatus, + ELanguageCode, } from "@objects/enum/index.js"; import { AccountIndexingError, @@ -80,6 +84,9 @@ import { DuplicateIdInSchema, MissingWalletDataTypeError, ParserError, + ScraperError, + InvalidURLError, + LLMError, MethodSupportError, } from "@objects/errors/index.js"; import { IConsentCapacity } from "@objects/interfaces/IConsentCapacity.js"; @@ -118,6 +125,9 @@ import { URLString, BlockNumber, RefreshToken, + HTMLString, + PageNumber, + Year, JSONString, } from "@objects/primitives/index.js"; /** @@ -716,6 +726,48 @@ export interface IStorageMethods { ): ResultAsync; } +export interface IScraperMethods { + scrape( + url: URLString, + html: HTMLString, + suggestedDomainTask: DomainTask, + ): ResultAsync; + classifyURL( + url: URLString, + language: ELanguageCode, + ): ResultAsync; +} + +export interface IScraperNavigationMethods { + amazon: { + getOrderHistoryPage(lang: ELanguageCode, page: PageNumber): URLString; + getYears(html: HTMLString): Year[]; + getOrderHistoryPageByYear( + lang: ELanguageCode, + year: Year, + page: PageNumber, + ): URLString; + getPageCount(html: HTMLString, year: Year): number; + }; +} + +export interface IPurchaseMethods { + getPurchasedProducts(): ResultAsync; + getByMarketplace( + marketPlace: DomainName, + ): ResultAsync; + getByMarketplaceAndDate( + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ): ResultAsync; + getShoppingDataConnectionStatus(): ResultAsync< + ShoppingDataConnectionStatus[], + PersistenceError + >; + setShoppingDataConnectionStatus( + ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ): ResultAsync; +} export interface IQuestionnaireMethods { /** * Returns a list of questionnaires that the user can complete (that do not already have answers), @@ -1088,6 +1140,9 @@ export interface ISnickerdoodleCore { twitter: ICoreTwitterMethods; metrics: IMetricsMethods; storage: IStorageMethods; + purchase: IPurchaseMethods; + scraper: IScraperMethods; + scraperNavigation: IScraperNavigationMethods; nft: INftMethods; questionnaire: IQuestionnaireMethods; } diff --git a/packages/objects/src/primitives/HTMLString.ts b/packages/objects/src/primitives/HTMLString.ts new file mode 100644 index 0000000000..a914934524 --- /dev/null +++ b/packages/objects/src/primitives/HTMLString.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type HTMLString = Brand; +export const HTMLString = make(); diff --git a/packages/objects/src/primitives/Month.ts b/packages/objects/src/primitives/Month.ts new file mode 100644 index 0000000000..a9f5f032db --- /dev/null +++ b/packages/objects/src/primitives/Month.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type Month = Brand; +export const Month = make(); diff --git a/packages/objects/src/primitives/PageNumber.ts b/packages/objects/src/primitives/PageNumber.ts new file mode 100644 index 0000000000..728b2630c7 --- /dev/null +++ b/packages/objects/src/primitives/PageNumber.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type PageNumber = Brand; +export const PageNumber = make(); diff --git a/packages/objects/src/primitives/WebPageText.ts b/packages/objects/src/primitives/WebPageText.ts new file mode 100644 index 0000000000..b38786494a --- /dev/null +++ b/packages/objects/src/primitives/WebPageText.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type WebPageText = Brand; +export const WebPageText = make(); diff --git a/packages/objects/src/primitives/Year.ts b/packages/objects/src/primitives/Year.ts new file mode 100644 index 0000000000..cfdae9ddda --- /dev/null +++ b/packages/objects/src/primitives/Year.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type Year = Brand; +export const Year = make(); diff --git a/packages/objects/src/primitives/index.ts b/packages/objects/src/primitives/index.ts index 99bb34be5b..a3121fe934 100644 --- a/packages/objects/src/primitives/index.ts +++ b/packages/objects/src/primitives/index.ts @@ -46,6 +46,7 @@ export * from "@objects/primitives/HostName.js"; export * from "@objects/primitives/HexColorString.js"; export * from "@objects/primitives/HexString.js"; export * from "@objects/primitives/HexString32.js"; +export * from "@objects/primitives/HTMLString.js"; export * from "@objects/primitives/ISDQLAnyEvaluatableString.js"; export * from "@objects/primitives/ISDQLConditionString.js"; export * from "@objects/primitives/ISDQLExpressionString.js"; @@ -60,6 +61,7 @@ export * from "@objects/primitives/JsonWebToken.js"; export * from "@objects/primitives/LanguageCode.js"; export * from "@objects/primitives/MarketplaceTag.js"; export * from "@objects/primitives/MillisecondTimestamp.js"; +export * from "@objects/primitives/Month.js"; export * from "@objects/primitives/NftAddressesWithTokenId.js"; export * from "@objects/primitives/NftIdWithMeasurementDate.js"; export * from "@objects/primitives/OAuthAuthorizationCode.js"; @@ -69,6 +71,7 @@ export * from "@objects/primitives/PEMEncodedRSAPrivateKey.js"; export * from "@objects/primitives/PEMEncodedRSAPublicKey.js"; export * from "@objects/primitives/ProofString.js"; export * from "@objects/primitives/ProviderUrl.js"; +export * from "@objects/primitives/PageNumber.js"; export * from "@objects/primitives/PublicKey.js"; export * from "@objects/primitives/QueryTypePermissionMap.js"; export * from "@objects/primitives/QueryTypes.js"; @@ -105,3 +108,8 @@ export * from "@objects/primitives/Username.js"; export * from "@objects/primitives/Version.js"; export * from "@objects/primitives/VolatileStorageKey.js"; export * from "@objects/primitives/Web2Credential.js"; +export * from "@objects/primitives/WebPageText.js"; +export * from "@objects/primitives/Year.js"; + +export * from "@objects/primitives/scraper/index.js"; +export * from "@objects/primitives/shoppingData/index.js"; diff --git a/packages/objects/src/primitives/scraper/Exemplar.ts b/packages/objects/src/primitives/scraper/Exemplar.ts new file mode 100644 index 0000000000..b154915804 --- /dev/null +++ b/packages/objects/src/primitives/scraper/Exemplar.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type Exemplar = Brand; +export const Exemplar = make(); diff --git a/packages/ai-scraper/src/interfaces/primitives/Keyword.ts b/packages/objects/src/primitives/scraper/Keyword.ts similarity index 100% rename from packages/ai-scraper/src/interfaces/primitives/Keyword.ts rename to packages/objects/src/primitives/scraper/Keyword.ts diff --git a/packages/objects/src/primitives/scraper/LLMAnswerStructure.ts b/packages/objects/src/primitives/scraper/LLMAnswerStructure.ts new file mode 100644 index 0000000000..7503fa969d --- /dev/null +++ b/packages/objects/src/primitives/scraper/LLMAnswerStructure.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type LLMAnswerStructure = Brand; +export const LLMAnswerStructure = make(); diff --git a/packages/objects/src/primitives/scraper/LLMData.ts b/packages/objects/src/primitives/scraper/LLMData.ts new file mode 100644 index 0000000000..ed9e905dcd --- /dev/null +++ b/packages/objects/src/primitives/scraper/LLMData.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type LLMData = Brand; +export const LLMData = make(); diff --git a/packages/objects/src/primitives/scraper/LLMQuestion.ts b/packages/objects/src/primitives/scraper/LLMQuestion.ts new file mode 100644 index 0000000000..379d867df0 --- /dev/null +++ b/packages/objects/src/primitives/scraper/LLMQuestion.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type LLMQuestion = Brand; +export const LLMQuestion = make(); diff --git a/packages/objects/src/primitives/scraper/LLMResponse.ts b/packages/objects/src/primitives/scraper/LLMResponse.ts new file mode 100644 index 0000000000..8b3615a09d --- /dev/null +++ b/packages/objects/src/primitives/scraper/LLMResponse.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type LLMResponse = Brand; +export const LLMResponse = make(); diff --git a/packages/objects/src/primitives/scraper/LLMRole.ts b/packages/objects/src/primitives/scraper/LLMRole.ts new file mode 100644 index 0000000000..03aa6f15cb --- /dev/null +++ b/packages/objects/src/primitives/scraper/LLMRole.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type LLMRole = Brand; +export const LLMRole = make(); diff --git a/packages/objects/src/primitives/scraper/Prompt.ts b/packages/objects/src/primitives/scraper/Prompt.ts new file mode 100644 index 0000000000..02c750672b --- /dev/null +++ b/packages/objects/src/primitives/scraper/Prompt.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type Prompt = Brand; +export const Prompt = make(); diff --git a/packages/objects/src/primitives/scraper/index.ts b/packages/objects/src/primitives/scraper/index.ts new file mode 100644 index 0000000000..1a06312486 --- /dev/null +++ b/packages/objects/src/primitives/scraper/index.ts @@ -0,0 +1,8 @@ +export * from "@objects/primitives/scraper/Exemplar.js"; +export * from "@objects/primitives/scraper/Keyword.js"; +export * from "@objects/primitives/scraper/LLMAnswerStructure.js"; +export * from "@objects/primitives/scraper/LLMData.js"; +export * from "@objects/primitives/scraper/LLMQuestion.js"; +export * from "@objects/primitives/scraper/LLMResponse.js"; +export * from "@objects/primitives/scraper/LLMRole.js"; +export * from "@objects/primitives/scraper/Prompt.js"; diff --git a/packages/objects/src/primitives/shoppingData/ProductId.ts b/packages/objects/src/primitives/shoppingData/ProductId.ts new file mode 100644 index 0000000000..2b9edf13ca --- /dev/null +++ b/packages/objects/src/primitives/shoppingData/ProductId.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type ProductId = Brand; +export const ProductId = make(); diff --git a/packages/objects/src/primitives/shoppingData/ProductKeyword.ts b/packages/objects/src/primitives/shoppingData/ProductKeyword.ts new file mode 100644 index 0000000000..c94746f2b5 --- /dev/null +++ b/packages/objects/src/primitives/shoppingData/ProductKeyword.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type ProductKeyword = Brand; +export const ProductKeyword = make(); diff --git a/packages/objects/src/primitives/shoppingData/PurchaseId.ts b/packages/objects/src/primitives/shoppingData/PurchaseId.ts new file mode 100644 index 0000000000..f73cfc20c5 --- /dev/null +++ b/packages/objects/src/primitives/shoppingData/PurchaseId.ts @@ -0,0 +1,4 @@ +import { Brand, make } from "ts-brand"; + +export type PurchaseId = Brand; +export const PurchaseId = make(); diff --git a/packages/objects/src/primitives/shoppingData/index.ts b/packages/objects/src/primitives/shoppingData/index.ts new file mode 100644 index 0000000000..988f8fb7fc --- /dev/null +++ b/packages/objects/src/primitives/shoppingData/index.ts @@ -0,0 +1,3 @@ +export * from "@objects/primitives/shoppingData/ProductKeyword.js"; +export * from "@objects/primitives/shoppingData/ProductId.js"; +export * from "@objects/primitives/shoppingData/PurchaseId.js"; diff --git a/packages/persistence/package.json b/packages/persistence/package.json index a90cc4b511..739f660d9b 100644 --- a/packages/persistence/package.json +++ b/packages/persistence/package.json @@ -41,6 +41,7 @@ "@snickerdoodlelabs/common-utils": "workspace:^", "@snickerdoodlelabs/node-utils": "workspace:^", "@snickerdoodlelabs/objects": "workspace:^", + "@snickerdoodlelabs/shopping-data": "workspace:^", "@snickerdoodlelabs/utils": "workspace:^", "ethers": "^6.10.0", "inversify": "^6.0.2", diff --git a/packages/persistence/src/IPersistence.ts b/packages/persistence/src/IPersistence.ts new file mode 100644 index 0000000000..6ddad9b866 --- /dev/null +++ b/packages/persistence/src/IPersistence.ts @@ -0,0 +1,64 @@ +import { + EBackupPriority, + EFieldKey, + ERecordKey, + PersistenceError, + VersionedObject, + VolatileStorageKey, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +import { IVolatileCursor } from "@persistence/volatile/IVolatileCursor.js"; + +export interface IPersistence { + getObject( + recordKey: ERecordKey, + key: VolatileStorageKey, + ): ResultAsync; + getCursor( + recordKey: ERecordKey, + indexName?: string, + query?: IDBValidKey | IDBKeyRange, + direction?: IDBCursorDirection | undefined, + mode?: IDBTransactionMode, + ): ResultAsync, PersistenceError>; + getAll( + recordKey: ERecordKey, + indexName?: string, + ): ResultAsync; + getAllByIndex( + recordKey: ERecordKey, + indexName: string, + query: IDBValidKey | IDBKeyRange, + priority?: EBackupPriority, + ): ResultAsync; + getAllByMultiIndex( + recordKey: ERecordKey, + indices: string[], + values: IDBValidKey | IDBKeyRange, + ): ResultAsync; + getAllKeys( + recordKey: ERecordKey, + indexName?: string, + query?: IDBValidKey | IDBKeyRange, + count?: number | undefined, + ): ResultAsync; + updateRecord( + recordKey: ERecordKey, + value: T, + ): ResultAsync; + deleteRecord( + recordKey: ERecordKey, + key: VolatileStorageKey, + ): ResultAsync; + // #endregion + + // #region Fields + getField(fieldKey: EFieldKey): ResultAsync; + updateField( + fieldKey: EFieldKey, + value: object, + ): ResultAsync; +} + +export const IPersistenceType = Symbol.for("IPersistence"); diff --git a/packages/persistence/src/database/objectStores.ts b/packages/persistence/src/database/objectStores.ts index 416a252865..9a9c73a9d3 100644 --- a/packages/persistence/src/database/objectStores.ts +++ b/packages/persistence/src/database/objectStores.ts @@ -21,6 +21,8 @@ import { InvitationForStorageMigrator, QuestionnaireMigrator, QuestionnaireHistoryMigrator, + ShoppingDataConnectionStatusMigrator, + PurchasedProductMigrator, } from "@snickerdoodlelabs/objects"; import { IPersistenceConfig } from "@persistence/IPersistenceConfig"; @@ -267,5 +269,33 @@ export const getObjectStoreDefinitions = (config?: IPersistenceConfig) => { [[["deleted", "id", "measurementDate"], false]], ), ], + [ + ERecordKey.PURCHASED_PRODUCT, + new VolatileTableIndex( + ERecordKey.PURCHASED_PRODUCT, + ["id", false], + new PurchasedProductMigrator(), + EBackupPriority.NORMAL, + config?.dataWalletBackupIntervalMS ?? testTimeValue, + config?.backupChunkSizeTarget ?? testTimeValue, + [ + ["marketPlace", false], + ["category", false], + ["datePurchased", false], + [["marketPlace", "datePurchased"], false], + ], + ), + ], + [ + ERecordKey.SHOPPING_DATA_CONNECTION_STATUS, + new VolatileTableIndex( + ERecordKey.SHOPPING_DATA_CONNECTION_STATUS, + ["type", false], + new ShoppingDataConnectionStatusMigrator(), + EBackupPriority.NORMAL, + config?.dataWalletBackupIntervalMS ?? testTimeValue, + config?.backupChunkSizeTarget ?? testTimeValue, + ), + ], ]); }; diff --git a/packages/persistence/src/index.ts b/packages/persistence/src/index.ts index 0742d45f65..a8abc32774 100644 --- a/packages/persistence/src/index.ts +++ b/packages/persistence/src/index.ts @@ -1,3 +1,4 @@ +export * from "@persistence/IPersistence.js"; export * from "@persistence/database/index.js"; export * from "@persistence/IPersistenceConfig.js"; diff --git a/packages/persistence/tsconfig.json b/packages/persistence/tsconfig.json index ad3b6ef826..a33aa4419a 100644 --- a/packages/persistence/tsconfig.json +++ b/packages/persistence/tsconfig.json @@ -23,6 +23,9 @@ { "path": "../objects" }, + { + "path": "../shopping-data" + }, { "path": "../utils" }, diff --git a/packages/shared-components/src/v2/constants/index.ts b/packages/shared-components/src/v2/constants/index.ts index 1b585eeb63..894ebac66a 100644 --- a/packages/shared-components/src/v2/constants/index.ts +++ b/packages/shared-components/src/v2/constants/index.ts @@ -1,4 +1,8 @@ -import { EWalletDataType } from "@snickerdoodlelabs/objects"; +import { + EKnownDomains, + EWalletDataType, + URLString, +} from "@snickerdoodlelabs/objects"; export const FF_SUPPORTED_PERMISSIONS: { description: string; @@ -53,3 +57,16 @@ export const FF_SUPPORTED_PERMISSIONS: { export const FF_SUPPORTED_ALL_PERMISSIONS: EWalletDataType[] = FF_SUPPORTED_PERMISSIONS.map((item) => item.key).flat(); + +export const SCRAPING_URLS: { key: EKnownDomains; url: URLString }[] = [ + { + key: EKnownDomains.Amazon, + url: URLString( + "https://www.amazon.com/gp/css/order-history?ie=UTF8&ref_=nav_AccountFlyout_orders", + ), + }, +]; + +export const SCRAPING_INDEX: Map = new Map([ + [EKnownDomains.Amazon, "AmazonShoppingdataSDL"], +]); diff --git a/packages/shared-components/src/v2/index.ts b/packages/shared-components/src/v2/index.ts index 003223055c..b2b7c7d149 100644 --- a/packages/shared-components/src/v2/index.ts +++ b/packages/shared-components/src/v2/index.ts @@ -1,5 +1,5 @@ export * from "@shared-components/v2/components"; -export * from "@shared-components/v2/hooks"; export * from "@shared-components/v2/constants"; +export * from "@shared-components/v2/hooks"; export * from "@shared-components/v2/theme"; export * from "@shared-components/v2/widgets"; diff --git a/packages/shopping-data/jest.config.ts b/packages/shopping-data/jest.config.ts new file mode 100644 index 0000000000..b58d7a0c9c --- /dev/null +++ b/packages/shopping-data/jest.config.ts @@ -0,0 +1,22 @@ +import type { Config } from "@jest/types"; + +const config: Config.InitialOptions = { + testEnvironment: "node", + testMatch: ["/dist/test/**/*.test.js"], + + // This does not seem to support blacklisting any folder which means we can't enable parent directory and disable child + // We should be using peer directories for coverage and non-coverage tests. + collectCoverageFrom: [ + // Enabling following means we can't disable src/tests from coverage report + // "/src/**/*.ts", + + // Add other allowed folders to the list below. + "/dist/implementations/**/*.js", + "!/src/implementations/**/index.ts", + + // Disabled because we don't want it to end up in coverage report, + // "/src/tests/**/*.ts", + ], +}; + +export default config; diff --git a/packages/shopping-data/package.json b/packages/shopping-data/package.json new file mode 100644 index 0000000000..a14986f118 --- /dev/null +++ b/packages/shopping-data/package.json @@ -0,0 +1,52 @@ +{ + "name": "@snickerdoodlelabs/shopping-data", + "version": "0.0.17", + "description": "Shopping data library for Data Wallet extension", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/SnickerdoodleLabs/protocol.git" + }, + "bugs": { + "url": "https://github.com/SnickerdoodleLabs/protocol/issues" + }, + "homepage": "https://github.com/SnickerdoodleLabs/protocol/tree/master/documentation/shopping-data", + "author": "Golam Muktadir ", + "keywords": [ + "Snickerdoodle", + "SDQL" + ], + "main": "dist/index.js", + "files": [ + "dist", + "!dist/test", + "!dist/tsconfig.tsbuildinfo", + "!test", + "!src", + "!tsconfig.json" + ], + "scripts": { + "alias": "tsc-alias && tsc-alias -p test/tsconfig.json", + "alias-with-copyfiles": "yarn copy-files && tsc-alias", + "build": "yarn clean && yarn compile", + "clean": "npx rimraf dist tsconfig.tsbuildinfo", + "compile": "npx tsc --build && cd ../.. && yarn alias", + "copy-files": "copyfiles -u 1 src/**/*.d.ts dist/", + "prepare": "yarn build", + "prepublish": "yarn build", + "test": "NODE_OPTIONS=--experimental-vm-modules npx jest --maxWorkers=50% --coverage --passWithNoTests" + }, + "type": "module", + "types": "dist/index.d.ts", + "dependencies": { + "@snickerdoodlelabs/common-utils": "workspace:^", + "@snickerdoodlelabs/nlp": "workspace:^", + "@snickerdoodlelabs/objects": "workspace:^", + "ethers": "^5.6.6", + "html-to-text": "^9.0.5", + "inversify": "^6.0.1", + "neverthrow": "^5.1.0", + "neverthrow-result-utils": "^2.0.2", + "openai": "^4.0.1" + } +} diff --git a/packages/shopping-data/src/implementations/ProductUtils.ts b/packages/shopping-data/src/implementations/ProductUtils.ts new file mode 100644 index 0000000000..d7f2fb12b8 --- /dev/null +++ b/packages/shopping-data/src/implementations/ProductUtils.ts @@ -0,0 +1,41 @@ +import { IStemmerServiceType, IStemmerService } from "@snickerdoodlelabs/nlp"; +import { ELanguageCode, NLPError } from "@snickerdoodlelabs/objects"; +import { inject, injectable } from "inversify"; +import { ResultAsync } from "neverthrow"; + +import { IProductUtils } from "@shopping-data/interfaces/IProductUtils"; + +@injectable() +export class ProductUtils implements IProductUtils { + constructor( + @inject(IStemmerServiceType) + private stemmerService: IStemmerService, + ) {} + + public getProductHashSync( + language: ELanguageCode, + productName: string, + ): string { + const tokens = this.stemmerService.tokenizeSync(language, productName); + return this.getHashFromTokens(tokens); + } + + public getProductHash( + language: ELanguageCode, + productName: string, + ): ResultAsync { + const result = this.getProductNameTokens(language, productName); + return result.map((tokens) => this.getHashFromTokens(tokens)); + } + + private getProductNameTokens( + language: ELanguageCode, + productName: string, + ): ResultAsync { + return this.stemmerService.tokenize(language, productName); + } + + private getHashFromTokens(tokens: string[]): string { + return tokens.sort().slice(0, 10).join("-"); + } +} diff --git a/packages/shopping-data/src/implementations/PurchaseUtils.ts b/packages/shopping-data/src/implementations/PurchaseUtils.ts new file mode 100644 index 0000000000..c73fb5aa2d --- /dev/null +++ b/packages/shopping-data/src/implementations/PurchaseUtils.ts @@ -0,0 +1,109 @@ +import { IStemmerServiceType, IStemmerService } from "@snickerdoodlelabs/nlp"; +import { + DomainName, + UnixTimestamp, + PurchasedProduct, + UnknownProductCategory, +} from "@snickerdoodlelabs/objects"; +import { inject, injectable } from "inversify"; +import { ResultAsync, okAsync } from "neverthrow"; + +import { + IProductUtils, + IProductUtilsType, + IPurchaseUtils, +} from "@shopping-data/interfaces/index.js"; + +@injectable() +export class PurchaseUtils implements IPurchaseUtils { + constructor( + @inject(IStemmerServiceType) + private stemmerService: IStemmerService, + @inject(IProductUtilsType) + private productUtils: IProductUtils, + ) {} + + public contains( + purchases: PurchasedProduct[], + purchase: PurchasedProduct, + ): ResultAsync { + return this.filterByMPAndDate( + purchases, + purchase.marketPlace, + purchase.datePurchased, + ).andThen((filtered) => { + return this.containsWithSimilarNameAndPrice(filtered, purchase); + }); + } + + public findSame( + purchases: PurchasedProduct[], + purchase: PurchasedProduct, + ): ResultAsync { + return this.filterByMPAndDate( + purchases, + purchase.marketPlace, + purchase.datePurchased, + ).andThen((filtered) => { + return this.findSameWithSimilarNameAndPrice(filtered, purchase); + }); + } + + public containsWithSimilarNameAndPrice( + purchasesWithSameMPAndDate: PurchasedProduct[], + purchase: PurchasedProduct, + ): ResultAsync { + return this.findSameWithSimilarNameAndPrice( + purchasesWithSameMPAndDate, + purchase, + ).map((matched) => { + return matched != null; + }); + } + + public findSameWithSimilarNameAndPrice( + purchasesWithSameMPAndDate: PurchasedProduct[], + purchase: PurchasedProduct, + ): ResultAsync { + // TODO, instead of bag-of-words, use word2vec model to do a similarity comparison on vectors. https://github.com/georgegach/w2v + // TODO: talk to Charlie about bunlding nlp.js with the app https://github.com/axa-group/nlp.js/blob/master/docs/v4/webandreact.md + + const searchHash = this.productUtils.getProductHashSync( + purchase.language, + purchase.name, + ); + const matched = purchasesWithSameMPAndDate.find( + (p) => + this.productUtils.getProductHashSync(p.language, p.name) == + searchHash && p.price == purchase.price, + ); + return okAsync(matched ?? null); + } + + private filterByMPAndDate( + purchases: PurchasedProduct[], + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ): ResultAsync { + const filtered = purchases.reduce((acc, curr) => { + if ( + curr.marketPlace == marketPlace && + curr.datePurchased == datePurchased + ) { + acc.push(curr); + } + return acc; + }, [] as PurchasedProduct[]); + return okAsync(filtered); + } + + public getNullCategoryPurchases( + purchases: PurchasedProduct[], + ): PurchasedProduct[] { + return purchases.filter((purchase) => { + return ( + purchase.category == UnknownProductCategory || purchase.category == null + ); + }); + } +} diff --git a/packages/shopping-data/src/implementations/index.ts b/packages/shopping-data/src/implementations/index.ts new file mode 100644 index 0000000000..16ea7b9194 --- /dev/null +++ b/packages/shopping-data/src/implementations/index.ts @@ -0,0 +1,2 @@ +export * from "@shopping-data/implementations/ProductUtils.js"; +export * from "@shopping-data/implementations/PurchaseUtils.js"; diff --git a/packages/shopping-data/src/index.ts b/packages/shopping-data/src/index.ts new file mode 100644 index 0000000000..48667cc62f --- /dev/null +++ b/packages/shopping-data/src/index.ts @@ -0,0 +1,3 @@ +export * from "@shopping-data/interfaces/index.js"; +export * from "@shopping-data/implementations/index.js"; +export * from "@shopping-data/shoppingData.module.js"; diff --git a/packages/shopping-data/src/interfaces/IProductUtils.ts b/packages/shopping-data/src/interfaces/IProductUtils.ts new file mode 100644 index 0000000000..b51a4fe0c4 --- /dev/null +++ b/packages/shopping-data/src/interfaces/IProductUtils.ts @@ -0,0 +1,26 @@ +import { ELanguageCode, NLPError } from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface IProductUtils { + /** + * + * @param language + * @param productName + * @returns a hash containing first 10 non stop words stemmed and sorted alphabetically and glued together with a hypen + */ + getProductHash( + language: ELanguageCode, + productName: string, + ): ResultAsync; + + /** + * + * @param language + * @param productName + * @returns a hash containing first 10 non stop words stemmed and sorted alphabetically and glued together with a hypen + * @throws NLPError + */ + getProductHashSync(language: ELanguageCode, productName: string): string; +} + +export const IProductUtilsType = Symbol.for("IProductUtils"); diff --git a/packages/shopping-data/src/interfaces/IPurchaseRepository.ts b/packages/shopping-data/src/interfaces/IPurchaseRepository.ts new file mode 100644 index 0000000000..14b0e8ddb4 --- /dev/null +++ b/packages/shopping-data/src/interfaces/IPurchaseRepository.ts @@ -0,0 +1,29 @@ +import { + DomainName, + PersistenceError, + ShoppingDataConnectionStatus, + UnixTimestamp, + PurchasedProduct, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface IPurchaseRepository { + add(purchase: PurchasedProduct): ResultAsync; + getPurchasedProducts(): ResultAsync; + getByMarketplace( + marketPlace: DomainName, + ): ResultAsync; + getByMarketplaceAndDate( + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ): ResultAsync; + getShoppingDataConnectionStatus(): ResultAsync< + ShoppingDataConnectionStatus[], + PersistenceError + >; + setShoppingDataConnectionStatus( + ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ): ResultAsync; +} + +export const IPurchaseRepositoryType = Symbol.for("IPurchaseRepository"); diff --git a/packages/shopping-data/src/interfaces/IPurchaseUtils.ts b/packages/shopping-data/src/interfaces/IPurchaseUtils.ts new file mode 100644 index 0000000000..bd589bae0d --- /dev/null +++ b/packages/shopping-data/src/interfaces/IPurchaseUtils.ts @@ -0,0 +1,56 @@ +import { + ELanguageCode, + NLPError, + PurchasedProduct, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface IPurchaseUtils { + /** + * Returns the purchase if the purchases array contains a purchase with the same marketplace, date of purchase, name and price as the purchase parameter. + * @param purchases + * @param purchase + */ + findSame( + purchases: PurchasedProduct[], + purchase: PurchasedProduct, + ): ResultAsync; + + /** + * Returns the purchase if the purchases array contains a purchase with the similar name and price as the purchase parameter. + * @param purchasesWithSameMPAndDate + * @param purchase + */ + findSameWithSimilarNameAndPrice( + purchasesWithSameMPAndDate: PurchasedProduct[], + purchase: PurchasedProduct, + ): ResultAsync; + + /** + * Returns true if the purchases array contains a purchase with the same marketplace, date of purchase, name and price as the purchase parameter. + * @param purchases + * @param purchase + */ + contains( + purchases: PurchasedProduct[], + purchase: PurchasedProduct, + ): ResultAsync; + + /** + * Returns true if the purchases array contains a purchase with the name and price as the purchase parameter. + * @param purchasesWithSameMPAndDate + * @param purchase + * + */ + containsWithSimilarNameAndPrice( + purchasesWithSameMPAndDate: PurchasedProduct[], + purchase: PurchasedProduct, + ): ResultAsync; + + /** + * Returns purchases missing categories (with unknowns only) + * @param purchases + */ + getNullCategoryPurchases(purchases: PurchasedProduct[]): PurchasedProduct[]; +} +export const IPurchaseUtilsType = Symbol.for("IPurchaseUtils"); diff --git a/packages/shopping-data/src/interfaces/index.ts b/packages/shopping-data/src/interfaces/index.ts new file mode 100644 index 0000000000..b64d31920b --- /dev/null +++ b/packages/shopping-data/src/interfaces/index.ts @@ -0,0 +1,3 @@ +export * from "@shopping-data/interfaces/IProductUtils.js"; +export * from "@shopping-data/interfaces/IPurchaseRepository.js"; +export * from "@shopping-data/interfaces/IPurchaseUtils.js"; diff --git a/packages/shopping-data/src/shoppingData.module.ts b/packages/shopping-data/src/shoppingData.module.ts new file mode 100644 index 0000000000..c570a5dfdf --- /dev/null +++ b/packages/shopping-data/src/shoppingData.module.ts @@ -0,0 +1,26 @@ +import { ContainerModule, interfaces } from "inversify"; + +import { + ProductUtils, + PurchaseUtils, +} from "@shopping-data/implementations/index.js"; +import { + IProductUtils, + IProductUtilsType, + IPurchaseUtils, + IPurchaseUtilsType, +} from "@shopping-data/interfaces/index.js"; + +export const shoppingDataModule = new ContainerModule( + ( + bind: interfaces.Bind, + _unbind: interfaces.Unbind, + _isBound: interfaces.IsBound, + _rebind: interfaces.Rebind, + ) => { + bind(IPurchaseUtilsType) + .to(PurchaseUtils) + .inSingletonScope(); + bind(IProductUtilsType).to(ProductUtils).inSingletonScope(); + }, +); diff --git a/packages/shopping-data/test/mock/index.ts b/packages/shopping-data/test/mock/index.ts new file mode 100644 index 0000000000..3bc2b879ab --- /dev/null +++ b/packages/shopping-data/test/mock/index.ts @@ -0,0 +1,2 @@ +export * from "@shopping-data-test/mock/products.js"; +export * from "@shopping-data-test/mock/purchases.js"; diff --git a/packages/shopping-data/test/mock/products.ts b/packages/shopping-data/test/mock/products.ts new file mode 100644 index 0000000000..008f7f4a47 --- /dev/null +++ b/packages/shopping-data/test/mock/products.ts @@ -0,0 +1,4 @@ +export const prod1 = "IPhone 12 is great"; +export const prod1Hash = "12-great-iphone"; +export const prod2 = "Aveeno Baby Lotion"; +export const prod2Hash = "aveeno-baby-lotion"; diff --git a/packages/shopping-data/test/mock/purchases.ts b/packages/shopping-data/test/mock/purchases.ts new file mode 100644 index 0000000000..0db5d18021 --- /dev/null +++ b/packages/shopping-data/test/mock/purchases.ts @@ -0,0 +1,243 @@ +import { TimeUtils } from "@snickerdoodlelabs/common-utils"; +import { + DomainName, + ELanguageCode, + ProductKeyword, + PurchaseId, + PurchasedProduct, + UnknownProductCategory, +} from "@snickerdoodlelabs/objects"; + +const timeUtils = new TimeUtils(); +export const janDate = timeUtils.parseToSDTimestamp("2021-01-01"); +export const febDate = timeUtils.parseToSDTimestamp("2021-02-11"); +export const mp1 = DomainName("amazon.com"); +export const mp2 = DomainName("ebay.com"); + +export const iphone12JanVariant = new PurchasedProduct( + mp1, + ELanguageCode.English, + PurchaseId("amazon-iphone-12-2021-01-01"), + "The IPhone 12", + "Orange", + 1000, + janDate!, + febDate!, + null, + null, + null, + "Unknown", + [ + ProductKeyword("phone"), + ProductKeyword("orange"), + ProductKeyword("smart phone"), + ], +); + +export const janPruchasesAmazon = [ + new PurchasedProduct( + mp1, + ELanguageCode.English, + PurchaseId("amazon-iphone-12-2021-01-01"), + "IPhone 12", + "Apple", + 1000, + janDate!, + janDate!, + null, + null, + null, + "Electronics", + [ + ProductKeyword("phone"), + ProductKeyword("apple"), + ProductKeyword("smart phone"), + ], + ), + new PurchasedProduct( + mp1, + ELanguageCode.English, + PurchaseId("amazon-iphone-11-2021-01-11"), + "IPhone 11", + "Apple", + 600, + janDate!, + janDate!, + null, + null, + null, + "Electronics", + [ + ProductKeyword("phone"), + ProductKeyword("apple"), + ProductKeyword("smart phone"), + ], + ), + new PurchasedProduct( + mp1, + ELanguageCode.English, + PurchaseId("amazon-aveeno-baby-lotion-2021-01-11"), + "Aveeno Baby Lotion", + "Aveeno", + 15, + janDate!, + janDate!, + null, + null, + null, + "Skin Care", + [ + ProductKeyword("baby"), + ProductKeyword("skincare"), + ProductKeyword("body lotion"), + ], + ), +]; + +export const janPruchases = [ + ...janPruchasesAmazon, + new PurchasedProduct( + mp2, + ELanguageCode.English, + PurchaseId("ebay-aveeno-baby-lotion-2021-01-11"), + "Aveeno Baby Lotion", + "Aveeno", + 11, + janDate!, + janDate!, + null, + null, + null, + "Skin Care", + [ + ProductKeyword("baby"), + ProductKeyword("skincare"), + ProductKeyword("body lotion"), + ], + ), +]; + +export const febPruchases = [ + new PurchasedProduct( + mp1, + ELanguageCode.English, + PurchaseId("amazon-iphone-12-2021-02-11"), + "IPhone 12", + "Apple", + 1000, + febDate!, + febDate!, + null, + null, + null, + "Electronics", + [ + ProductKeyword("phone"), + ProductKeyword("apple"), + ProductKeyword("smart phone"), + ], + ), + new PurchasedProduct( + mp1, + ELanguageCode.English, + PurchaseId("amazon-iphone-11-2021-02-11"), + "IPhone 11", + "Apple", + 1800, + febDate!, + febDate!, + null, + null, + null, + "Electronics", + [ + ProductKeyword("phone"), + ProductKeyword("apple"), + ProductKeyword("smart phone"), + ], + ), + new PurchasedProduct( + mp1, + ELanguageCode.English, + PurchaseId("amazon-aveeno-baby-lotion-2021-02-11"), + "Aveeno Baby Lotion", + "Aveeno", + 20, + febDate!, + febDate!, + null, + null, + null, + "Skin Care", + [ + ProductKeyword("baby"), + ProductKeyword("skincare"), + ProductKeyword("body lotion"), + ], + ), + new PurchasedProduct( + mp2, + ELanguageCode.English, + PurchaseId("ebay-aveeno-baby-lotion-2021-02-11"), + "Aveeno Baby Lotion", + "Aveeno", + 3.5, + febDate!, + febDate!, + null, + null, + null, + "Skin Care", + [ + ProductKeyword("baby"), + ProductKeyword("skincare"), + ProductKeyword("body lotion"), + ], + ), +]; + +export const nullCategoryPruchases = [ + new PurchasedProduct( + mp2, + ELanguageCode.English, + PurchaseId("null-ebay-aveeno-baby-lotion-2021-01-11"), + "Aveeno Baby Lotion", + "Aveeno", + 11, + janDate!, + janDate!, + null, + null, + null, + UnknownProductCategory, + [ + ProductKeyword("baby"), + ProductKeyword("skincare"), + ProductKeyword("body lotion"), + ], + ), + new PurchasedProduct( + mp2, + ELanguageCode.English, + PurchaseId("ebay-aveeno-baby-lotion-2021-02-11"), + "Aveeno Baby Lotion", + "Aveeno", + 3.5, + febDate!, + febDate!, + null, + null, + null, + UnknownProductCategory, + [ + ProductKeyword("baby"), + ProductKeyword("skincare"), + ProductKeyword("body lotion"), + ], + ), +]; +export const allPurchases = [ + ...janPruchases, + ...febPruchases, + ...nullCategoryPruchases, +]; diff --git a/packages/shopping-data/test/tsconfig.json b/packages/shopping-data/test/tsconfig.json new file mode 100644 index 0000000000..1b2f7e64a3 --- /dev/null +++ b/packages/shopping-data/test/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "outDir": "../dist/test", + "rootDir": "..", + "module": "ES2022", + }, + "references": [ + { + "path": ".." + } + ], + "include": ["../src/**/*.ts", "../test/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/packages/shopping-data/test/unit/ProductUtils.test.ts b/packages/shopping-data/test/unit/ProductUtils.test.ts new file mode 100644 index 0000000000..963c4283e5 --- /dev/null +++ b/packages/shopping-data/test/unit/ProductUtils.test.ts @@ -0,0 +1,46 @@ +import "reflect-metadata"; +import { IStemmerService } from "@snickerdoodlelabs/nlp"; +import { ELanguageCode } from "@snickerdoodlelabs/objects"; +import { okAsync } from "neverthrow"; +import * as td from "testdouble"; + +import { ProductUtils } from "@shopping-data/implementations"; +import { IProductUtils } from "@shopping-data/interfaces"; +import { prod1, prod1Hash, prod2, prod2Hash } from "@shopping-data-test/mock"; + +class Mocks { + public stemmerService = td.object(); + public constructor() { + td.when( + this.stemmerService.tokenize(ELanguageCode.English, prod1), + ).thenReturn(okAsync(prod1Hash.split("-"))); + td.when( + this.stemmerService.tokenize(ELanguageCode.English, prod2), + ).thenReturn(okAsync(prod2Hash.split("-"))); + } + public factory(): IProductUtils { + return new ProductUtils(this.stemmerService); + } +} + +describe("ProductUtils", () => { + test("product hash", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result1 = await utils.getProductHash(ELanguageCode.English, prod1); + const result2 = await utils.getProductHash(ELanguageCode.English, prod2); + + // Assert + expect(result1.isOk()).toBeTruthy(); + expect(result2.isOk()).toBeTruthy(); + + const hash1 = result1._unsafeUnwrap(); + const hash2 = result2._unsafeUnwrap(); + + expect(hash1).toEqual(prod1Hash); + expect(hash2).toEqual(prod2Hash); + }); +}); diff --git a/packages/shopping-data/test/unit/PurchaseUtils.test.ts b/packages/shopping-data/test/unit/PurchaseUtils.test.ts new file mode 100644 index 0000000000..2f9e703ef2 --- /dev/null +++ b/packages/shopping-data/test/unit/PurchaseUtils.test.ts @@ -0,0 +1,153 @@ +import "reflect-metadata"; +import { StemmerService } from "@snickerdoodlelabs/nlp"; +import * as td from "testdouble"; + +import { ProductUtils, PurchaseUtils } from "@shopping-data/implementations"; +import { IPurchaseUtils } from "@shopping-data/interfaces"; +import { + allPurchases, + febPruchases, + iphone12JanVariant, + janPruchases, + janPruchasesAmazon, + nullCategoryPruchases, +} from "@shopping-data-test/mock"; + +class Mocks { + public stemmerService = new StemmerService(); + public constructor() {} + public factory(): IPurchaseUtils { + return new PurchaseUtils( + this.stemmerService, + new ProductUtils(this.stemmerService), + ); + } +} + +describe("PurchaseUtils slow search", () => { + test("findSame truthy", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.findSame(janPruchases, janPruchases[0]); + + // Assert + expect(result.isOk()).toBeTruthy(); + const matched = result._unsafeUnwrap(); + expect(matched).not.toBeNull(); + expect(matched?.name).toEqual(janPruchases[0].name); + expect(matched?.price).toEqual(janPruchases[0].price); + expect(matched?.marketPlace).toEqual(janPruchases[0].marketPlace); + expect(matched?.datePurchased).toEqual(janPruchases[0].datePurchased); + expect(matched?.language).toEqual(janPruchases[0].language); + }); + + test("findSame falsy", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.findSame(febPruchases, janPruchases[0]); + + // Assert + expect(result.isOk()).toBeTruthy(); + const matched = result._unsafeUnwrap(); + expect(matched).toBeNull(); + }); + + test("contains", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.contains(janPruchases, janPruchases[0]); + + // Assert + expect(result.isOk()).toBeTruthy(); + expect(result._unsafeUnwrap()).toBeTruthy(); + }); + + test("contains iphone 12 variant", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.contains(janPruchases, iphone12JanVariant); + + // Assert + expect(result.isOk()).toBeTruthy(); + expect(result._unsafeUnwrap()).toBeTruthy(); + }); + + test("does not contain", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.contains(febPruchases, janPruchases[0]); + + // Assert + expect(result.isOk()).toBeTruthy(); + expect(result._unsafeUnwrap()).toBeFalsy(); + }); +}); + +describe("PurchaseUtils fast search", () => { + test("findSame truthy", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.findSameWithSimilarNameAndPrice( + janPruchasesAmazon, + janPruchases[0], + ); + + // Assert + expect(result.isOk()).toBeTruthy(); + const matched = result._unsafeUnwrap(); + expect(matched).not.toBeNull(); + expect(matched?.name).toEqual(janPruchases[0].name); + expect(matched?.price).toEqual(janPruchases[0].price); + expect(matched?.marketPlace).toEqual(janPruchases[0].marketPlace); + expect(matched?.datePurchased).toEqual(janPruchases[0].datePurchased); + expect(matched?.language).toEqual(janPruchases[0].language); + }); + + test("contains iphone 12 variant", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const result = await utils.containsWithSimilarNameAndPrice( + janPruchasesAmazon, + iphone12JanVariant, + ); + + // Assert + expect(result.isOk()).toBeTruthy(); + expect(result._unsafeUnwrap()).toBeTruthy(); + }); +}); + +describe("PurchaseUtils other tools", () => { + test("get null category products", async () => { + // Arrange + const mocks = new Mocks(); + const utils = mocks.factory(); + + // Act + const got = utils.getNullCategoryPurchases(allPurchases); + + // Assert + expect(got).toEqual(nullCategoryPruchases); + }); +}); diff --git a/packages/shopping-data/tsconfig.json b/packages/shopping-data/tsconfig.json new file mode 100644 index 0000000000..649cef5098 --- /dev/null +++ b/packages/shopping-data/tsconfig.json @@ -0,0 +1,30 @@ +{ + "extends": "../../tsconfig.build.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src", + "module": "ES2022", + "paths": { + "@shopping-data/*": ["./packages/shopping-data/src/*"], + "@shopping-data-test/*": ["./packages/shopping-data/test/*"], + }, + }, + "include": [ + "./src/**/*.ts" + ], + "exclude": [ + "node_modules", + "dist" + ], + "references": [ + { + "path": "../objects", + }, + { + "path": "../common-utils" + }, + { + "path": "../nlp" + } + ] +} \ No newline at end of file diff --git a/packages/synamint-extension-sdk/src/content/DataWalletProxy.ts b/packages/synamint-extension-sdk/src/content/DataWalletProxy.ts index 285f113082..fa8bbe5944 100644 --- a/packages/synamint-extension-sdk/src/content/DataWalletProxy.ts +++ b/packages/synamint-extension-sdk/src/content/DataWalletProxy.ts @@ -56,8 +56,10 @@ import { ECoreProxyType, BlockNumber, RefreshToken, - TransactionFilter, IProxyAccountMethods, + IProxyPurchaseMethods, + TransactionFilter, + ShoppingDataConnectionStatus, INftProxyMethods, JSONString, IProxyQuestionnaireMethods, @@ -162,6 +164,7 @@ export class _DataWalletProxy extends EventEmitter implements ISdlDataWallet { public nft: INftProxyMethods; public questionnaire: IProxyQuestionnaireMethods; public events: PublicEvents; + public purchase: IProxyPurchaseMethods; public requestDashboardView = undefined; public proxyType: ECoreProxyType = ECoreProxyType.EXTENSION_INJECTED; @@ -419,6 +422,34 @@ export class _DataWalletProxy extends EventEmitter implements ISdlDataWallet { }, }; + this.purchase = { + getPurchasedProducts: () => { + return coreGateway.purchase.getPurchasedProducts(); + }, + getByMarketplace: (marketPlace: DomainName) => { + return coreGateway.purchase.getByMarketplace(marketPlace); + }, + getByMarketplaceAndDate: ( + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ) => { + return coreGateway.purchase.getByMarketplaceAndDate( + marketPlace, + datePurchased, + ); + }, + getShoppingDataConnectionStatus: () => { + return coreGateway.purchase.getShoppingDataConnectionStatus(); + }, + setShoppingDataConnectionStatus: ( + ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ) => { + return coreGateway.purchase.setShoppingDataConnectionStatus( + ShoppingDataConnectionStatus, + ); + }, + }; + this.storage = { // @TODO below functions are not added to ISDLDataWallet interface and iframe getDropboxAuth: () => { diff --git a/packages/synamint-extension-sdk/src/content/components/App/App.tsx b/packages/synamint-extension-sdk/src/content/components/App/App.tsx index 63e805f47f..328644d362 100644 --- a/packages/synamint-extension-sdk/src/content/components/App/App.tsx +++ b/packages/synamint-extension-sdk/src/content/components/App/App.tsx @@ -7,11 +7,11 @@ import { ENotificationTypes, EVMContractAddress, EWalletDataType, + LinkedAccount, IOldUserAgreement, IPaletteOverrides, IUserAgreement, Invitation, - LinkedAccount, Signature, UnixTimestamp, } from "@snickerdoodlelabs/objects"; @@ -24,6 +24,26 @@ import { createDefaultTheme, createThemeWithOverrides, } from "@snickerdoodlelabs/shared-components"; +import endOfStream from "end-of-stream"; +import PortStream from "extension-port-stream"; +import { JsonRpcEngine } from "json-rpc-engine"; +import { createStreamMiddleware } from "json-rpc-middleware-stream"; +import { err, okAsync } from "neverthrow"; +import ObjectMultiplex from "obj-multiplex"; +import LocalMessageStream from "post-message-stream"; +import pump from "pump"; +import React, { + useEffect, + useMemo, + useState, + useCallback, + useRef, + FC, +} from "react"; +import { parse } from "tldts"; +import Browser from "webextension-polyfill"; + +import { ShoppingDataService } from "@synamint-extension-sdk/content/components/ShoppingDataService"; import { EAppState } from "@synamint-extension-sdk/content/constants"; import usePath from "@synamint-extension-sdk/content/hooks/usePath"; import DataWalletProxyInjectionUtils from "@synamint-extension-sdk/content/utils/DataWalletProxyInjectionUtils"; @@ -43,24 +63,6 @@ import { GetConsentContractCIDParams, } from "@synamint-extension-sdk/shared"; import { UpdatableEventEmitterWrapper } from "@synamint-extension-sdk/utils"; -import endOfStream from "end-of-stream"; -import PortStream from "extension-port-stream"; -import { JsonRpcEngine } from "json-rpc-engine"; -import { createStreamMiddleware } from "json-rpc-middleware-stream"; -import { err, okAsync } from "neverthrow"; -import ObjectMultiplex from "obj-multiplex"; -import LocalMessageStream from "post-message-stream"; -import pump from "pump"; -import React, { - useEffect, - useMemo, - useState, - useCallback, - useRef, - FC, -} from "react"; -import { parse } from "tldts"; -import Browser from "webextension-polyfill"; // #region connection let coreGateway: ExternalCoreGateway; @@ -383,6 +385,12 @@ const App: FC = ({ paletteOverrides }) => { }; }, []); + const getAccounts = () => { + coreGateway.account.getAccounts().map((linkedAccounts) => { + setAccounts(linkedAccounts); + }); + }; + const handleNotification = (notification: BaseNotification) => { if (notification.type === ENotificationTypes.ACCOUNT_ADDED) { getAccounts(); @@ -391,12 +399,6 @@ const App: FC = ({ paletteOverrides }) => { // #endregion - const getAccounts = () => { - coreGateway.account.getAccounts().map((linkedAccounts) => { - setAccounts(linkedAccounts); - }); - }; - const renderComponent = useMemo(() => { if (!currentInvitation) return null; // delay showing popup until user link an account @@ -458,6 +460,7 @@ const App: FC = ({ paletteOverrides }) => { > <> {renderComponent && {renderComponent}} + ); diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/ShoppingDataService.tsx b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/ShoppingDataService.tsx new file mode 100644 index 0000000000..43aafbdad4 --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/ShoppingDataService.tsx @@ -0,0 +1,148 @@ +import { + Amazon, + EKnownDomains, + ELanguageCode, + HTMLString, + PageNumber, + ShoppingDataConnectionStatus, + URLString, +} from "@snickerdoodlelabs/objects"; +import { + ModalContainer, + SCRAPING_INDEX, +} from "@snickerdoodlelabs/shared-components"; +import React, { useEffect, useMemo, useState } from "react"; + +import { + ShoppingDataDone, + ShoppingDataINIT, + ShoppingDataProcess, +} from "@synamint-extension-sdk/content/components/ShoppingDataService/components/Screens"; +import { + EShoppingDataState, + windowFeatures, +} from "@synamint-extension-sdk/content/components/ShoppingDataService/constants"; +import { ExternalCoreGateway } from "@synamint-extension-sdk/gateways"; + +interface IShoppingDataProcessProps { + coreGateway: ExternalCoreGateway; +} + +export const ShoppingDataService: React.FC = ({ + coreGateway, +}: IShoppingDataProcessProps) => { + const [shoppingDataScrapeStart, setShoppingDataScrapeStart] = + useState(false); + const [shoppingDataState, setShoppingDataState] = + useState(EShoppingDataState.SHOPPINGDATA_IDLE); + + const AMAZONINDEX: string | undefined = SCRAPING_INDEX.get( + EKnownDomains.Amazon, + ); + useEffect(() => { + checkURLAMAZON(); + }, [shoppingDataScrapeStart]); + + const getAmazonUrlsAndScrape = () => { + const html = document.documentElement.outerHTML; + let openWindowCount = 0; + return coreGateway.scraperNavigation + .getYears(HTMLString(html)) + .map((years) => { + for (const year of years) { + for (let i = 1; i <= 5; i++) { + coreGateway.scraperNavigation + .getOrderHistoryPageByYear( + ELanguageCode.English, + year, + PageNumber(i), + ) + .map(async (url) => { + const newWindow = window.open(url, "_blank", windowFeatures); + if (newWindow) { + openWindowCount++; + await new Promise((resolve) => { + newWindow.onload = resolve; + }); + + const windowHTML = + newWindow.document.documentElement.outerHTML; + coreGateway.scraper + .classifyURL(URLString(url), ELanguageCode.English) + .andThen((DomainTask) => { + return coreGateway.scraper + .scrape( + URLString(url), + HTMLString(windowHTML), + DomainTask, + ) + .map((result) => + console.log("scrape function result", result), + ) + .mapErr((err) => + console.log("scrape function error", err), + ); + }) + .mapErr((err) => + console.log("clasifyUrl function error", err), + ); + + newWindow.close(); + openWindowCount--; + if (openWindowCount === 0) { + setShoppingDataState( + EShoppingDataState.SHOPPINGDATA_SCRAPE_DONE, + ); + const amazonConnectionStatus: ShoppingDataConnectionStatus = + new Amazon(true); + coreGateway.purchase.setShoppingDataConnectionStatus( + amazonConnectionStatus, + ); + } + } + }); + } + } + }); + }; + + const checkURLAMAZON = () => { + const searchParams = new URLSearchParams(window.location.search); + const SDLStep = searchParams.get("SDLStep"); + if (AMAZONINDEX !== undefined && SDLStep && SDLStep === AMAZONINDEX) { + setShoppingDataState(EShoppingDataState.SHOPPINGDATA_INIT); + if (shoppingDataScrapeStart) { + setShoppingDataState(EShoppingDataState.SHOPPINGDATA_SCRAPE_PROCESS); + getAmazonUrlsAndScrape(); + } + } + }; + + const exitScraper = () => { + setShoppingDataState(EShoppingDataState.SHOPPINGDATA_INIT); + }; + + const renderShoppingDataComponent = useMemo(() => { + switch (true) { + case shoppingDataState === EShoppingDataState.SHOPPINGDATA_INIT: + return ( + + ); + case shoppingDataState === EShoppingDataState.SHOPPINGDATA_SCRAPE_PROCESS: + return ; + case shoppingDataState === EShoppingDataState.SHOPPINGDATA_SCRAPE_DONE: + return ; + default: + return null; + } + }, [shoppingDataState]); + return ( + <> + {renderShoppingDataComponent && ( + {renderShoppingDataComponent} + )} + + ); +}; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/LinkCard/LinkCard.style.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/LinkCard/LinkCard.style.ts new file mode 100644 index 0000000000..75f5d27abb --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/LinkCard/LinkCard.style.ts @@ -0,0 +1,18 @@ +import { makeStyles } from "@material-ui/core/styles"; + +export const useStyles = makeStyles((theme) => ({ + container: { + border: "1px solid #ECECEC", + borderRadius: 8, + height: 60, + cursor: "pointer", + }, + linkTitle: { + fontFamily: "Space Grotesk", + fontStyle: "normal", + fontWeight: 400, + fontSize: "16px", + lineHeight: "20px", + color: "#FFF", + }, +})); diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/LinkCard/LinkCard.tsx b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/LinkCard/LinkCard.tsx new file mode 100644 index 0000000000..96974a3a40 --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/LinkCard/LinkCard.tsx @@ -0,0 +1,43 @@ +import { Box, Typography } from "@material-ui/core"; +import React from "react"; +import Browser from "webextension-polyfill"; + +import { useStyles } from "@synamint-extension-sdk/content/components/ShoppingDataService/components/LinkCard/LinkCard.style"; +import { ExternalCoreGateway } from "@synamint-extension-sdk/gateways"; + +interface ILinkCardProps { + navigateTo: string; + icon: string; + title: string; + coreGateway: ExternalCoreGateway; +} +export const LinkCard = ({ + navigateTo, + icon, + title, + coreGateway, +}: ILinkCardProps) => { + const classes = useStyles(); + const navigate = () => { + console.log(coreGateway); + coreGateway.getConfig().map((config) => { + console.log(config); + window.open(`${config.onboardingURL}${navigateTo}`, "_blank"); + }); + }; + return ( + + + + + {title} + + ); +}; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/LinkCard/index.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/LinkCard/index.ts new file mode 100644 index 0000000000..2f2bf27f91 --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/LinkCard/index.ts @@ -0,0 +1 @@ +export * from "@synamint-extension-sdk/content/components/ShoppingDataService/components/LinkCard/LinkCard"; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Progress-Bar/ProgressBar.style.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Progress-Bar/ProgressBar.style.ts new file mode 100644 index 0000000000..ffa9307135 --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Progress-Bar/ProgressBar.style.ts @@ -0,0 +1,19 @@ +import { makeStyles, createStyles } from "@material-ui/core/styles"; + +export const useStyles = makeStyles((theme) => + createStyles({ + percentText: { + fontWeight: 500, + fontSize: "20px", + lineHeight: "28px", + color: "#000000", + }, + bar: { + width: "280px", + height: "16px", + color: "#6E62A6", + backgroundColor: "#D2CEE3", + borderRadius: "6px", + }, + }), +); diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Progress-Bar/ProgressBar.tsx b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Progress-Bar/ProgressBar.tsx new file mode 100644 index 0000000000..780dbf320e --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Progress-Bar/ProgressBar.tsx @@ -0,0 +1,87 @@ +import { + Box, + LinearProgress, + LinearProgressProps, + Typography, +} from "@material-ui/core"; +import React, { useEffect, useState } from "react"; + +import { useStyles } from "@synamint-extension-sdk/content/components/ShoppingDataService/components/Progress-Bar/ProgressBar.style"; + +const LinearProgressBar = (props: LinearProgressProps & { value: number }) => { + const classes = useStyles(); + + const [position, setPosition] = useState("0%"); + + useEffect(() => { + const timer = setTimeout(() => { + if (props.value >= 100) { + setPosition("90%"); + } else if (props.value >= 75) { + setPosition("70%"); + } else if (props.value >= 50) { + setPosition("45%"); + } else if (props.value >= 25) { + setPosition("20%"); + } else { + setPosition("0%"); + } + }, 0); + + return () => { + clearTimeout(timer); + }; + }, [props.value]); + + return ( + + {props.value > 0 && ( + + {`${Math.round( + props.value, + )}%`} + + )} + + + + + ); +}; + +const ProgressBar = () => { + const [progress, setProgress] = useState(25); + + useEffect(() => { + const timer = setInterval(() => { + setProgress((prevProgress) => { + if (prevProgress >= 100) { + clearInterval(timer); + return prevProgress; + } + return prevProgress + 25; + }); + }, 1000); + return () => { + clearInterval(timer); + }; + }, []); + + return ( + + + + ); +}; + +export default ProgressBar; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Progress-Bar/index.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Progress-Bar/index.ts new file mode 100644 index 0000000000..25d009f08d --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Progress-Bar/index.ts @@ -0,0 +1 @@ +export { default } from "@synamint-extension-sdk/content/components/ShoppingDataService/components/Progress-Bar/ProgressBar"; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataDone/ShoppingDataDone.style.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataDone/ShoppingDataDone.style.ts new file mode 100644 index 0000000000..7f7de0d5c4 --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataDone/ShoppingDataDone.style.ts @@ -0,0 +1,64 @@ +import { makeStyles, createStyles } from "@material-ui/core/styles"; + +export const useStyles = makeStyles((theme) => + createStyles({ + container: { + position: "fixed", + top: 0, + right: 78, + zIndex: 9999, + borderRadius: 0, + boxShadow: "none", + }, + socialButtonWrapper: { + cursor: "pointer", + }, + link: { + fontFamily: "'Inter' !important", + fontStyle: "normal", + fontWeight: 500, + fontSize: "12px !important", + lineHeight: "12px !important", + color: "rgba(36, 34, 58, 0.8)", + textDecorationLine: "underline", + cursor: "pointer", + }, + url: { + fontFamily: "'Inter' !important", + fontStyle: "normal !important", + fontWeight: 500, + fontSize: "12px !important", + lineHeight: "12px !important", + color: "rgba(36, 34, 58, 0.8)", + cursor: "pointer", + }, + copyrightLogo: { + height: 16, + width: "auto", + }, + button: { + backgroundColor: "#6E62A6", + height: "54px", + borderRadius: "4px", + border: "1px solid #6E62A6", + fontWeight: 500, + fontSize: "14px", + lineHeight: "16px", + color: "#FFFFFF", + }, + title: { + fontWeight: 700, + fontSize: "24px", + lineHeight: "16px", + textAlign: "center", + color: "#424242", + }, + subText: { + fontWeight: 400, + fontSize: "18px", + lineHeight: "24px", + textAlign: "center", + color: "#54A858", + }, + }), +); diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataDone/ShoppingDataDone.tsx b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataDone/ShoppingDataDone.tsx new file mode 100644 index 0000000000..7f74aaf47a --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataDone/ShoppingDataDone.tsx @@ -0,0 +1,162 @@ +import { Box, Typography, Dialog, Button } from "@material-ui/core"; +import React from "react"; +import Browser from "webextension-polyfill"; + +import { LinkCard } from "@synamint-extension-sdk/content/components/ShoppingDataService/components/LinkCard/LinkCard"; +import { useStyles } from "@synamint-extension-sdk/content/components/ShoppingDataService/components/Screens/ShoppingDataDone/ShoppingDataDone.style"; +import { + PRIVACY_POLICY_URL, + SPA_PATHS, + WEBSITE_URL, +} from "@synamint-extension-sdk/content/components/ShoppingDataService/constants"; +import { ExternalCoreGateway } from "@synamint-extension-sdk/gateways"; + +interface IShoppingDataDoneProps { + coreGateway: ExternalCoreGateway; +} + +export const SOCIAL_LINKS = [ + { + iconSrc: Browser.runtime.getURL("assets/icons/twitter.svg"), + url: "https://twitter.com/yosnickerdoodle", + }, + { + iconSrc: Browser.runtime.getURL("assets/icons/linkedin.svg"), + url: "https://www.linkedin.com/company/snickerdoodlelabs/", + }, + { + iconSrc: Browser.runtime.getURL("assets/icons/instagram.svg"), + url: "https://www.instagram.com/yosnickerdoodle/", + }, + { + iconSrc: Browser.runtime.getURL("assets/icons/facebook.svg"), + url: "https://www.facebook.com/profile.php?id=100068000334616", + }, +]; + +export const ShoppingDataDone: React.FC = ({ + coreGateway, +}: IShoppingDataDoneProps) => { + const classes = useStyles(); + const navigateShoppingData = () => { + console.log(coreGateway); + coreGateway.getConfig().map((config) => { + console.log(config); + window.open(`${config.onboardingURL}${SPA_PATHS.shoppingData}`, "_blank"); + }); + }; + return ( + + + + + + + Congratulations! + + + + Your Amazon data successfully added to your Data Wallet. + + + + + + + + + + + + + + { + window.open(PRIVACY_POLICY_URL, "_blank"); + }} + > + Privacy Policy + + { + window.open(WEBSITE_URL, "_blank"); + }} + > + snickerdoodle.com + + + + + { + window.open(PRIVACY_POLICY_URL, "_blank"); + }} + > + Terms of Service + + + {SOCIAL_LINKS.map((link, index) => { + return ( + { + window.open(link.url, "_blank"); + }} + key={index} + > + + + ); + })} + + + + + + ); +}; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataDone/index.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataDone/index.ts new file mode 100644 index 0000000000..4afd3fd6dc --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataDone/index.ts @@ -0,0 +1 @@ +export * from "@synamint-extension-sdk/content/components/ShoppingDataService/components/Screens/ShoppingDataDone/ShoppingDataDone"; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataINIT/ShoppingDataINIT.style.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataINIT/ShoppingDataINIT.style.ts new file mode 100644 index 0000000000..2121b8a569 --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataINIT/ShoppingDataINIT.style.ts @@ -0,0 +1,63 @@ +import { makeStyles, createStyles } from "@material-ui/core/styles"; + +export const useStyles = makeStyles((theme) => + createStyles({ + container: { + position: "fixed", + top: 0, + right: 78, + zIndex: 9999, + borderRadius: 0, + boxShadow: "none", + }, + socialButtonWrapper: { + cursor: "pointer", + }, + link: { + fontFamily: "'Inter' !important", + fontStyle: "normal", + fontWeight: 500, + fontSize: "12px !important", + lineHeight: "12px !important", + color: "rgba(36, 34, 58, 0.8)", + textDecorationLine: "underline", + cursor: "pointer", + }, + url: { + fontFamily: "'Inter' !important", + fontStyle: "normal !important", + fontWeight: 500, + fontSize: "12px !important", + lineHeight: "12px !important", + color: "rgba(36, 34, 58, 0.8)", + cursor: "pointer", + }, + copyrightLogo: { + height: 16, + width: "auto", + }, + title: { + fontSize: "20px", + fontWeight: 700, + lineHeight: "30px", + color: "#424242", + textAlign: "center", + }, + bodyText: { + fontSize: "16px", + fontWeight: 400, + lineHeight: "20px", + color: "#424242", + }, + button: { + backgroundColor: "#6E62A6", + height: "54px", + borderRadius: "4px", + border: "1px solid #6E62A6", + fontWeight: 500, + fontSize: "14px", + lineHeight: "16px", + color: "#FFFFFF", + }, + }), +); diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataINIT/ShoppingDataINIT.tsx b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataINIT/ShoppingDataINIT.tsx new file mode 100644 index 0000000000..c09e6cb743 --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataINIT/ShoppingDataINIT.tsx @@ -0,0 +1,142 @@ +import { Box, Typography, Dialog, Button } from "@material-ui/core"; +import React from "react"; +import Browser from "webextension-polyfill"; + +import { useStyles } from "@synamint-extension-sdk/content/components/ShoppingDataService/components/Screens/ShoppingDataINIT/ShoppingDataINIT.style"; +import { + PRIVACY_POLICY_URL, + WEBSITE_URL, +} from "@synamint-extension-sdk/content/components/ShoppingDataService/constants"; + +interface IShoppingDataINITProps { + setShoppingDataScrapeStart; +} + +const SOCIAL_LINKS = [ + { + iconSrc: Browser.runtime.getURL("assets/icons/twitter.svg"), + url: "https://twitter.com/yosnickerdoodle", + }, + { + iconSrc: Browser.runtime.getURL("assets/icons/linkedin.svg"), + url: "https://www.linkedin.com/company/snickerdoodlelabs/", + }, + { + iconSrc: Browser.runtime.getURL("assets/icons/instagram.svg"), + url: "https://www.instagram.com/yosnickerdoodle/", + }, + { + iconSrc: Browser.runtime.getURL("assets/icons/facebook.svg"), + url: "https://www.facebook.com/profile.php?id=100068000334616", + }, +]; + +export const ShoppingDataINIT: React.FC = ({ + setShoppingDataScrapeStart, +}: IShoppingDataINITProps) => { + const classes = useStyles(); + + return ( + + + + + + + + Add Your Amazon Data to Your Profile + + + + + + Your data is anonymized, so you cannot be identified. + + + + + The more data you consent to lease, the more rewards you'll earn. + It's always your choice. + + + + + + + + + + { + window.open(PRIVACY_POLICY_URL, "_blank"); + }} + > + Privacy Policy + + { + window.open(WEBSITE_URL, "_blank"); + }} + > + snickerdoodle.com + + + + + { + window.open(PRIVACY_POLICY_URL, "_blank"); + }} + > + Terms of Service + + + {SOCIAL_LINKS.map((link, index) => { + return ( + { + window.open(link.url, "_blank"); + }} + key={index} + > + + + ); + })} + + + + + + ); +}; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataINIT/index.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataINIT/index.ts new file mode 100644 index 0000000000..17f7a3f4bf --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataINIT/index.ts @@ -0,0 +1 @@ +export * from "@synamint-extension-sdk/content/components/ShoppingDataService/components/Screens/ShoppingDataINIT/ShoppingDataINIT"; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataProcess/ShoppingDataProcess.style.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataProcess/ShoppingDataProcess.style.ts new file mode 100644 index 0000000000..97007b6af7 --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataProcess/ShoppingDataProcess.style.ts @@ -0,0 +1,38 @@ +import { makeStyles, createStyles } from "@material-ui/core/styles"; + +export const useStyles = makeStyles((theme) => + createStyles({ + container: { + position: "fixed", + top: 0, + right: 48, + zIndex: 9999, + borderRadius: "12px", + boxShadow: "none", + padding: "24px", + }, + title: { + fontWeight: 500, + fontSize: "16px", + lineHeight: "20px", + textAlign: "center", + color: "#424242", + }, + subText: { + fontWeight: 500, + fontSize: "14px", + lineHeight: "22px", + textAlign: "center", + color: "#54A858", + }, + button: { + borderRadius: "4px", + height: "40px", + backgroundColor: "#6E62A6", + fontWeight: 500, + fontSize: "14px", + lineHeight: "20px", + color: "#FFFFFF", + }, + }), +); diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataProcess/ShoppingDataProcess.tsx b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataProcess/ShoppingDataProcess.tsx new file mode 100644 index 0000000000..215abe8768 --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataProcess/ShoppingDataProcess.tsx @@ -0,0 +1,58 @@ +import { + Box, + Typography, + Button, + IconButton, + CircularProgress, +} from "@material-ui/core"; +import CloseIcon from "@material-ui/icons/Close"; +import React from "react"; + +import { useStyles } from "@synamint-extension-sdk/content/components/ShoppingDataService/components/Screens/ShoppingDataProcess/ShoppingDataProcess.style"; + +interface IShoppingDataProcessProps { + onCloseClick: () => void; +} + +export const ShoppingDataProcess: React.FC = ({ + onCloseClick, +}: IShoppingDataProcessProps) => { + const classes = useStyles(); + + return ( + + + + + + + + + + + Adding your data to your Data Wallet + + + + + + + + please wait for the process! + + + + ); +}; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataProcess/index.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataProcess/index.ts new file mode 100644 index 0000000000..1d872e3663 --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/ShoppingDataProcess/index.ts @@ -0,0 +1 @@ +export * from "@synamint-extension-sdk/content/components/ShoppingDataService/components/Screens/ShoppingDataProcess/ShoppingDataProcess"; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/index.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/index.ts new file mode 100644 index 0000000000..d600c36adc --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/components/Screens/index.ts @@ -0,0 +1,3 @@ +export * from "@synamint-extension-sdk/content/components/ShoppingDataService/components/Screens/ShoppingDataDone"; +export * from "@synamint-extension-sdk/content/components/ShoppingDataService/components/Screens/ShoppingDataINIT"; +export * from "@synamint-extension-sdk/content/components/ShoppingDataService/components/Screens/ShoppingDataProcess"; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/constants/index.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/constants/index.ts new file mode 100644 index 0000000000..f22ed20b77 --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/constants/index.ts @@ -0,0 +1,27 @@ +export enum EShoppingDataState { + SHOPPINGDATA_IDLE, + SHOPPINGDATA_INIT, + SHOPPINGDATA_SCRAPE_PROCESS, + SHOPPINGDATA_SCRAPE_DONE, +} + +export const SPA_PATHS = { + settings: "settings", + dataPermissions: "data-permissions", + dashboard: "data-dashboard/transaction-history", + shoppingData: "data-dashboard/shopping-data", +}; + +export const PRIVACY_POLICY_URL = + "https://policy.snickerdoodle.com/snickerdoodle-labs-data-privacy-policy"; + +export const WEBSITE_URL = "https://www.snickerdoodle.com/"; + +const width = 100; +const height = 100; + +const left = (window.screen.width - width) / 2; +const top = (window.screen.height - height) / 2; + +export const windowFeatures = + "width=" + width + ",height=" + height + ",left=" + left + ",top=" + top; diff --git a/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/index.ts b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/index.ts new file mode 100644 index 0000000000..6fc94771b0 --- /dev/null +++ b/packages/synamint-extension-sdk/src/content/components/ShoppingDataService/index.ts @@ -0,0 +1 @@ +export * from "@synamint-extension-sdk/content/components/ShoppingDataService/ShoppingDataService"; diff --git a/packages/synamint-extension-sdk/src/content/constants/index.ts b/packages/synamint-extension-sdk/src/content/constants/index.ts index 7f12d469f8..5123fd86fe 100644 --- a/packages/synamint-extension-sdk/src/content/constants/index.ts +++ b/packages/synamint-extension-sdk/src/content/constants/index.ts @@ -4,6 +4,9 @@ export enum EAppState { IDLE, AUDIENCE_PREVIEW, PERMISSION_SELECTION, + SUBSCRIPTION_CONFIRMATION, + SUBSCRIPTION_SUCCESS, + LOADING, } export interface IRewardItem { diff --git a/packages/synamint-extension-sdk/src/core/implementations/ExtensionCore.module.ts b/packages/synamint-extension-sdk/src/core/implementations/ExtensionCore.module.ts index b4160f2462..8f58ada2e3 100644 --- a/packages/synamint-extension-sdk/src/core/implementations/ExtensionCore.module.ts +++ b/packages/synamint-extension-sdk/src/core/implementations/ExtensionCore.module.ts @@ -28,6 +28,9 @@ import { MetricsService, PIIService, PortConnectionService, + PurchaseService, + ScraperNavigationService, + ScraperService, TokenPriceService, TwitterService, UserSiteInteractionService, @@ -73,6 +76,12 @@ import { IPIIServiceType, IPortConnectionService, IPortConnectionServiceType, + IPurchaseService, + IPurchaseServiceType, + IScraperNavigationService, + IScraperNavigationServiceType, + IScraperService, + IScraperServiceType, ITokenPriceService, ITokenPriceServiceType, ITwitterService, @@ -196,5 +205,19 @@ export const extensionCoreModule = new ContainerModule( bind(IRpcEngineFactoryType) .to(RpcEngineFactory) .inSingletonScope(); + + //Scraper + + bind(IScraperServiceType) + .to(ScraperService) + .inSingletonScope(); + + bind(IScraperNavigationServiceType) + .to(ScraperNavigationService) + .inSingletonScope(); + + bind(IPurchaseServiceType) + .to(PurchaseService) + .inSingletonScope(); }, ); diff --git a/packages/synamint-extension-sdk/src/core/implementations/api/RpcCallHandler.ts b/packages/synamint-extension-sdk/src/core/implementations/api/RpcCallHandler.ts index 086978c708..25b2470000 100644 --- a/packages/synamint-extension-sdk/src/core/implementations/api/RpcCallHandler.ts +++ b/packages/synamint-extension-sdk/src/core/implementations/api/RpcCallHandler.ts @@ -41,6 +41,18 @@ import { IUserSiteInteractionService, IUserSiteInteractionServiceType, } from "@synamint-extension-sdk/core/interfaces/business"; +import { + IPurchaseService, + IPurchaseServiceType, +} from "@synamint-extension-sdk/core/interfaces/business/IPurchaseService"; +import { + IScraperNavigationService, + IScraperNavigationServiceType, +} from "@synamint-extension-sdk/core/interfaces/business/IScraperNavigationService"; +import { + IScraperService, + IScraperServiceType, +} from "@synamint-extension-sdk/core/interfaces/business/IScraperService"; import { IConfigProvider, IConfigProviderType, @@ -123,14 +135,25 @@ import { GetCurrentCloudStorageParams, RejectInvitationParams, GetQueryStatusesParams, - GetTransactionsParams, - GetTransactionValueByChainParams, + ScraperScrapeParams, + ScraperClassifyUrlParams, + ScraperGetOrderHistoryPageParams, + ScraperGetYearsParams, + ScraperGetOrderHistoryPageByYearParams, + ScraperGetPageCountParams, AddAccountWithExternalSignatureParams, AddAccountWithExternalTypedDataSignatureParams, ERequestChannel, + PurchaseGetByMarketPlaceParams, + PurchaseGetByMarketPlaceAndDateParams, + GetTransactionValueByChainParams, + GetTransactionsParams, UpdateAgreementPermissionsParams, SnickerDoodleCoreError, GetConsentContractURLsParams, + PurchaseGetShoppingDataConnectionStatusParams, + PurchaseSetShoppingDataConnectionStatusParams, + PurchaseGetPurchasedProductsParams, GetPersistenceNFTsParams, GetAccountNFTHistoryParams, GetAccountNftCacheParams, @@ -798,6 +821,25 @@ export class RpcCallHandler implements IRpcCallHandler { ), // #endregion + // #region Scraper + new CoreActionHandler( + ScraperScrapeParams.getCoreAction(), + (params) => { + return this.scraperService.scrape( + params.url, + params.html, + params.suggestedDomainTask, + ); + }, + ), + new CoreActionHandler( + ScraperClassifyUrlParams.getCoreAction(), + (params) => { + return this.scraperService.classifyURL(params.url, params.language); + }, + ), + // #endregion + // #region external local storage calls new CoreActionHandler( GetUIStateParams.getCoreAction(), @@ -819,6 +861,89 @@ export class RpcCallHandler implements IRpcCallHandler { ), // #endregion + // #region Scraper Navigation + + new CoreActionHandler( + ScraperGetOrderHistoryPageParams.getCoreAction(), + (params) => { + const url = this.scrapernavigationService.getOrderHistoryPage( + params.lang, + params.page, + ); + return okAsync(url); + }, + ), + + new CoreActionHandler( + ScraperGetYearsParams.getCoreAction(), + (params) => { + const url = this.scrapernavigationService.getYears(params.html); + return okAsync(url); + }, + ), + + new CoreActionHandler( + ScraperGetOrderHistoryPageByYearParams.getCoreAction(), + (params) => { + const url = this.scrapernavigationService.getOrderHistoryPageByYear( + params.lang, + params.year, + params.page, + ); + return okAsync(url); + }, + ), + + new CoreActionHandler( + ScraperGetPageCountParams.getCoreAction(), + (params) => { + const url = this.scrapernavigationService.getPageCount( + params.html, + params.year, + ); + return okAsync(url); + }, + ), + + // #endregion + + // #region Purchase + new CoreActionHandler( + PurchaseGetPurchasedProductsParams.getCoreAction(), + (_params) => { + return this.purchaseService.getPurchasedProducts(); + }, + ), + new CoreActionHandler( + PurchaseGetByMarketPlaceParams.getCoreAction(), + (params) => { + return this.purchaseService.getByMarketplace(params.marketPlace); + }, + ), + new CoreActionHandler( + PurchaseGetByMarketPlaceAndDateParams.getCoreAction(), + (params) => { + return this.purchaseService.getByMarketplaceAndDate( + params.marketPlace, + params.datePurchased, + ); + }, + ), + new CoreActionHandler( + PurchaseGetShoppingDataConnectionStatusParams.getCoreAction(), + (_params) => { + return this.purchaseService.getShoppingDataConnectionStatus(); + }, + ), + new CoreActionHandler( + PurchaseSetShoppingDataConnectionStatusParams.getCoreAction(), + (params) => { + return this.purchaseService.setShoppingDataConnectionStatus( + params.ShoppingDataConnectionStatus, + ); + }, + ), + // #endregion // #region questionnaires new CoreActionHandler( GetAllQuestionnairesParams.getCoreAction(), @@ -886,6 +1011,12 @@ export class RpcCallHandler implements IRpcCallHandler { protected userSiteInteractionService: IUserSiteInteractionService, @inject(IDiscordServiceType) protected discordService: IDiscordService, + @inject(IPurchaseServiceType) + protected purchaseService: IPurchaseService, + @inject(IScraperServiceType) + protected scraperService: IScraperService, + @inject(IScraperNavigationServiceType) + protected scrapernavigationService: IScraperNavigationService, @inject(ITwitterServiceType) protected twitterService: ITwitterService, @inject(IConfigProviderType) diff --git a/packages/synamint-extension-sdk/src/core/implementations/business/PurchaseService.ts b/packages/synamint-extension-sdk/src/core/implementations/business/PurchaseService.ts new file mode 100644 index 0000000000..81f310f461 --- /dev/null +++ b/packages/synamint-extension-sdk/src/core/implementations/business/PurchaseService.ts @@ -0,0 +1,72 @@ +import { + DomainName, + ISnickerdoodleCore, + ISnickerdoodleCoreType, + PersistenceError, + PurchasedProduct, + ShoppingDataConnectionStatus, + UnixTimestamp, +} from "@snickerdoodlelabs/objects"; +import { inject, injectable } from "inversify"; +import { ResultAsync } from "neverthrow"; + +import { IPurchaseService } from "@synamint-extension-sdk/core/interfaces/business/IPurchaseService"; +import { + IErrorUtils, + IErrorUtilsType, +} from "@synamint-extension-sdk/core/interfaces/utilities"; + +@injectable() +export class PurchaseService implements IPurchaseService { + constructor( + @inject(ISnickerdoodleCoreType) protected core: ISnickerdoodleCore, + @inject(IErrorUtilsType) protected errorUtils: IErrorUtils, + ) {} + getPurchasedProducts(): ResultAsync { + return this.core.purchase.getPurchasedProducts().mapErr((error) => { + this.errorUtils.emit(error); + return new PersistenceError((error as Error).message, error); + }); + } + getByMarketplace( + marketPlace: DomainName, + ): ResultAsync { + return this.core.purchase.getByMarketplace(marketPlace).mapErr((error) => { + this.errorUtils.emit(error); + return new PersistenceError((error as Error).message, error); + }); + } + getByMarketplaceAndDate( + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ): ResultAsync { + return this.core.purchase + .getByMarketplaceAndDate(marketPlace, datePurchased) + .mapErr((error) => { + this.errorUtils.emit(error); + return new PersistenceError((error as Error).message, error); + }); + } + + getShoppingDataConnectionStatus(): ResultAsync< + ShoppingDataConnectionStatus[], + PersistenceError + > { + return this.core.purchase + .getShoppingDataConnectionStatus() + .mapErr((error) => { + this.errorUtils.emit(error); + return new PersistenceError((error as Error).message, error); + }); + } + setShoppingDataConnectionStatus( + ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ): ResultAsync { + return this.core.purchase + .setShoppingDataConnectionStatus(ShoppingDataConnectionStatus) + .mapErr((error) => { + this.errorUtils.emit(error); + return new PersistenceError((error as Error).message, error); + }); + } +} diff --git a/packages/synamint-extension-sdk/src/core/implementations/business/ScraperNavigationService.ts b/packages/synamint-extension-sdk/src/core/implementations/business/ScraperNavigationService.ts new file mode 100644 index 0000000000..723aada1de --- /dev/null +++ b/packages/synamint-extension-sdk/src/core/implementations/business/ScraperNavigationService.ts @@ -0,0 +1,44 @@ +import { + ELanguageCode, + HTMLString, + ISnickerdoodleCore, + ISnickerdoodleCoreType, + PageNumber, + URLString, + Year, +} from "@snickerdoodlelabs/objects"; +import { inject, injectable } from "inversify"; + +import { IScraperNavigationService } from "@synamint-extension-sdk/core/interfaces/business/IScraperNavigationService"; +import { + IErrorUtils, + IErrorUtilsType, +} from "@synamint-extension-sdk/core/interfaces/utilities"; + +@injectable() +export class ScraperNavigationService implements IScraperNavigationService { + constructor( + @inject(ISnickerdoodleCoreType) protected core: ISnickerdoodleCore, + @inject(IErrorUtilsType) protected errorUtils: IErrorUtils, + ) {} + getOrderHistoryPage(lang: ELanguageCode, page: PageNumber): URLString { + return this.core.scraperNavigation.amazon.getOrderHistoryPage(lang, page); + } + getYears(html: HTMLString): Year[] { + return this.core.scraperNavigation.amazon.getYears(html); + } + getOrderHistoryPageByYear( + lang: ELanguageCode, + year: Year, + page: PageNumber, + ): URLString { + return this.core.scraperNavigation.amazon.getOrderHistoryPageByYear( + lang, + year, + page, + ); + } + getPageCount(html: HTMLString, year: Year): number { + return this.core.scraperNavigation.amazon.getPageCount(html, year); + } +} diff --git a/packages/synamint-extension-sdk/src/core/implementations/business/ScraperService.ts b/packages/synamint-extension-sdk/src/core/implementations/business/ScraperService.ts new file mode 100644 index 0000000000..b498e22ff2 --- /dev/null +++ b/packages/synamint-extension-sdk/src/core/implementations/business/ScraperService.ts @@ -0,0 +1,51 @@ +import { + DiscordGuildProfile, + DiscordID, + DiscordProfile, + DomainName, + DomainTask, + ELanguageCode, + HTMLString, + ISnickerdoodleCore, + ISnickerdoodleCoreType, + OAuthAuthorizationCode, + ScraperError, + URLString, +} from "@snickerdoodlelabs/objects"; +import { inject, injectable } from "inversify"; +import { ResultAsync } from "neverthrow"; + +import { IScraperService } from "@synamint-extension-sdk/core/interfaces/business/IScraperService"; +import { + IErrorUtils, + IErrorUtilsType, +} from "@synamint-extension-sdk/core/interfaces/utilities"; + +@injectable() +export class ScraperService implements IScraperService { + constructor( + @inject(ISnickerdoodleCoreType) protected core: ISnickerdoodleCore, + @inject(IErrorUtilsType) protected errorUtils: IErrorUtils, + ) {} + scrape( + url: URLString, + html: HTMLString, + suggestedDomainTask: DomainTask, + ): ResultAsync { + return this.core.scraper + .scrape(url, html, suggestedDomainTask) + .mapErr((error) => { + this.errorUtils.emit(error); + return new ScraperError((error as Error).message, error); + }); + } + classifyURL( + url: URLString, + language: ELanguageCode, + ): ResultAsync { + return this.core.scraper.classifyURL(url, language).mapErr((error) => { + this.errorUtils.emit(error); + return new ScraperError((error as Error).message, error); + }); + } +} diff --git a/packages/synamint-extension-sdk/src/core/implementations/business/index.ts b/packages/synamint-extension-sdk/src/core/implementations/business/index.ts index e93257fd4f..f71e5c50a1 100644 --- a/packages/synamint-extension-sdk/src/core/implementations/business/index.ts +++ b/packages/synamint-extension-sdk/src/core/implementations/business/index.ts @@ -5,6 +5,9 @@ export * from "@synamint-extension-sdk/core/implementations/business/InvitationS export * from "@synamint-extension-sdk/core/implementations/business/MetricsService"; export * from "@synamint-extension-sdk/core/implementations/business/PIIService"; export * from "@synamint-extension-sdk/core/implementations/business/PortConnectionService"; +export * from "@synamint-extension-sdk/core/implementations/business/PurchaseService"; +export * from "@synamint-extension-sdk/core/implementations/business/ScraperNavigationService"; +export * from "@synamint-extension-sdk/core/implementations/business/ScraperService"; export * from "@synamint-extension-sdk/core/implementations/business/TokenPriceService"; export * from "@synamint-extension-sdk/core/implementations/business/TwitterService"; export * from "@synamint-extension-sdk/core/implementations/business/UserSiteInteractionService"; diff --git a/packages/synamint-extension-sdk/src/core/interfaces/business/IPurchaseService.ts b/packages/synamint-extension-sdk/src/core/interfaces/business/IPurchaseService.ts new file mode 100644 index 0000000000..8b46fbbf16 --- /dev/null +++ b/packages/synamint-extension-sdk/src/core/interfaces/business/IPurchaseService.ts @@ -0,0 +1,28 @@ +import { + DomainName, + PersistenceError, + PurchasedProduct, + ShoppingDataConnectionStatus, + UnixTimestamp, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface IPurchaseService { + getPurchasedProducts(): ResultAsync; + getByMarketplace( + marketPlace: DomainName, + ): ResultAsync; + getByMarketplaceAndDate( + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ): ResultAsync; + getShoppingDataConnectionStatus(): ResultAsync< + ShoppingDataConnectionStatus[], + PersistenceError + >; + setShoppingDataConnectionStatus( + ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ): ResultAsync; +} + +export const IPurchaseServiceType = Symbol.for("IPurchaseService"); diff --git a/packages/synamint-extension-sdk/src/core/interfaces/business/IScraperNavigationService.ts b/packages/synamint-extension-sdk/src/core/interfaces/business/IScraperNavigationService.ts new file mode 100644 index 0000000000..12e58b9125 --- /dev/null +++ b/packages/synamint-extension-sdk/src/core/interfaces/business/IScraperNavigationService.ts @@ -0,0 +1,21 @@ +import { + ELanguageCode, + HTMLString, + PageNumber, + URLString, + Year, +} from "@snickerdoodlelabs/objects"; + +export interface IScraperNavigationService { + getOrderHistoryPage(lang: ELanguageCode, page: PageNumber): URLString; + getYears(html: HTMLString): Year[]; + getOrderHistoryPageByYear( + lang: ELanguageCode, + year: Year, + page: PageNumber, + ): URLString; + getPageCount(html: HTMLString, year: Year): number; +} +export const IScraperNavigationServiceType = Symbol.for( + "IScraperNavigationService", +); diff --git a/packages/synamint-extension-sdk/src/core/interfaces/business/IScraperService.ts b/packages/synamint-extension-sdk/src/core/interfaces/business/IScraperService.ts new file mode 100644 index 0000000000..c0476112a3 --- /dev/null +++ b/packages/synamint-extension-sdk/src/core/interfaces/business/IScraperService.ts @@ -0,0 +1,35 @@ +import { + URLString, + HTMLString, + ScraperError, + ELanguageCode, + DomainTask, +} from "@snickerdoodlelabs/objects"; +import { ResultAsync } from "neverthrow"; + +export interface IScraperService { + /** + * This method to extract information from a website + * @param url + * @param html + * @param suggestedDomainTask + */ + scrape( + url: URLString, + html: HTMLString, + suggestedDomainTask: DomainTask, + ): ResultAsync; + + /** + * + * @param url url of a website to classify + * @param language language. Just pass en for now + */ + + classifyURL( + url: URLString, + language: ELanguageCode, + ): ResultAsync; +} + +export const IScraperServiceType = Symbol.for("IScraperService"); diff --git a/packages/synamint-extension-sdk/src/core/interfaces/business/index.ts b/packages/synamint-extension-sdk/src/core/interfaces/business/index.ts index 60de2853dd..92856cbcba 100644 --- a/packages/synamint-extension-sdk/src/core/interfaces/business/index.ts +++ b/packages/synamint-extension-sdk/src/core/interfaces/business/index.ts @@ -5,6 +5,9 @@ export * from "@synamint-extension-sdk/core/interfaces/business/IInvitationServi export * from "@synamint-extension-sdk/core/interfaces/business/IMetricsService"; export * from "@synamint-extension-sdk/core/interfaces/business/IPIIService"; export * from "@synamint-extension-sdk/core/interfaces/business/IPortConnectionService"; +export * from "@synamint-extension-sdk/core/interfaces/business/IPurchaseService"; +export * from "@synamint-extension-sdk/core/interfaces/business/IScraperNavigationService"; +export * from "@synamint-extension-sdk/core/interfaces/business/IScraperService"; export * from "@synamint-extension-sdk/core/interfaces/business/ITokenPriceService"; export * from "@synamint-extension-sdk/core/interfaces/business/ITwitterService"; export * from "@synamint-extension-sdk/core/interfaces/business/IUserSiteInteractionService"; diff --git a/packages/synamint-extension-sdk/src/gateways/ExternalCoreGateway.ts b/packages/synamint-extension-sdk/src/gateways/ExternalCoreGateway.ts index 700c3b566f..2c81c1ffa2 100644 --- a/packages/synamint-extension-sdk/src/gateways/ExternalCoreGateway.ts +++ b/packages/synamint-extension-sdk/src/gateways/ExternalCoreGateway.ts @@ -47,15 +47,27 @@ import { QueryStatus, ECloudStorageType, OAuth2Tokens, + ScraperError, + DomainTask, + HTMLString, + ELanguageCode, + PageNumber, + Year, TransactionFlowInsight, ChainTransaction, IProxyAccountMethods, LanguageCode, EChain, Signature, + IProxyPurchaseMethods, + PurchasedProduct, + IScraperNavigationMethods, + IScraperMethods, + GetResultAsyncValueType, IUserAgreement, PageInvitation, Invitation, + ShoppingDataConnectionStatus, INftProxyMethods, JSONString, IProxyQuestionnaireMethods, @@ -65,6 +77,7 @@ import { import { ethers } from "ethers"; import { JsonRpcEngine } from "json-rpc-engine"; import { ResultAsync } from "neverthrow"; +import { FunctionKeys } from "utility-types"; import CoreHandler from "@synamint-extension-sdk/gateways/handler/CoreHandler"; import { @@ -136,12 +149,23 @@ import { GetCurrentCloudStorageParams, RejectInvitationParams, GetQueryStatusesParams, - GetTransactionValueByChainParams, - GetTransactionsParams, + ScraperScrapeParams, + ScraperClassifyUrlParams, + ScraperGetOrderHistoryPageParams, + ScraperGetYearsParams, + ScraperGetOrderHistoryPageByYearParams, + ScraperGetPageCountParams, AddAccountWithExternalSignatureParams, AddAccountWithExternalTypedDataSignatureParams, + PurchaseGetByMarketPlaceParams, + PurchaseGetByMarketPlaceAndDateParams, + GetTransactionsParams, + GetTransactionValueByChainParams, UpdateAgreementPermissionsParams, GetConsentContractURLsParams, + PurchaseGetShoppingDataConnectionStatusParams, + PurchaseSetShoppingDataConnectionStatusParams, + PurchaseGetPurchasedProductsParams, GetPersistenceNFTsParams, GetAccountNFTHistoryParams, GetAccountNftCacheParams, @@ -155,12 +179,34 @@ import { } from "@synamint-extension-sdk/shared"; import { IExtensionConfig } from "@synamint-extension-sdk/shared/interfaces/IExtensionConfig"; +// We are not passing all the functions in IScraperNavigationMethods to proxy thats why we need to redifine type for gateway +type IGatewayScraperNavigationMethods = { + [key in FunctionKeys]: ( + ...args: [...Parameters] + ) => ResultAsync< + ReturnType, + ProxyError + >; +}; + +type IGatewayScraperMethods = { + [key in FunctionKeys]: ( + ...args: [...Parameters] + ) => ResultAsync< + GetResultAsyncValueType>, + ProxyError + >; +}; + export class ExternalCoreGateway { public account: IProxyAccountMethods; public discord: IProxyDiscordMethods; public integration: IProxyIntegrationMethods; public metrics: IProxyMetricsMethods; public twitter: IProxyTwitterMethods; + public purchase: IProxyPurchaseMethods; + public scraper: IGatewayScraperMethods; + public scraperNavigation: IGatewayScraperNavigationMethods; public nft: INftProxyMethods; public questionnaire: IProxyQuestionnaireMethods; protected _handler: CoreHandler; @@ -366,6 +412,91 @@ export class ExternalCoreGateway { return this._handler.call(new TwitterGetLinkedProfilesParams()); }, }; + + this.scraper = { + scrape: ( + url: URLString, + html: HTMLString, + suggestedDomainTask: DomainTask, + ): ResultAsync => { + return this._handler.call( + new ScraperScrapeParams(url, html, suggestedDomainTask), + ); + }, + classifyURL: ( + url: URLString, + language: ELanguageCode, + ): ResultAsync => { + return this._handler.call(new ScraperClassifyUrlParams(url, language)); + }, + }; + + this.scraperNavigation = { + getOrderHistoryPage: ( + lang: ELanguageCode, + page: PageNumber, + ): ResultAsync => { + return this._handler.call( + new ScraperGetOrderHistoryPageParams(lang, page), + ); + }, + getYears: (html: HTMLString): ResultAsync => { + return this._handler.call(new ScraperGetYearsParams(html)); + }, + getOrderHistoryPageByYear: ( + lang: ELanguageCode, + year: Year, + page: PageNumber, + ): ResultAsync => { + return this._handler.call( + new ScraperGetOrderHistoryPageByYearParams(lang, year, page), + ); + }, + getPageCount: ( + html: HTMLString, + year: Year, + ): ResultAsync => { + return this._handler.call(new ScraperGetPageCountParams(html, year)); + }, + }; + + this.purchase = { + getPurchasedProducts: (): ResultAsync => { + return this._handler.call(new PurchaseGetPurchasedProductsParams()); + }, + getByMarketplace: ( + marketPlace: DomainName, + ): ResultAsync => { + return this._handler.call( + new PurchaseGetByMarketPlaceParams(marketPlace), + ); + }, + getByMarketplaceAndDate: ( + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ): ResultAsync => { + return this._handler.call( + new PurchaseGetByMarketPlaceAndDateParams(marketPlace, datePurchased), + ); + }, + getShoppingDataConnectionStatus: (): ResultAsync< + ShoppingDataConnectionStatus[], + ProxyError + > => { + return this._handler.call( + new PurchaseGetShoppingDataConnectionStatusParams(), + ); + }, + setShoppingDataConnectionStatus: ( + ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ): ResultAsync => { + return this._handler.call( + new PurchaseSetShoppingDataConnectionStatusParams( + ShoppingDataConnectionStatus, + ), + ); + }, + }; } public updateRpcEngine(rpcEngine: JsonRpcEngine) { diff --git a/packages/synamint-extension-sdk/src/shared/enums/ECoreActions.ts b/packages/synamint-extension-sdk/src/shared/enums/ECoreActions.ts index c3708bef9e..b5d87a632f 100644 --- a/packages/synamint-extension-sdk/src/shared/enums/ECoreActions.ts +++ b/packages/synamint-extension-sdk/src/shared/enums/ECoreActions.ts @@ -91,6 +91,19 @@ export enum ECoreActions { GET_AVAILABLE_CLOUD_STORAGE_OPTIONS = "GET_AVAILABLE_CLOUD_STORAGE_OPTIONS", GET_CURRENT_STORAGE_TYPE = "GET_CURRENT_STORAGE_TYPE", + SCRAPER_CLASSIFY_URL_PARAMS = "SCRAPER_CLASSIFY_URL_PARAMS", + SCRAPER_SCRAPE_PARAMS = "SCRAPER_SCRAPE_PARAMS", + + SCRAPER_NAVIGATION_GET_ORDER_HISTORY_PAGE_PARAMS = "SCRAPER_NAVIGATION_GET_ORDER_HISTORY_PAGE_PARAMS", + SCRAPER_NAVIGATION_GET_YEARS_PARAMS = "SCRAPER_NAVIGATION_GET_YEARS_PARAMS", + SCRAPER_NAVIGATION_GET_ORDER_HISTORY_PAGE_BY_YEAR_PARAMS = "SCRAPER_NAVIGATION_GET_ORDER_HISTORY_PAGE_BY_YEAR_PARAMS", + SCRAPER_NAVIGATION_GET_PAGE_COUNT_PARAMS = "SCRAPER_NAVIGATION_GET_PAGE_COUNT_PARAMS", + + PURCHASE_GET_PURCHASED_PRODUCTS_PARAMS = "PURCHASE_GET_PURCHASED_PRODUCTS_PARAMS", + PURCHASE_GET_BY_MARKET_PLACE = "PURCHASE_GET_BY_MARKET_PLACE", + PURCHASE_GET_BY_MARKET_PLACE_AND_DATE = "PURCHASE_GET_BY_MARKET_PLACE_AND_DATE", + PURCHASE_GET_SHOPPINGDATA_CONNECTION_STATUS = "PURCHASE_GET_SHOPPINGDATA_CONNECTION_STATUS", + PURCHASE_SET_SHOPPINGDATA_CONNECTION_STATUS = "PURCHASE_SET_SHOPPINGDATA_CONNECTION_STATUS", // External local storage calls SET_UI_STATE = "SET_UI_STATE", GET_UI_STATE = "GET_UI_STATE", diff --git a/packages/synamint-extension-sdk/src/shared/interfaces/actions.ts b/packages/synamint-extension-sdk/src/shared/interfaces/actions.ts index fd10b300d9..ec61b83321 100644 --- a/packages/synamint-extension-sdk/src/shared/interfaces/actions.ts +++ b/packages/synamint-extension-sdk/src/shared/interfaces/actions.ts @@ -52,10 +52,17 @@ import { BlockNumber, RefreshToken, OAuth2Tokens, + HTMLString, + DomainTask, + ELanguageCode, + PageNumber, + Year, + PurchasedProduct, TransactionFlowInsight, ChainTransaction, TransactionFilter, IUserAgreement, + ShoppingDataConnectionStatus, WalletNFTHistory, WalletNFTData, Questionnaire, @@ -889,6 +896,133 @@ export class GetCurrentCloudStorageParams extends CoreActionParams { + public constructor( + public url: URLString, + public html: HTMLString, + public suggestedDomainTask: DomainTask, + ) { + super(ScraperScrapeParams.getCoreAction()); + } + static getCoreAction(): ECoreActions { + return ECoreActions.SCRAPER_SCRAPE_PARAMS; + } +} + +export class ScraperClassifyUrlParams extends CoreActionParams { + public constructor(public url: URLString, public language: ELanguageCode) { + super(ScraperClassifyUrlParams.getCoreAction()); + } + static getCoreAction(): ECoreActions { + return ECoreActions.SCRAPER_CLASSIFY_URL_PARAMS; + } +} +// #endregion + +// #region Scraper Navigation + +export class ScraperGetOrderHistoryPageParams extends CoreActionParams { + public constructor(public lang: ELanguageCode, public page: PageNumber) { + super(ScraperGetOrderHistoryPageParams.getCoreAction()); + } + static getCoreAction(): ECoreActions { + return ECoreActions.SCRAPER_NAVIGATION_GET_ORDER_HISTORY_PAGE_PARAMS; + } +} + +export class ScraperGetYearsParams extends CoreActionParams { + public constructor(public html: HTMLString) { + super(ScraperGetYearsParams.getCoreAction()); + } + static getCoreAction(): ECoreActions { + return ECoreActions.SCRAPER_NAVIGATION_GET_YEARS_PARAMS; + } +} + +export class ScraperGetOrderHistoryPageByYearParams extends CoreActionParams { + public constructor( + public lang: ELanguageCode, + public year: Year, + public page: PageNumber, + ) { + super(ScraperGetOrderHistoryPageByYearParams.getCoreAction()); + } + static getCoreAction(): ECoreActions { + return ECoreActions.SCRAPER_NAVIGATION_GET_ORDER_HISTORY_PAGE_BY_YEAR_PARAMS; + } +} +export class ScraperGetPageCountParams extends CoreActionParams { + public constructor(public html: HTMLString, public year: Year) { + super(ScraperGetPageCountParams.getCoreAction()); + } + static getCoreAction(): ECoreActions { + return ECoreActions.SCRAPER_NAVIGATION_GET_PAGE_COUNT_PARAMS; + } +} + +// #endregion + +// #region Purchase +export class PurchaseGetPurchasedProductsParams extends CoreActionParams< + PurchasedProduct[] +> { + public constructor() { + super(PurchaseGetPurchasedProductsParams.getCoreAction()); + } + static getCoreAction(): ECoreActions { + return ECoreActions.PURCHASE_GET_PURCHASED_PRODUCTS_PARAMS; + } +} + +export class PurchaseGetByMarketPlaceParams extends CoreActionParams< + PurchasedProduct[] +> { + public constructor(public marketPlace: DomainName) { + super(PurchaseGetByMarketPlaceParams.getCoreAction()); + } + static getCoreAction(): ECoreActions { + return ECoreActions.PURCHASE_GET_BY_MARKET_PLACE; + } +} + +export class PurchaseGetByMarketPlaceAndDateParams extends CoreActionParams< + PurchasedProduct[] +> { + public constructor( + public marketPlace: DomainName, + public datePurchased: UnixTimestamp, + ) { + super(PurchaseGetByMarketPlaceAndDateParams.getCoreAction()); + } + static getCoreAction(): ECoreActions { + return ECoreActions.PURCHASE_GET_BY_MARKET_PLACE_AND_DATE; + } +} + +export class PurchaseGetShoppingDataConnectionStatusParams extends CoreActionParams< + ShoppingDataConnectionStatus[] +> { + public constructor() { + super(PurchaseGetShoppingDataConnectionStatusParams.getCoreAction()); + } + static getCoreAction(): ECoreActions { + return ECoreActions.PURCHASE_GET_SHOPPINGDATA_CONNECTION_STATUS; + } +} +export class PurchaseSetShoppingDataConnectionStatusParams extends CoreActionParams { + public constructor( + public ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ) { + super(PurchaseSetShoppingDataConnectionStatusParams.getCoreAction()); + } + static getCoreAction(): ECoreActions { + return ECoreActions.PURCHASE_SET_SHOPPINGDATA_CONNECTION_STATUS; + } +} // #endregion // #region External local storage calls diff --git a/packages/test-harness/package.json b/packages/test-harness/package.json index 04589355ef..5f71e843f6 100644 --- a/packages/test-harness/package.json +++ b/packages/test-harness/package.json @@ -31,7 +31,9 @@ "clean": "npx rimraf dist tsconfig.tsbuildinfo", "compile": "npx tsc --build && cd ../.. && yarn alias", "start": "yarn compile && node dist/ConsoleApp.js", - "start-old": "yarn compile && node dist/index-deleted-but-kept-for-ref.js", + "just-start": "node dist/ConsoleApp.js", + "just-start-16": "node --experimental-json-modules dist/ConsoleApp.js", + "start-old": "yarn compile && node dist/index-deleted-but-kept-for-ref.js", "test": "NODE_OPTIONS=--experimental-vm-modules npx jest --maxWorkers=50% --coverage --passWithNoTests", "test-check": "npx ts-node-esm ../../node_modules/@cucumber/cucumber/bin/cucumber-js -p default --dry-run", "test-int": "npx cucumber-js" diff --git a/packages/test-harness/src/prompts/CorePrompt.ts b/packages/test-harness/src/prompts/CorePrompt.ts index afe879ec36..b51266286e 100644 --- a/packages/test-harness/src/prompts/CorePrompt.ts +++ b/packages/test-harness/src/prompts/CorePrompt.ts @@ -27,6 +27,7 @@ import { inquiryWrapper } from "@test-harness/prompts/inquiryWrapper.js"; import { OptInCampaign } from "@test-harness/prompts/OptInCampaign.js"; import { OptOutCampaign } from "@test-harness/prompts/OptOutCampaign.js"; import { RemoveAccount } from "@test-harness/prompts/RemoveAccount.js"; +import { ScraperPrompt } from "@test-harness/prompts/ScraperPrompt.js"; import { SelectProfile } from "@test-harness/prompts/SelectProfile.js"; import { UpdateDataPermissions } from "@test-harness/prompts/UpdateDataPermissions.js"; @@ -37,6 +38,7 @@ export class CorePrompt extends DataWalletPrompt { private optOutCampaign: OptOutCampaign; private selectProfile: SelectProfile; private updateDataPermissions: UpdateDataPermissions; + private scraperService: ScraperPrompt; public constructor(public env: Environment, protected timeUtils: ITimeUtils) { super(env); @@ -47,6 +49,7 @@ export class CorePrompt extends DataWalletPrompt { this.optOutCampaign = new OptOutCampaign(this.env); this.selectProfile = new SelectProfile(this.env); this.updateDataPermissions = new UpdateDataPermissions(this.env); + this.scraperService = new ScraperPrompt(this.env); } public start(): ResultAsync { @@ -118,6 +121,7 @@ export class CorePrompt extends DataWalletPrompt { const choices = [ { name: "NOOP", value: "NOOP" }, { name: "Switch Profile", value: "selectProfile" }, + { name: "Scraper Service", value: "scraperService" }, new inquirer.Separator(), ...choicesWhenUnlocked, new inquirer.Separator(), @@ -148,6 +152,8 @@ export class CorePrompt extends DataWalletPrompt { return okAsync(undefined); case "selectProfile": return this.selectProfile.start(); + case "scraperService": + return this.scraperService.start(); case "addAccount": return this.addAccount.start(); case "removeAccount": diff --git a/packages/test-harness/src/prompts/ScraperPrompt.ts b/packages/test-harness/src/prompts/ScraperPrompt.ts new file mode 100644 index 0000000000..a82ee47221 --- /dev/null +++ b/packages/test-harness/src/prompts/ScraperPrompt.ts @@ -0,0 +1,67 @@ +import { IConfigProvider, IConfigProviderType } from "@snickerdoodlelabs/core"; +import { Container } from "inversify"; +import { okAsync, ResultAsync } from "neverthrow"; + +import { inquiryWrapper } from "@test-harness/prompts/inquiryWrapper.js"; +import { Prompt } from "@test-harness/prompts/Prompt.js"; + +export class ScraperPrompt extends Prompt { + public start(): ResultAsync { + const choices = [ + { name: "Cancel", value: "Cancel" }, + { name: "Add OPENAI_API_KEY", value: "OPENAI_API_KEY" }, + { name: "classify URL", value: "classifyURL" }, + { name: "scrape a page", value: "scrape" }, + ]; + + return inquiryWrapper([ + { + type: "list", + name: "scraperPromptSelector", + message: "Choose action", + choices: choices, + }, + ]) + .andThen((answers) => { + switch (answers.scraperPromptSelector) { + case "OPENAI_API_KEY": + // TODO initialize with OPENAI_API_KEY + + const choice = [ + { + type: "input", + name: "OPENAI_API_KEY", + message: "Enter your OpenAI API key", + }, + ]; + return inquiryWrapper(choice).andThen((answers) => { + const apiKey = answers.OPENAI_API_KEY; + const iocContainer = this.core["iocContainer"] as Container; + const configProvider = + iocContainer.get(IConfigProviderType); + configProvider.setConfigOverrides({ + scraper: { + OPENAI_API_KEY: apiKey, + timeout: 5 * 60 * 1000, // 5 minutes + }, + }); + return okAsync(undefined); + }); + case "classifyURL": + // TODO initialize with classifyURL + break; + case "scrape": + // TODO initialize with scrape + break; + case "Cancel": + return okAsync(undefined); //going back + } + return this.start(); + }) + .mapErr((e) => { + console.error(e); + return e; + }) + .map(() => {}); + } +} diff --git a/packages/web-integration/src/implementations/proxy/SnickerdoodleIFrameProxy.ts b/packages/web-integration/src/implementations/proxy/SnickerdoodleIFrameProxy.ts index 11372462e6..b3dab66475 100644 --- a/packages/web-integration/src/implementations/proxy/SnickerdoodleIFrameProxy.ts +++ b/packages/web-integration/src/implementations/proxy/SnickerdoodleIFrameProxy.ts @@ -72,11 +72,21 @@ import { BlockNumber, RefreshToken, OAuth2Tokens, + ELanguageCode, + HTMLString, + ScraperError, + IProxyScraperNavigationMethods, + PageNumber, + Year, TransactionFlowInsight, IProxyAccountMethods, + IProxyPurchaseMethods, + PersistenceError, + PurchasedProduct, ChainTransaction, TransactionFilter, IUserAgreement, + ShoppingDataConnectionStatus, INftProxyMethods, WalletNFTHistory, NftRepositoryCache, @@ -775,6 +785,57 @@ export class SnickerdoodleIFrameProxy }, }; + public scrapernavigation: IProxyScraperNavigationMethods = { + getOrderHistoryPage: ( + lang: ELanguageCode, + page: PageNumber, + ): ResultAsync => { + return this._createCall("scrapernavigation.amazon.getOrderHistoryPage", { + lang, + page, + }); + }, + }; + + public purchase: IProxyPurchaseMethods = { + getPurchasedProducts: (): ResultAsync => { + return this._createCall("purchase.getPurchasedProducts", {}); + }, + getByMarketplace: ( + marketPlace: DomainName, + ): ResultAsync => { + return this._createCall("purchase.getByMarketplace", { marketPlace }); + }, + getByMarketplaceAndDate: ( + marketPlace: DomainName, + datePurchased: UnixTimestamp, + ): ResultAsync => { + return this._createCall("purchase.getByMarketplaceAndDate", { + marketPlace, + datePurchased, + }); + }, + getShoppingDataConnectionStatus: (): ResultAsync< + ShoppingDataConnectionStatus[], + ProxyError + > => { + return this._createCall("purchase.getShoppingDataConnectionStatus", {}); + }, + setShoppingDataConnectionStatus: ( + ShoppingDataConnectionStatus: ShoppingDataConnectionStatus, + ): ResultAsync => { + return this._createCall("purchase.setShoppingDataConnectionStatus", { + ShoppingDataConnectionStatus, + }); + }, + }; + public setUIState(state: JSONString): ResultAsync { + return this._createCall("setUIState", { state }); + } + public getUIState(): ResultAsync { + return this._createCall("getUIState", null); + } + public questionnaire: IProxyQuestionnaireMethods = { getAllQuestionnaires: (pagingRequest: PagingRequest) => { return this._createCall("questionnaire.getAllQuestionnaires", { @@ -817,13 +878,6 @@ export class SnickerdoodleIFrameProxy }, }; - public setUIState(state: JSONString): ResultAsync { - return this._createCall("setUIState", { state }); - } - public getUIState(): ResultAsync { - return this._createCall("getUIState", null); - } - public events: PublicEvents; private _displayCoreIFrame(): void { diff --git a/tsconfig.json b/tsconfig.json index d9068cbb1f..e1813dc4bd 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -56,6 +56,12 @@ { "path": "packages/insight-platform-api/test" }, + { + "path": "packages/nlp" + }, + { + "path": "packages/nlp/test" + }, { "path": "packages/node-utils" }, @@ -89,6 +95,12 @@ { "path": "packages/signatureVerification/test" }, + { + "path": "packages/shopping-data" + }, + { + "path": "packages/shopping-data/test" + }, { "path": "packages/static-web-integration" }, diff --git a/yarn.lock b/yarn.lock index 033dd9ba67..31aeb4dee5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6464,6 +6464,32 @@ __metadata: languageName: node linkType: hard +"@nlpjs/core@npm:^4.26.1": + version: 4.26.1 + resolution: "@nlpjs/core@npm:4.26.1" + checksum: 9ca4d73efec085b3873bbe67ed6189d1bae6d74b7daaf366b44153631706f13446a1d23451412cd4ae1fc128a9adc1a72a5513f424a4eaf663cb33415b249fe1 + languageName: node + linkType: hard + +"@nlpjs/lang-en-min@npm:^4.26.1": + version: 4.26.1 + resolution: "@nlpjs/lang-en-min@npm:4.26.1" + dependencies: + "@nlpjs/core": ^4.26.1 + checksum: 2f6171a17582dbc3357bb5d67060bbab725b70a2b9ba99b79dc264a4170634d4c8b10748f0a0fe6cceb079cb0e256fd358c5b67442b3ea13016c461e97d100f6 + languageName: node + linkType: hard + +"@nlpjs/lang-en@npm:^4.26.1": + version: 4.26.1 + resolution: "@nlpjs/lang-en@npm:4.26.1" + dependencies: + "@nlpjs/core": ^4.26.1 + "@nlpjs/lang-en-min": ^4.26.1 + checksum: b6c5cb51652bec8b0291af1d134b3c4f2ea111aea030e3ee397305bcd29d8e4e4255d483f521b9fd57f7e7c2194e771df7f16ad0a2b985fd873bf3614728bb5d + languageName: node + linkType: hard + "@noble/ciphers@npm:^0.3.0": version: 0.3.0 resolution: "@noble/ciphers@npm:0.3.0" @@ -8659,6 +8685,16 @@ __metadata: languageName: node linkType: hard +"@selderee/plugin-htmlparser2@npm:^0.11.0": + version: 0.11.0 + resolution: "@selderee/plugin-htmlparser2@npm:0.11.0" + dependencies: + domhandler: ^5.0.3 + selderee: ^0.11.0 + checksum: 6deafedd153e492359f8f0407d20903d82f2ef4950e420f4b2ee6ffbb955753524631aac7d6a5fe61dc7c7893e6928b4d8409e886157ad64a60ab37bc08b17c4 + languageName: node + linkType: hard + "@sentry/core@npm:5.30.0": version: 5.30.0 resolution: "@sentry/core@npm:5.30.0" @@ -8890,16 +8926,21 @@ __metadata: languageName: node linkType: hard -"@snickerdoodlelabs/ai-scraper@workspace:packages/ai-scraper": +"@snickerdoodlelabs/ai-scraper@workspace:^, @snickerdoodlelabs/ai-scraper@workspace:packages/ai-scraper": version: 0.0.0-use.local resolution: "@snickerdoodlelabs/ai-scraper@workspace:packages/ai-scraper" dependencies: "@snickerdoodlelabs/common-utils": "workspace:^" "@snickerdoodlelabs/objects": "workspace:^" + "@snickerdoodlelabs/persistence": "workspace:^" + "@snickerdoodlelabs/shopping-data": "workspace:^" ethers: ^6.10.0 + html-to-text: ^9.0.5 inversify: ^6.0.2 + js-tiktoken: ^1.0.8 neverthrow: ^5.1.0 neverthrow-result-utils: ^2.0.2 + openai: ^4.0.1 languageName: unknown linkType: soft @@ -9019,14 +9060,17 @@ __metadata: version: 0.0.0-use.local resolution: "@snickerdoodlelabs/core@workspace:packages/core" dependencies: + "@snickerdoodlelabs/ai-scraper": "workspace:^" "@snickerdoodlelabs/common-utils": "workspace:^" "@snickerdoodlelabs/contracts-sdk": "workspace:^" "@snickerdoodlelabs/indexers": "workspace:^" "@snickerdoodlelabs/insight-platform-api": "workspace:^" + "@snickerdoodlelabs/nlp": "workspace:^" "@snickerdoodlelabs/node-utils": "workspace:^" "@snickerdoodlelabs/objects": "workspace:^" "@snickerdoodlelabs/persistence": "workspace:^" "@snickerdoodlelabs/query-parser": "workspace:^" + "@snickerdoodlelabs/shopping-data": "workspace:^" "@snickerdoodlelabs/signature-verification": "workspace:^" ethers: ^6.10.0 inversify: ^6.0.2 @@ -9095,6 +9139,7 @@ __metadata: react-chartjs-2: ^4.3.1 react-dom: ^17.0.2 react-ga: ^3.3.1 + react-google-charts: ^4.0.1 react-google-login: ^5.2.2 react-hotjar: ^5.2.0 react-intersection-observer: ^9.4.3 @@ -9366,6 +9411,22 @@ __metadata: languageName: unknown linkType: soft +"@snickerdoodlelabs/nlp@workspace:^, @snickerdoodlelabs/nlp@workspace:packages/nlp": + version: 0.0.0-use.local + resolution: "@snickerdoodlelabs/nlp@workspace:packages/nlp" + dependencies: + "@nlpjs/lang-en": ^4.26.1 + "@snickerdoodlelabs/common-utils": "workspace:^" + "@snickerdoodlelabs/objects": "workspace:^" + ethers: ^5.6.6 + html-to-text: ^9.0.5 + inversify: ^6.0.1 + neverthrow: ^5.1.0 + neverthrow-result-utils: ^2.0.2 + openai: ^4.0.1 + languageName: unknown + linkType: soft + "@snickerdoodlelabs/node-utils@workspace:^, @snickerdoodlelabs/node-utils@workspace:packages/node-utils": version: 0.0.0-use.local resolution: "@snickerdoodlelabs/node-utils@workspace:packages/node-utils" @@ -9405,6 +9466,7 @@ __metadata: "@snickerdoodlelabs/common-utils": "workspace:^" "@snickerdoodlelabs/node-utils": "workspace:^" "@snickerdoodlelabs/objects": "workspace:^" + "@snickerdoodlelabs/shopping-data": "workspace:^" "@snickerdoodlelabs/utils": "workspace:^" ethers: ^6.10.0 fake-indexeddb: ^4.0.0 @@ -9455,6 +9517,22 @@ __metadata: languageName: unknown linkType: soft +"@snickerdoodlelabs/shopping-data@workspace:^, @snickerdoodlelabs/shopping-data@workspace:packages/shopping-data": + version: 0.0.0-use.local + resolution: "@snickerdoodlelabs/shopping-data@workspace:packages/shopping-data" + dependencies: + "@snickerdoodlelabs/common-utils": "workspace:^" + "@snickerdoodlelabs/nlp": "workspace:^" + "@snickerdoodlelabs/objects": "workspace:^" + ethers: ^5.6.6 + html-to-text: ^9.0.5 + inversify: ^6.0.1 + neverthrow: ^5.1.0 + neverthrow-result-utils: ^2.0.2 + openai: ^4.0.1 + languageName: unknown + linkType: soft + "@snickerdoodlelabs/signature-verification@workspace:^, @snickerdoodlelabs/signature-verification@workspace:packages/signatureVerification": version: 0.0.0-use.local resolution: "@snickerdoodlelabs/signature-verification@workspace:packages/signatureVerification" @@ -11100,6 +11178,16 @@ __metadata: languageName: node linkType: hard +"@types/node-fetch@npm:^2.6.4": + version: 2.6.11 + resolution: "@types/node-fetch@npm:2.6.11" + dependencies: + "@types/node": "*" + form-data: ^4.0.0 + checksum: 180e4d44c432839bdf8a25251ef8c47d51e37355ddd78c64695225de8bc5dc2b50b7bb855956d471c026bb84bd7295688a0960085e7158cbbba803053492568b + languageName: node + linkType: hard + "@types/node@npm:*, @types/node@npm:>= 8, @types/node@npm:>=13.7.0": version: 20.8.5 resolution: "@types/node@npm:20.8.5" @@ -11158,6 +11246,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:^18.11.18": + version: 18.19.21 + resolution: "@types/node@npm:18.19.21" + dependencies: + undici-types: ~5.26.4 + checksum: a8c66d97426e8bbe341a6bdf42af9659ff5ad500c790b9046c981e234db42a1354a6ae86d5d1b75c86dfa018c742fb65c400a0e31d8f9f7505d4cfe5bd11ba72 + languageName: node + linkType: hard + "@types/node@npm:^8.0.0": version: 8.10.66 resolution: "@types/node@npm:8.10.66" @@ -17791,7 +17888,7 @@ __metadata: languageName: node linkType: hard -"charenc@npm:>= 0.0.1": +"charenc@npm:0.0.2, charenc@npm:>= 0.0.1": version: 0.0.2 resolution: "charenc@npm:0.0.2" checksum: 81dcadbe57e861d527faf6dd3855dc857395a1c4d6781f4847288ab23cffb7b3ee80d57c15bba7252ffe3e5e8019db767757ee7975663ad2ca0939bb8fcaf2e5 @@ -19252,7 +19349,7 @@ __metadata: languageName: node linkType: hard -"crypt@npm:>= 0.0.1": +"crypt@npm:0.0.2, crypt@npm:>= 0.0.1": version: 0.0.2 resolution: "crypt@npm:0.0.2" checksum: baf4c7bbe05df656ec230018af8cf7dbe8c14b36b98726939cef008d473f6fe7a4fad906cfea4062c93af516f1550a3f43ceb4d6615329612c6511378ed9fe34 @@ -20709,6 +20806,16 @@ __metadata: languageName: node linkType: hard +"digest-fetch@npm:^1.3.0": + version: 1.3.0 + resolution: "digest-fetch@npm:1.3.0" + dependencies: + base-64: ^0.1.0 + md5: ^2.3.0 + checksum: 8ebdb4b9ef02b1ac0da532d25c7d08388f2552813dfadabfe7c4630e944bb4a48093b997fc926440a10e1ccf4912f2ce9adcf2d6687b0518dab8480e08f22f9d + languageName: node + linkType: hard + "dijkstrajs@npm:^1.0.1": version: 1.0.3 resolution: "dijkstrajs@npm:1.0.3" @@ -20945,7 +21052,7 @@ __metadata: languageName: node linkType: hard -"domutils@npm:^3.1.0": +"domutils@npm:^3.0.1, domutils@npm:^3.1.0": version: 3.1.0 resolution: "domutils@npm:3.1.0" dependencies: @@ -23065,7 +23172,7 @@ __metadata: languageName: node linkType: hard -"ethers@npm:^5.0.0, ethers@npm:^5.0.1, ethers@npm:^5.0.2, ethers@npm:^5.5.1, ethers@npm:^5.5.2, ethers@npm:^5.6.1, ethers@npm:^5.7.1, ethers@npm:^5.7.2": +"ethers@npm:^5.0.0, ethers@npm:^5.0.1, ethers@npm:^5.0.2, ethers@npm:^5.5.1, ethers@npm:^5.5.2, ethers@npm:^5.6.1, ethers@npm:^5.6.6, ethers@npm:^5.7.1, ethers@npm:^5.7.2": version: 5.7.2 resolution: "ethers@npm:5.7.2" dependencies: @@ -24345,6 +24452,13 @@ __metadata: languageName: node linkType: hard +"form-data-encoder@npm:1.7.2": + version: 1.7.2 + resolution: "form-data-encoder@npm:1.7.2" + checksum: aeebd87a1cb009e13cbb5e4e4008e6202ed5f6551eb6d9582ba8a062005178907b90f4887899d3c993de879159b6c0c940af8196725b428b4248cec5af3acf5f + languageName: node + linkType: hard + "form-data@npm:^2.2.0": version: 2.5.1 resolution: "form-data@npm:2.5.1" @@ -24411,6 +24525,16 @@ __metadata: languageName: node linkType: hard +"formdata-node@npm:^4.3.2": + version: 4.4.1 + resolution: "formdata-node@npm:4.4.1" + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + checksum: d91d4f667cfed74827fc281594102c0dabddd03c9f8b426fc97123eedbf73f5060ee43205d89284d6854e2fc5827e030cd352ef68b93beda8decc2d72128c576 + languageName: node + linkType: hard + "formdata-polyfill@npm:^4.0.10": version: 4.0.10 resolution: "formdata-polyfill@npm:4.0.10" @@ -26367,6 +26491,19 @@ __metadata: languageName: node linkType: hard +"html-to-text@npm:^9.0.5": + version: 9.0.5 + resolution: "html-to-text@npm:9.0.5" + dependencies: + "@selderee/plugin-htmlparser2": ^0.11.0 + deepmerge: ^4.3.1 + dom-serializer: ^2.0.0 + htmlparser2: ^8.0.2 + selderee: ^0.11.0 + checksum: 205e0faa9b9aa281b369122acdffc5f348848e400f4037fde1fb12d68a6baa11644d2b64c3cc6821a79d3bc7316d89e85cc733d86f7f709858cb5c5b72faac65 + languageName: node + linkType: hard + "html-webpack-plugin@npm:^5.3.2, html-webpack-plugin@npm:^5.5.0": version: 5.5.3 resolution: "html-webpack-plugin@npm:5.5.3" @@ -26418,6 +26555,18 @@ __metadata: languageName: node linkType: hard +"htmlparser2@npm:^8.0.2": + version: 8.0.2 + resolution: "htmlparser2@npm:8.0.2" + dependencies: + domelementtype: ^2.3.0 + domhandler: ^5.0.3 + domutils: ^3.0.1 + entities: ^4.4.0 + checksum: 29167a0f9282f181da8a6d0311b76820c8a59bc9e3c87009e21968264c2987d2723d6fde5a964d4b7b6cba663fca96ffb373c06d8223a85f52a6089ced942700 + languageName: node + linkType: hard + "http-basic@npm:^2.5.1": version: 2.5.1 resolution: "http-basic@npm:2.5.1" @@ -27302,7 +27451,7 @@ __metadata: languageName: node linkType: hard -"inversify@npm:^6.0.2": +"inversify@npm:^6.0.1, inversify@npm:^6.0.2": version: 6.0.2 resolution: "inversify@npm:6.0.2" checksum: 9aafd41de12cb1dc1cebe6e0cc51fbda8a0f9cbcdc024874a7bc078f1317bb3e0f4a51cb28b2f3c69cc00f3e1f16e77f6e153c063b19da56f3592903dc21552c @@ -27583,7 +27732,7 @@ __metadata: languageName: node linkType: hard -"is-buffer@npm:^1.1.5": +"is-buffer@npm:^1.1.5, is-buffer@npm:~1.1.6": version: 1.1.6 resolution: "is-buffer@npm:1.1.6" checksum: 4a186d995d8bbf9153b4bd9ff9fd04ae75068fe695d29025d25e592d9488911eeece84eefbd8fa41b8ddcc0711058a71d4c466dcf6f1f6e1d83830052d8ca707 @@ -30264,6 +30413,15 @@ __metadata: languageName: node linkType: hard +"js-tiktoken@npm:^1.0.8": + version: 1.0.10 + resolution: "js-tiktoken@npm:1.0.10" + dependencies: + base64-js: ^1.5.1 + checksum: 94810d3b903d4ec6b1fbaf91b98870ee2c9f0b38de1d7924a0dfa17baaa083af84f31c37c3b1914bf37e6a012b099e922f548072ddf0c06d95511f9b72c8296d + languageName: node + linkType: hard + "js-tokens@npm:^3.0.0 || ^4.0.0, js-tokens@npm:^4.0.0": version: 4.0.0 resolution: "js-tokens@npm:4.0.0" @@ -31224,6 +31382,13 @@ __metadata: languageName: node linkType: hard +"leac@npm:^0.6.0": + version: 0.6.0 + resolution: "leac@npm:0.6.0" + checksum: a7a722cfc2ddfd6fb2620e5dee3ac8e9b0af4eb04325f3c8286a820de78becba3010a4d7026ff5189bb159eb7a851c3a1ac73e076eb0d54fcee0adaf695291ba + languageName: node + linkType: hard + "lerna@npm:^3.10.2": version: 3.22.1 resolution: "lerna@npm:3.22.1" @@ -32763,6 +32928,17 @@ __metadata: languageName: node linkType: hard +"md5@npm:^2.3.0": + version: 2.3.0 + resolution: "md5@npm:2.3.0" + dependencies: + charenc: 0.0.2 + crypt: 0.0.2 + is-buffer: ~1.1.6 + checksum: a63cacf4018dc9dee08c36e6f924a64ced735b37826116c905717c41cebeb41a522f7a526ba6ad578f9c80f02cb365033ccd67fe186ffbcc1a1faeb75daa9b6e + languageName: node + linkType: hard + "mdn-data@npm:2.0.14": version: 2.0.14 resolution: "mdn-data@npm:2.0.14" @@ -34662,7 +34838,7 @@ __metadata: languageName: node linkType: hard -"node-domexception@npm:^1.0.0": +"node-domexception@npm:1.0.0, node-domexception@npm:^1.0.0": version: 1.0.0 resolution: "node-domexception@npm:1.0.0" checksum: ee1d37dd2a4eb26a8a92cd6b64dfc29caec72bff5e1ed9aba80c294f57a31ba4895a60fd48347cf17dd6e766da0ae87d75657dfd1f384ebfa60462c2283f5c7f @@ -35938,6 +36114,25 @@ __metadata: languageName: node linkType: hard +"openai@npm:^4.0.1": + version: 4.28.4 + resolution: "openai@npm:4.28.4" + dependencies: + "@types/node": ^18.11.18 + "@types/node-fetch": ^2.6.4 + abort-controller: ^3.0.0 + agentkeepalive: ^4.2.1 + digest-fetch: ^1.3.0 + form-data-encoder: 1.7.2 + formdata-node: ^4.3.2 + node-fetch: ^2.6.7 + web-streams-polyfill: ^3.2.1 + bin: + openai: bin/cli + checksum: 1eeb392bfb495a8ddc44622950de25d7782a3cc00f748919ec3bfca0775bef6653e32b0edb11cee028f140c352e1a123acad4996d7cfdcf8eec078f20e5e63d0 + languageName: node + linkType: hard + "opener@npm:^1.5.1, opener@npm:^1.5.2": version: 1.5.2 resolution: "opener@npm:1.5.2" @@ -36557,6 +36752,16 @@ __metadata: languageName: node linkType: hard +"parseley@npm:^0.12.0": + version: 0.12.1 + resolution: "parseley@npm:0.12.1" + dependencies: + leac: ^0.6.0 + peberminta: ^0.9.0 + checksum: 147760bce6c4a4f8c62af021a84ced262f078f60a1119e6891eba69567a953e06295ad2c70e5e89892ad1d4af0126f0856742d657a19a29ebf58422cf3bfd4f3 + languageName: node + linkType: hard + "parseurl@npm:~1.3.2, parseurl@npm:~1.3.3": version: 1.3.3 resolution: "parseurl@npm:1.3.3" @@ -36813,6 +37018,13 @@ __metadata: languageName: node linkType: hard +"peberminta@npm:^0.9.0": + version: 0.9.0 + resolution: "peberminta@npm:0.9.0" + checksum: b983b68077269ca8a3327520a0a3f027fa930faa9fb3cb53bed1cb3847ebc0ed55db936d70b1745a756149911f5f450e898e87e25ab207f1b8b892bed48fb540 + languageName: node + linkType: hard + "pegjs@npm:^0.10.0": version: 0.10.0 resolution: "pegjs@npm:0.10.0" @@ -39088,6 +39300,16 @@ __metadata: languageName: node linkType: hard +"react-google-charts@npm:^4.0.1": + version: 4.0.1 + resolution: "react-google-charts@npm:4.0.1" + peerDependencies: + react: ">=16.3.0" + react-dom: ">=16.3.0" + checksum: 874a552b07cc67d6b830718dd5e71055c46fbd0c7a12bca14078c0744111d8fead833e36a16bd0b0ea5c26f6cff0eb84b4b6de62845c945b3c47c6cc75233f9d + languageName: node + linkType: hard + "react-google-login@npm:^5.2.2": version: 5.2.2 resolution: "react-google-login@npm:5.2.2" @@ -41896,6 +42118,15 @@ __metadata: languageName: node linkType: hard +"selderee@npm:^0.11.0": + version: 0.11.0 + resolution: "selderee@npm:0.11.0" + dependencies: + parseley: ^0.12.0 + checksum: af8a68c1f4cde858152943b6fc9f2b7164c8fb1a1c9f01b44350dffd1f79783930d77a0ae33548a036816d17c8130eeb9d15f1db65c9262ca368ad3a0d750f66 + languageName: node + linkType: hard + "select-hose@npm:^2.0.0": version: 2.0.0 resolution: "select-hose@npm:2.0.0" @@ -45984,6 +46215,13 @@ __metadata: languageName: node linkType: hard +"undici-types@npm:~5.26.4": + version: 5.26.5 + resolution: "undici-types@npm:5.26.5" + checksum: 3192ef6f3fd5df652f2dc1cd782b49d6ff14dc98e5dced492aa8a8c65425227da5da6aafe22523c67f035a272c599bb89cfe803c1db6311e44bed3042fc25487 + languageName: node + linkType: hard + "undici@npm:^5.14.0": version: 5.26.3 resolution: "undici@npm:5.26.3" @@ -47112,6 +47350,13 @@ __metadata: languageName: node linkType: hard +"web-streams-polyfill@npm:4.0.0-beta.3": + version: 4.0.0-beta.3 + resolution: "web-streams-polyfill@npm:4.0.0-beta.3" + checksum: dfec1fbf52b9140e4183a941e380487b6c3d5d3838dd1259be81506c1c9f2abfcf5aeb670aeeecfd9dff4271a6d8fef931b193c7bedfb42542a3b05ff36c0d16 + languageName: node + linkType: hard + "web-streams-polyfill@npm:^3.0.3": version: 3.2.1 resolution: "web-streams-polyfill@npm:3.2.1" @@ -47119,6 +47364,13 @@ __metadata: languageName: node linkType: hard +"web-streams-polyfill@npm:^3.2.1": + version: 3.3.3 + resolution: "web-streams-polyfill@npm:3.3.3" + checksum: 21ab5ea08a730a2ef8023736afe16713b4f2023ec1c7085c16c8e293ee17ed085dff63a0ad8722da30c99c4ccbd4ccd1b2e79c861829f7ef2963d7de7004c2cb + languageName: node + linkType: hard + "web3-bzz@npm:1.10.0": version: 1.10.0 resolution: "web3-bzz@npm:1.10.0"