add webhook param support with the screenshot and extraction apis

Author: mazen-r

__tests__/client.test.ts

@@ -510,7 +510,7 @@ it('succeeds', async () => {
 
         const result = await client.screenshot(new ScreenshotConfig({ url: url }));
         expect(result).toBeDefined();
-        expect(result.metadata.format).toBe('png');
+        expect(result.metadata.extension_name).toBe('png');
         expect(result.metadata.upstream_url).toEqual(url);
         expect(result.metadata.upstream_status_code).toBe(200);
         expect(spy).toHaveBeenCalledTimes(1);

examples/extraction/extraction_webhook.js

@@ -0,0 +1,43 @@
+/*
+This example shows how to use the webhook feature with Scrapfly's extraction API
+*/
+import { ScrapflyClient, ScrapeConfig, ExtractionConfig } from 'scrapfly-sdk';
+
+const key = 'YOUR SCRAPFLY KEY';
+const client = new ScrapflyClient({ key });
+
+// First, scrape the web page to retrieve its HTML
+const scrapeResult = await client.scrape(
+    new ScrapeConfig({
+        url: 'https://web-scraping.dev/products',
+        render_js: true,
+    }),
+);
+
+// raw HTML content
+const html = scrapeResult.result.content;
+
+// Second, pass the HTML and an extraction prompt
+// In this example, we'll ask a question about the data
+const extractionResult = await client.extract(
+    new ExtractionConfig({
+        body: html, // pass the HTML content
+        content_type: 'text/html', // data content type
+        charset: 'utf-8', // passed content charset, use `auto` if you aren't sure
+        extraction_prompt: 'what is the flavor of the dark energy potion?', // LLM extraction prompt
+        webhook: 'my-webhook'
+    }),
+);
+
+// raw result
+console.log(extractionResult.result)
+`
+{
+    job_uuid: '7a3aa96d-fb0e-4c45-9b01-7c42f295dcac',
+    success: true,
+    webhook_name: 'my-webhook',
+    webhook_queue_limit: 10000,
+    webhook_queued_element: 7,
+    webhook_uuid: 'd7131802-1eba-4cc4-a6fd-5da6c8cf1f35'    
+}
+`

examples/screenshot/screenshot-webhook.js

@@ -0,0 +1,27 @@
+/*
+This example shows how to use the webhook feature with Scrapfly's screenshot API
+*/
+import { ScrapflyClient, ScreenshotConfig } from 'scrapfly-sdk';
+
+const key = 'YOUR SCRAPFLY KEY';
+const client = new ScrapflyClient({ key });
+
+const screenshotResult = await client.screenshot(
+    new ScreenshotConfig({
+        url: 'https://web-scraping.dev/products',
+        webhook: 'my-webhook'
+    }),
+);
+
+// raw result
+console.log(screenshotResult.result)
+`
+{
+    job_uuid: 'a0e6f3e8-be35-438a-942a-be77aa545d30',
+    success: true,
+    webhook_name: 'my-webhook',
+    webhook_queue_limit: 10000,
+    webhook_queued_element: 7,
+    webhook_uuid: 'cdf37252-fea7-4267-a568-aa0e5964ee21'
+}
+`

examples/screenshot/screenshot-with-screenshot-api.js

@@ -24,7 +24,7 @@ const result = await client.screenshot(
     }),
 );
 
-// screenshot metadata (format, upstream_status_code, upstream_url)
+// screenshot metadata (extension_name, upstream_status_code, upstream_url)
 const metadata = result.metadata;
 
 // screenshot binary

src/client.ts

@@ -272,14 +272,14 @@ export class ScrapflyClient {
             throw new Error('Screenshot binary does not exist');
         }
 
-        const format = result.metadata.format;
+        const extension_name = result.metadata.extension_name;
         let file_path;
 
         if (savePath) {
             fs.mkdirSync(savePath, { recursive: true });
-            file_path = path.join(savePath, `${name}.${format}`);
+            file_path = path.join(savePath, `${name}.${extension_name}`);
         } else {
-            file_path = `${name}.${format}`;
+            file_path = `${name}.${extension_name}`;
         }
 
         const content = Buffer.from(result.image);
@@ -292,13 +292,11 @@ export class ScrapflyClient {
     async handleScreenshotResponse(response: Response): Promise<ScreenshotResult> {
         if (response.headers.get('content-encoding') != 'gzip') {
             const data = (await response.json()) as any;
-            if (response.headers.get('content-encoding') !== 'gzip') {
-                if (data.http_code == 401 || response.status == 401) {
-                    throw new errors.BadApiKeyError(JSON.stringify(data));
-                }
-                if ('error_id' in data) {
-                    throw new errors.ScreenshotApiError(JSON.stringify(data));
-                }
+            if (data.http_code == 401 || response.status == 401) {
+                throw new errors.BadApiKeyError(JSON.stringify(data));
+            }
+            if ('error_id' in data) {
+                throw new errors.ScreenshotApiError(JSON.stringify(data));
             }
         }
         if (!response.ok) {
@@ -350,7 +348,7 @@ export class ScrapflyClient {
         }
         if (!response.ok) {
             throw new errors.ApiHttpClientError(JSON.stringify(await response.json()));
-        }        
+        }
         const result = new ExtractionResult(data);
         return result;
     }
@@ -381,7 +379,7 @@ export class ScrapflyClient {
             log.error('error', e);
             throw e;
         }
-        const result = await this.handleExtractionResponse(response)
+        const result = await this.handleExtractionResponse(response);
         return result;
     }
 }

src/extractionconfig.ts

@@ -10,6 +10,7 @@ export class ExtractionConfig {
     epehemeral_template?: object; // epehemeraly declared json template
     extraction_prompt?: string = null;
     extraction_model?: string = null;
+    webhook?: string = null;
 
     constructor(options: {
         body: string;
@@ -20,6 +21,7 @@ export class ExtractionConfig {
         epehemeral_template?: object; // epehemeraly declared json template
         extraction_prompt?: string;
         extraction_model?: string;
+        webhook?: string;
     }) {
         this.body = options.body;
         this.content_type = options.content_type;
@@ -29,6 +31,7 @@ export class ExtractionConfig {
         this.epehemeral_template = options.epehemeral_template;
         this.extraction_prompt = options.extraction_prompt;
         this.extraction_model = options.extraction_model;
+        this.webhook = options.webhook;
     }
 
     toApiParams(options: { key: string }): Record<string, any> {
@@ -65,6 +68,11 @@ export class ExtractionConfig {
         if (this.extraction_model) {
             params.extraction_model = this.extraction_model;
         }
+
+        if (this.webhook) {
+            params.webhook_name = this.webhook;
+        }
+        
         return params;
     }
 }

src/result.ts

@@ -267,41 +267,51 @@ export class AccountData {
 }
 
 export type ScreenshotMetadata = {
-    format: string;
+    extension_name: string;
     upstream_status_code: number;
     upstream_url: string;
 };
 
 export class ScreenshotResult {
     image: ArrayBuffer;
     metadata: ScreenshotMetadata;
+    result: object;
 
     constructor(response: Response, data: ArrayBuffer) {
         this.image = data;
         this.metadata = this.defineMetadata(response);
+        this.result = this.returnRaw(response, data); // raw result
     }
 
     private defineMetadata(response: Response): ScreenshotMetadata {
         const contentType = response.headers.get('content-type');
-        let format;
+        let extension_name;
         if (contentType) {
-            format = contentType.split('/')[1].split(';')[0];
-            format = format === 'jpeg' ? 'jpg' : format;
+            extension_name = contentType.split('/')[1].split(';')[0];
         }
         return {
-            format: format,
+            extension_name: extension_name,
             upstream_status_code: parseInt(response.headers.get('X-Scrapfly-Upstream-Http-Code'), 10),
             upstream_url: response.headers.get('X-Scrapfly-Upstream-Url'),
         };
     }
+
+    private returnRaw(response: Response, data: ArrayBuffer): Promise<any> {
+        if (response.headers.get('content-encoding') == 'gzip') {
+            return null
+        }
+        return JSON.parse(new TextDecoder().decode(data));
+    }
 }
 
 export class ExtractionResult {
     data: string;
     content_type: string;
+    result: object;
 
     constructor(response: { data: string; content_type: string }) {
         this.data = response.data;
         this.content_type = response.content_type;
+        this.result = response; // raw data
     }
 }

src/screenshotconfig.ts

@@ -47,6 +47,7 @@ export class ScreenshotConfig {
     cache?: boolean = null;
     cache_ttl?: boolean = null;
     cache_clear?: boolean = null;
+    webhook?: string = null;
 
     constructor(options: {
         url: string;
@@ -63,14 +64,15 @@ export class ScreenshotConfig {
         cache?: boolean;
         cache_ttl?: boolean;
         cache_clear?: boolean;
+        webhook?: string;
     }) {
         if (options.format && !Object.values(Format).includes(options.format)) {
             throw new ScreenshotConfigError(`Invalid format param value: ${options.format}`);
         }
         this.format = options.format ?? this.format;
         // Validate options against the enum
         if (options.options) {
-            options.options.forEach(opt => {
+            options.options.forEach((opt) => {
                 if (!Object.values(Options).includes(opt)) {
                     throw new ScreenshotConfigError(`Invalid options param value: ${opt}`);
                 }
@@ -90,6 +92,7 @@ export class ScreenshotConfig {
         this.cache = options.cache ?? this.cache;
         this.cache_ttl = options.cache_ttl ?? this.cache_ttl;
         this.cache_clear = options.cache_clear ?? this.cache_clear;
+        this.webhook = options.webhook;
     }
 
     toApiParams(options: { key: string }): Record<string, any> {
@@ -155,6 +158,10 @@ export class ScreenshotConfig {
             }
         }
 
+        if (this.webhook) {
+            params.webhook_name = this.webhook;
+        }
+        
         return params;
     }
 }