fix bug where Content-Type header would be overriden to text/plain if set manually with request body

Author: Granitosaurus

__tests__/config/scrape.test.ts

@@ -65,6 +65,17 @@ Deno.test('scrapeconfig POST/PUT/PATCH body defaults as content-type text/plain'
     assertEquals(config.body, 'foo+bar');
 });
 
+Deno.test('scrapeconfig POST/PUT/PATCH body does not override content-type when set', async () => {
+    const config = new ScrapeConfig({
+        url: 'http://httpbin.dev/get',
+        method: 'POST',
+        body: 'foo+bar',
+        headers: { 'content-type': 'application/json' },
+    });
+    assertEquals((config.headers || {})['content-type'], 'application/json');
+    assertEquals(config.body, 'foo+bar');
+});
+
 Deno.test('scrapeconfig POST/PUT/PATCH data encodes when formdata content-type is set', async () => {
     const config = new ScrapeConfig({
         url: 'http://httpbin.dev/get',

deno.json

@@ -4,7 +4,7 @@
   },
   "name": "@scrapfly/scrapfly-sdk",
   "exports": "./src/main.ts",
-  "version": "0.6.7",
+  "version": "0.6.8",
   "description": "SDK for Scrapfly.io API for web scraping, screenshotting and data extraction",
   "tasks": {
     "start": "deno run --allow-net --allow-read src/main.ts",

src/client.ts

@@ -122,12 +122,12 @@ export class ScrapflyClient {
       content = await response.text();
       result.format = 'text';
     }
-    
+
     if (format === 'blob') {
       content = new Uint8Array(await response.arrayBuffer());
       result.format = 'binary';
     }
-    
+
     result.content = content;
     return result;
   }

src/extractionconfig.ts

@@ -45,9 +45,14 @@ export class ExtractionConfig {
 
   constructor(options: ExtractionConfigOptions) {
     this.validateOptions(options);
-    if (options.document_compression_format && !Object.values(CompressionFormat).includes(options.document_compression_format as CompressionFormat)) {
-      throw new errors.ExtractionConfigError(`Invalid CompressionFormat param value: ${options.document_compression_format}`);
-    }    
+    if (
+      options.document_compression_format &&
+      !Object.values(CompressionFormat).includes(options.document_compression_format as CompressionFormat)
+    ) {
+      throw new errors.ExtractionConfigError(
+        `Invalid CompressionFormat param value: ${options.document_compression_format}`,
+      );
+    }
     this.body = options.body;
     this.content_type = options.content_type;
     this.url = options.url ?? this.url;

src/scrapeconfig.ts

@@ -38,7 +38,7 @@ export enum Format {
 export enum FormatOption {
   NO_LINKS = 'no_links',
   NO_IMAGES = 'no_images',
-  ONLY_CONTENT = 'only_content'
+  ONLY_CONTENT = 'only_content',
 }
 
 type ScrapeConfigOptions = {
@@ -70,7 +70,8 @@ type ScrapeConfigOptions = {
   rendering_wait?: number;
   wait_for_selector?: string;
   screenshots?: Rec<any>;
-  screenshot_flags?: ('load_images' | 'dark_mode' | 'block_banners' | 'print_media_format' | 'high_quality' | ScreenshotFlags)[];
+  screenshot_flags?:
+    ('load_images' | 'dark_mode' | 'block_banners' | 'print_media_format' | 'high_quality' | ScreenshotFlags)[];
   session_sticky_proxy?: boolean;
   webhook?: string;
   timeout?: number;
@@ -113,7 +114,8 @@ export class ScrapeConfig {
   wait_for_selector?: string;
   session_sticky_proxy = false;
   screenshots?: Rec<any>;
-  screenshot_flags?: ('load_images' | 'dark_mode' | 'block_banners' | 'print_media_format' | 'high_quality' | ScreenshotFlags)[];
+  screenshot_flags?:
+    ('load_images' | 'dark_mode' | 'block_banners' | 'print_media_format' | 'high_quality' | ScreenshotFlags)[];
   webhook?: string;
   timeout?: number; // in milliseconds
   js_scenario?: Rec<any>;
@@ -202,7 +204,7 @@ export class ScrapeConfig {
             );
           }
         }
-      } else if (this.body && !this.data) {
+      } else if (this.body && !this.data && !this.headers['content-type']) {
         this.headers['content-type'] = 'text/plain';
       }
     }

src/screenshotconfig.ts

@@ -34,7 +34,7 @@ export enum Format {
 
 type ScreenshotConfigOptions = {
   url: string;
-  format?: 'jpg'| 'png' | 'webp'| 'gif' | Format;
+  format?: 'jpg' | 'png' | 'webp' | 'gif' | Format;
   capture?: string;
   resolution?: string;
   country?: string;
@@ -52,7 +52,7 @@ type ScreenshotConfigOptions = {
 
 export class ScreenshotConfig {
   url: string;
-  format?: 'jpg'| 'png' | 'webp'| 'gif' | Format;
+  format?: 'jpg' | 'png' | 'webp' | 'gif' | Format;
   capture?: string;
   resolution?: string;
   country?: string = undefined;
@@ -79,7 +79,7 @@ export class ScreenshotConfig {
           throw new ScreenshotConfigError(`Invalid Options param value: ${opt}`);
         }
       });
-    }    
+    }
     this.url = options.url;
     this.format = options.format ?? this.format;
     this.capture = options.capture ?? this.capture;