update to handle cf workers

Author: Granitosaurus

.github/workflows/publish.yaml

@@ -1,3 +1,4 @@
+
 name: Publish to NPM
 
 on:
@@ -12,17 +13,25 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - uses: volta-cli/action@v1
-    - run: npm ci --no-audit
-    - run: npm run lint --if-present
-    - run: npm test
-    - run: npm run build --if-present
-      env:
-          CI: true
+
+    - name: Install Deno
+      run: |
+        curl -fsSL https://deno.land/x/install/install.sh | sh
+        echo "DENO_INSTALL=/home/runner/.deno" >> $GITHUB_ENV
+        echo "$DENO_INSTALL/bin" >> $GITHUB_PATH
+
+    - name: Run Deno build script
+      run: deno run -A build.ts
+
+    - name: Navigate to npm directory
+      run: cd ./npm
+
     - name: Setup .npmrc file to publish to npm
       run: |
         echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > .npmrc
       env:
-        NPM_TOKEN: ${{secrets.NPM_AUTOMATION_TOKEN}}
+        NPM_TOKEN: ${{ secrets.NPM_AUTOMATION_TOKEN }}
+
     - name: Publish to NPM
-      run: npm publish
+      run: npm publish --dry-run
+      working-directory: ./npm

README.md

@@ -1,8 +1,10 @@
 # Scrapfly SDK
 
-`npm install scrapfly-sdk`
+`npm install scrapfly-sdk`  
+`deno add jsr:@scrapfly/scrapfly-sdk`  
+`bun jsr add @scrapfly/scrapfly-sdk`  
 
-Typescript/NodeJS SDK for [Scrapfly.io](https://scrapfly.io/) web scraping API which allows to:
+Typescript/Javascript SDK for [Scrapfly.io](https://scrapfly.io/) web scraping API which allows to:
 
 -   Scrape the web without being blocked.
 -   Use headless browsers to access Javascript-powered page data.
@@ -11,6 +13,10 @@ Typescript/NodeJS SDK for [Scrapfly.io](https://scrapfly.io/) web scraping API w
 
 For web scraping guides see [our blog](https://scrapfly.io/blog/) and [#scrapeguide](https://scrapfly.io/blog/tag/scrapeguide/) tag for how to scrape specific targets.
 
+The SDK is distributed through:
+- [npmjs.com/package/scrapfly-sdk](https://www.npmjs.com/package/scrapfly-sdk)
+- [jsr.io/@scrapfly/scrapfly-sdk](https://jsr.io/@scrapfly/scrapfly-sdk)
+
 ## Quick Intro
 
 1. Register a [Scrapfly account for free](https://scrapfly.io/register)

__tests__/config/extraction.test.ts

@@ -114,7 +114,8 @@ Deno.test('url param generation: sets extraction_model', async () => {
     });
 });
 
-Deno.test({
+// XXX: could add auto compression but support is difficult
+/* Deno.test({
     name: 'url param generation: compresses body', 
     async fn() {
         const config = new ExtractionConfig({
@@ -139,7 +140,7 @@ Deno.test({
     sanitizeResources: false,
     sanitizeOps: false,
 });
-
+ */
 Deno.test('url param generation: fails to missing compression state with declared compression format', async () => {
     const config = new ExtractionConfig({
         body: input_html,

__tests__/utils.test.ts

@@ -81,7 +81,7 @@ Deno.test('fetchRetry: fails after max retries', async () => {
 
   await assertRejects(
     async () => {
-      await fetchRetry(request, {}, 3, 1000);
+      await fetchRetry(request, 3);
     },
     Error,
     'Fetch failed with status: 500'
@@ -92,6 +92,8 @@ Deno.test('fetchRetry: fails after max retries', async () => {
   fetchStub.restore();
 });
 
+// XXX: should we support built-in timeout?
+/* 
 Deno.test('fetchRetry: fails due to timeout', async () => {
 
   const request = new Request('https://httpbin.dev/delay/3');
@@ -103,4 +105,4 @@ Deno.test('fetchRetry: fails due to timeout', async () => {
     Error,
     'The signal has been aborted'
   );
-});
+}); */

build.ts

@@ -1,31 +1,4 @@
 import { build, emptyDir } from "@deno/dnt";
-import { rollup } from "npm:rollup";
-import resolve from "npm:@rollup/plugin-node-resolve";
-import commonjs from "npm:@rollup/plugin-commonjs";
-import { terser } from "npm:rollup-plugin-terser";
-
-async function bundleForBrowser() {
-  console.log("Bundling for browser...");
-  const bundle = await rollup({
-    input: "npm/esm/main.js",
-    plugins: [
-      resolve({ browser: true, preferBuiltins: false }),
-      commonjs(),
-      terser(),
-    ],
-    logLevel: 'debug',
-  });
-
-  console.log("Writing to npm/dist/bundle.js");
-  await bundle.write({
-    file: "npm/dist/bundle.js",
-    format: "esm",
-    sourcemap: true,
-  });
-  console.log("Closing Rollup");
-  await bundle.close();
-}
-
 await emptyDir("./npm");
 
 const { version, description } = JSON.parse(Deno.readTextFileSync("deno.json"));
@@ -53,13 +26,12 @@ await build({
       url: "https://github.com/scrapfly/typescript-scrapfly/issues",
     },
     homepage: "https://scrapfly.io/",
-    main: "./esm/src/main.js", // Point to the ESM output
-    types: "./esm/src/main.d.ts", // Point to the TypeScript declarations
+    main: "./esm/main.js", 
+    types: "./esm/main.d.ts",
   },
   postBuild: async () => {
     Deno.copyFileSync("LICENSE", "npm/LICENSE");
     Deno.copyFileSync("README.md", "npm/README.md");
-    await bundleForBrowser();
   },
 });
 

src/deps.ts

@@ -1,4 +1,4 @@
-import * as cheerio from "npm:cheerio@1.0.0-rc.12";
+import * as cheerio from 'https://cdn.skypack.dev/cheerio@1.0.0-rc.12?dts';
 import * as path from 'jsr:@std/path@1.0.1';
 
-export { cheerio, path};
+export { cheerio, path };

src/extractionconfig.ts

@@ -1,7 +1,5 @@
 import * as errors from './errors.ts';
 import { urlsafe_b64encode } from './utils.ts';
-import { gzipSync } from 'node:zlib';
-import { Buffer } from 'node:buffer';
 
 export enum CompressionFormat {
   /**
@@ -114,17 +112,18 @@ export class ExtractionConfig {
         );
       }
       if (this.is_document_compressed === false) {
-        if (this.document_compression_format === CompressionFormat.GZIP) {
-          // XXX: This breaks cloudflare workers as they don't support node:zlib
-          const compressed = gzipSync(Buffer.from(this.body as string, 'utf-8'));
-          this.body = new Uint8Array(compressed);
-        } else {
-          throw new errors.ExtractionConfigError(
-            `Auto compression for ${this.document_compression_format} format isn't available. ` +
-              `You can manually compress to ${this.document_compression_format}` +
-              `or choose the gzip format for auto compression`,
-          );
-        }
+        // if (this.document_compression_format === CompressionFormat.GZIP) {
+        // XXX: This breaks cloudflare workers for some reason
+        // const compressed = gzip(new TextEncoder().encode(this.body as string));
+        // this.body = new Uint8Array(compressed);
+        // throw new Error("automatic gzip is not supported yet, pass gzipped ");
+        // } else {
+        throw new errors.ExtractionConfigError(
+          `Auto compression for ${this.document_compression_format} format isn't available. ` +
+            `You can manually compress to ${this.document_compression_format}` +
+            `or choose the gzip format for auto compression`,
+        );
+        // }
       }
     }
 

src/result.ts

@@ -312,7 +312,7 @@ export class ExtractionResult {
   content_type: string;
   data_quality?: string;
 
-  constructor(response: { data: string; content_type: string, data_quality?: string }) {
+  constructor(response: { data: string; content_type: string; data_quality?: string }) {
     this.data = response.data;
     this.content_type = response.content_type;
     this.data_quality = response.data_quality;

src/utils.ts

@@ -10,21 +10,15 @@ export function urlsafe_b64encode(data: string): string {
 
 export async function fetchRetry(
   config: Request,
-  init: RequestInit = {},
   retries: number = 3,
   retryDelay: number = 1000,
-  timeout: number = 160000, // Timeout in milliseconds
 ): Promise<Response> {
   let lastError: any = null;
 
   for (let attempt = 1; attempt <= retries; attempt++) {
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), timeout);
-
     try {
       // XXX: this breaks cloudflare workers as they don't support init options
-      const response = await fetch(config, { ...init, signal: controller.signal });
-      clearTimeout(timeoutId);
+      const response = await fetch(config);
       // retry 5xx status codes
       if (response.status >= 500 && response.status < 600) {
         lastError = new Error(`Fetch failed with status: ${response.status}`);
@@ -35,7 +29,6 @@ export async function fetchRetry(
         return response;
       }
     } catch (error) {
-      clearTimeout(timeoutId);
       lastError = error;
 
       if (attempt === retries || error.name === 'AbortError') {