resolve conflict

Author: Granitosaurus

.eslintrc.json

@@ -4,7 +4,7 @@
         "es6": true,
         "node": true
     },
-    "ignorePatterns": ["**/*.d.ts"],
+    "ignorePatterns": ["**/*.d.ts", "examples/**/*"],
     "parser": "@typescript-eslint/parser",
     "parserOptions": {
         "project": "tsconfig.json",

.github/workflows/publish.yaml

@@ -0,0 +1,28 @@
+name: Publish to NPM
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - 'v*' # This will run the workflow when you push a tag with a version format, like v1.0.0
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - uses: volta-cli/action@v1
+    - run: npm ci --no-audit
+    - run: npm run lint --if-present
+    - run: npm test
+    - run: npm run build --if-present
+      env:
+          CI: true
+    - name: Setup .npmrc file to publish to npm
+      run: |
+        echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > .npmrc
+      env:
+        NPM_TOKEN: ${{secrets.NPM_AUTOMATION_TOKEN}}
+    - name: Publish to NPM
+      run: npm publish

.gitleaks.toml

@@ -0,0 +1,5 @@
+[allowlist]
+description = "global allow lists"
+paths = [
+    '''.devcontainer''',
+]

.pre-commit-config.yaml

@@ -0,0 +1,5 @@
+repos:
+  - repo: https://github.com/zricethezav/gitleaks
+    rev: v8.17.0
+    hooks:
+      - id: gitleaks

.prettierignore

@@ -0,0 +1,3 @@
+/.history/
+/examples/.history/
+/.github/

LICENSE

@@ -0,0 +1,26 @@
+BSD 2-Clause License
+--------------------
+
+Copyright (c) 2013-2016, Scrapfly <https://scrapfly.io> and
+individual contributors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -2,35 +2,71 @@
 
 `npm install scrapfly-sdk`
 
-Quick use:
+Typescript/NodeJS SDK for [Scrapfly.io](https://scrapfly.io/) web scraping API which allows to:
+
+-   Scrape the web without being blocked.
+-   Use headless browsers to access Javascript-powered page data.
+-   Scale up web scraping.
+-   ... and [much more](https://scrapfly.io/docs/scrape-api/getting-started)!
+
+For web scraping guides see [our blog](https://scrapfly.io/blog/) and [#scrapeguide](https://scrapfly.io/blog/tag/scrapeguide/) tag for how to scrape specific targets.
+
+## Quick Intro
+
+1. Register a [Scrapfly account for free](https://scrapfly.io/register)
+2. Get your API Key on [scrapfly.io/dashboard](https://scrapfly.io/dashboard)
+3. Start scraping: 🚀
 
 ```javascript
-import { ScrapflyClient, ScrapeConfig } from "scrapfly-sdk";
-
-const client = new ScrapflyClient({key: "YOUR SCRAPFLY KEY"});
-const result = await client.scrape(new ScrapeConfig({
-    url: "https://httpbin.dev/html",
-    // optional:
-    aps: true,  // enable anti-scraping protection bypass
-    render_js: true, // enable headless browsers for javascript rendering
-    country: "us",  // use a US proxy
-    method: "GET",  // use GET, POST or other type of requests
-    data: {},  // what data to send if POST is used
-    ...
-}))
-console.log(result.result.content)  // html content
+import { ScrapflyClient, ScrapeConfig } from 'scrapfly-sdk';
+
+const key = 'YOUR SCRAPFLY KEY';
+const client = new ScrapflyClient({ key });
+const apiResponse = await client.scrape(
+    new ScrapeConfig({
+        url: 'https://web-scraping.dev/product/1',
+        // optional parameters:
+        // enable javascript rendering
+        render_js: true,
+        // set proxy country
+        country: 'us',
+        // enable anti-scraping protection bypass
+        asp: true,
+        // set residential proxies
+        proxy_pool: 'public_residential_pool',
+        // etc.
+    }),
+);
+console.log(apiResponse.result.content); // html content
+// Parse HTML directly with SDK (through cheerio)
+console.log(apiResponse.result.selector('h3').text());
 ```
 
-See [/examples](./examples/) for more.
+For more see [/examples](/examples/) directory.  
+For more on Scrapfly API see our [getting started documentation](https://scrapfly.io/docs/scrape-api/getting-started)
+For Python see [Scrapfly Python SDK](https://github.com/scrapfly/python-scrapfly)
+
+## Debugging
 
-## Get Your API Key
+To enable debug logs set Scrapfly's log level to `"DEBUG"`:
 
-You can create a free account on [Scrapfly](https://scrapfly.io/register) to get your API Key.
+```javascript
+import { log } from 'scrapfly-sdk';
+
+log.setLevel('DEBUG');
+```
 
--   [Usage](https://scrapfly.io/docs/sdk/python)
--   [Python API](https://scrapfly.github.io/python-scrapfly/scrapfly)
--   [Open API 3 Spec](https://scrapfly.io/docs/openapi#get-/scrape)
--   [Scrapy Integration](https://scrapfly.io/docs/sdk/scrapy)
+Additionally, set `debug=true` in `ScrapeConfig` to access debug information in [Scrapfly web dashboard](https://scrapfly.io/dashboard):
+
+```typescript
+import { ScrapflyClient } from 'scrapfly-sdk';
+
+new ScrapeConfig({
+    url: 'https://web-scraping.dev/product/1',
+    debug: true,
+    // ^ enable debug information - this will show extra details on web dashboard
+});
+```
 
 ## Development
 

__tests__/client.test.ts

@@ -1,7 +1,9 @@
 import axios from 'axios';
+import { AxiosRequestConfig } from 'axios';
 import { ScrapflyClient } from '../src/client.js';
 import * as errors from '../src/errors.js';
 import { ScrapeConfig } from '../src/scrapeconfig.js';
+import { describe, it, expect, jest, beforeEach } from '@jest/globals';
 
 jest.mock('axios');
 
@@ -37,7 +39,7 @@ describe('concurrent scrape', () => {
     // mock axios to return /account data and 2 types of results:
     // - success for /success endpoints
     // - ASP failure for /failure endpoints
-    mockedAxios.request.mockImplementation(async (config) => {
+    mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
         if (config.url.includes('/account')) {
             return {
                 status: 200,
@@ -123,7 +125,7 @@ describe('scrape', () => {
 
     it('GET success', async () => {
         const url = 'https://httpbin.dev/json';
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             // Ensure the URL matches the pattern
             expect(config.url).toMatch(client.HOST + '/scrape');
             expect(config.method).toEqual('GET');
@@ -147,7 +149,7 @@ describe('scrape', () => {
 
     it('GET complex urls', async () => {
         const url = 'https://httpbin.dev/anything/?website=https://httpbin.dev/anything';
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             // Ensure the URL matches the pattern
             expect(config.url).toMatch(client.HOST + '/scrape');
             expect(config.method).toEqual('GET');
@@ -169,11 +171,9 @@ describe('scrape', () => {
         expect(mockedAxios.request).toHaveBeenCalledTimes(1);
     });
 
-
-
     it('POST success', async () => {
         const url = 'https://httpbin.dev/json';
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             // Ensure the URL matches the pattern
             expect(config.url).toMatch(client.HOST + '/scrape');
             expect(config.method).toEqual('POST');
@@ -235,7 +235,7 @@ describe('client errors', () => {
 
     it('raises ApiHttpServerError on 500 and success', async () => {
         const url = 'https://httpbin.dev/json';
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status_code: 500,
@@ -249,7 +249,7 @@ describe('client errors', () => {
 
     it('raises BadApiKeyError on 401', async () => {
         const url = 'https://httpbin.dev/json';
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status_code: 401,
@@ -262,7 +262,7 @@ describe('client errors', () => {
     });
     it('raises TooManyRequests on 429 and success', async () => {
         const url = 'https://httpbin.dev/json';
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status_code: 429,
@@ -273,7 +273,7 @@ describe('client errors', () => {
         await expect(client.scrape(new ScrapeConfig({ url }))).rejects.toThrow(errors.TooManyRequests);
     });
     it('raises ScrapflyScrapeError on ::SCRAPE:: resource and success', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::SCRAPE::BAD_PROTOCOL',
@@ -286,7 +286,7 @@ describe('client errors', () => {
     });
 
     it('raises ScrapflyWebhookError on ::WEBHOOK:: resource and success', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::WEBHOOK::DISABLED ',
@@ -298,7 +298,7 @@ describe('client errors', () => {
         );
     });
     it('raises ScrapflyProxyError on ERR::PROXY::POOL_NOT_FOUND  resource and success', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::PROXY::POOL_NOT_FOUND ',
@@ -311,7 +311,7 @@ describe('client errors', () => {
     });
 
     it('raises ScrapflyScheduleError on ERR::SCHEDULE::DISABLED resource and success', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::SCHEDULE::DISABLED',
@@ -324,7 +324,7 @@ describe('client errors', () => {
     });
 
     it('raises ScrapflyAspError on ERR::ASP::SHIELD_ERROR resource and success', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::ASP::SHIELD_ERROR',
@@ -337,7 +337,7 @@ describe('client errors', () => {
     });
 
     it('raises ScrapflySessionError on ERR::SESSION::CONCURRENT_ACCESS resource and success', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::SESSION::CONCURRENT_ACCESS',
@@ -350,7 +350,7 @@ describe('client errors', () => {
     });
 
     it('raises ApiHttpClientError on success and unknown status', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 status: 'ERR::NEW',
@@ -362,7 +362,7 @@ describe('client errors', () => {
         );
     });
     it('raises UpstreamHttpServerError on failure, ERR::SCRAPE::BAD_UPSTREAM_RESPONSE and >=500', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 success: false,
@@ -375,7 +375,7 @@ describe('client errors', () => {
         );
     });
     it('raises UpstreamHttpClientError on failure, ERR::SCRAPE::BAD_UPSTREAM_RESPONSE and 4xx status', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 success: false,
@@ -398,7 +398,7 @@ describe('client errors', () => {
             SESSION: errors.ScrapflySessionError,
         };
         for (const [resource, err] of Object.entries(resourceErrMap)) {
-            mockedAxios.request.mockImplementation(async (config) => {
+            mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
                 return resultFactory({
                     url: config.url,
                     success: false,
@@ -410,7 +410,7 @@ describe('client errors', () => {
     });
 
     it('raises ScrapflyError on unhandled failure', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 success: false,
@@ -423,7 +423,7 @@ describe('client errors', () => {
         );
     });
     it('raises  on unhandled failure', async () => {
-        mockedAxios.request.mockImplementation(async (config) => {
+        mockedAxios.request.mockImplementation(async (config: AxiosRequestConfig): Promise<any> => {
             return resultFactory({
                 url: config.url,
                 success: false,