From 05926a90ea9d3e925ab7defb4149cace320802b6 Mon Sep 17 00:00:00 2001
From: garyking <garyking@gmail.com>
Date: Wed, 6 Nov 2019 11:51:35 -0500
Subject: [PATCH] website-scraper: Make `Source.attr` optional. Add more
 options. (#40137)

* Make `attr` optional.

As per the defaults:

https://github.com/website-scraper/node-website-scraper/blob/050cdd011231033477264df9f432a7925927575f/lib/config/defaults.js

* Capitalize interface `Scrape`

* Use `any[]` format for array

* add missing semicolon

* add line breaks between interfaces

* convert to single quotes

* remove `attr` for a test

* add more options
---
 types/website-scraper/index.d.ts              | 20 ++++++---
 .../website-scraper/website-scraper-tests.ts  | 44 ++++++++++---------
 2 files changed, 38 insertions(+), 26 deletions(-)

diff --git a/types/website-scraper/index.d.ts b/types/website-scraper/index.d.ts
index 07185a1102..3ab27fd60b 100644
--- a/types/website-scraper/index.d.ts
+++ b/types/website-scraper/index.d.ts
@@ -13,19 +13,23 @@ declare namespace websiteScraper {
         url: string;
         filename: string;
     }
+
     interface SubDirectory {
         directory: string;
         extensions: string[];
     }
+
     interface Source {
         selector: string;
-        attr: string;
+        attr?: string;
     }
+
     interface RequestOptions {
-        headers: request.Headers
+        headers: request.Headers;
     }
+
     interface Options {
-        urls: Array<string | Url>;
+        urls: (string | Url)[];
         directory: string;
         urlFilter?: (url: string) => boolean;
         filenameGenerator?: string;
@@ -37,21 +41,27 @@ declare namespace websiteScraper {
         recursive?: boolean;
         maxDepth?: number;
         ignoreErrors?: boolean;
+        maxRecursiveDepth?: number;
+        requestConcurrency?: number;
+        plugins?: object[];
     }
+
     interface Resource {
         url: string;
         filename: string;
         assets: Resource[];
     }
+
     interface Callback {
         (error: any | null, result: Resource[] | null): void;
     }
-    interface scrape {
+
+    interface Scrape {
         (options: Options, callback: Callback): void;
         (options: Options): Promise<Resource[]>;
     }
 }
 
-declare var websiteScraper: websiteScraper.scrape;
+declare const websiteScraper: websiteScraper.Scrape;
 
 export = websiteScraper;
diff --git a/types/website-scraper/website-scraper-tests.ts b/types/website-scraper/website-scraper-tests.ts
index bbcb1fda9d..b285a9ad22 100644
--- a/types/website-scraper/website-scraper-tests.ts
+++ b/types/website-scraper/website-scraper-tests.ts
@@ -1,31 +1,33 @@
-import scraper = require("website-scraper");
+import scraper = require('website-scraper');
 
 scraper({
     urls: [
-        "http://nodejs.org/",
-        { url: "http://nodejs.org/about", filename: "about.html" },
-        { url: "http://blog.nodejs.org/", filename: "blog.html" }
+        'http://nodejs.org/',
+        { url: 'http://nodejs.org/about', filename: 'about.html' },
+        { url: 'http://blog.nodejs.org/', filename: 'blog.html' },
     ],
-    directory: "/path/to/save",
+    directory: '/path/to/save',
     subdirectories: [
-        { directory: "img", extensions: [".jpg", ".png", ".svg"] },
-        { directory: "js", extensions: [".js"] },
-        { directory: "css", extensions: [".css"] }
+        { directory: 'img', extensions: ['.jpg', '.png', '.svg'] },
+        { directory: 'js', extensions: ['.js'] },
+        { directory: 'css', extensions: ['.css'] },
     ],
     sources: [
-        { selector: "img", attr: "src" },
-        { selector: "link[rel=\"stylesheet\"]", attr: "href" },
-        { selector: "script", attr: "src" }
+        { selector: 'img' },
+        { selector: 'link[rel="stylesheet"]', attr: 'href' },
+        { selector: 'script', attr: 'src' },
     ],
     request: {
         headers: {
-            "User-Agent":
-            "Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 4 Build/JOP40D)\
-             AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19"
-        }
-    }
-}).then(function(result) {
-    console.log(result);
-}).catch(function(err) {
-    console.log(err);
-});
+            'User-Agent':
+                'Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 4 Build/JOP40D)\
+             AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19',
+        },
+    },
+})
+    .then(function(result) {
+        console.log(result);
+    })
+    .catch(function(err) {
+        console.log(err);
+    });