Types definitions to simplecrawler module (#34997)

* simplecrawler modules type definition

* required fixes in export

types/simplecrawler/cache-backend-fs.d.ts

@@ -0,0 +1,16 @@
+import { QueueItem } from './queue';
+
+export default function(loadParameter?: string): FSBackend;
+
+export class FSBackend {
+  laoded: boolean;
+  index: number[];
+  location: string;
+
+  fileExists(location: string): boolean;
+  isDirectory(location: string): boolean;
+  load(): void;
+  saveCache(callback?: (err: NodeJS.ErrnoException | null) => void): void;
+  setItem(queueObject: QueueItem, data: any, callback?: () => void): void;
+  getItem(queueObject: QueueItem, callback?: (error?: Error, data?: any) => void): false;
+}

types/simplecrawler/cache.d.ts

@@ -0,0 +1,14 @@
+import { EventEmitter } from 'events';
+import { QueueItem } from './queue';
+import FilesystemBackend, { FSBackend } from './cache-backend-fs';
+
+export default class Cache extends EventEmitter {
+  datastore: FSBackend;
+  constructor(cacheLoadParameter: string[], cacheBackend: () => void);
+
+  setCacheData(queueObject: QueueItem, data: any, callback?: () => void): void;
+  getCacheData(queueObject: QueueItem, callback?: (error?: Error, data?: any) => void): void;
+  saveCache(): void;
+}
+
+export { FilesystemBackend, Cache };

types/simplecrawler/cookies.d.ts

@@ -0,0 +1,50 @@
+import { EventEmitter } from 'events';
+
+export class Cookie {
+  static fromString(content: string): Cookie;
+
+  name: string;
+  value: string;
+  expires: string | number;
+  path?: string;
+  domain?: string;
+  httponly?: boolean;
+
+  constructor(
+    name: string,
+    value: string,
+    expires: string | number,
+    path?: string,
+    domain?: string,
+    httponly?: boolean,
+  );
+
+  toOutboundString(): string;
+  toString(includeHeader?: boolean): string;
+  isExpired(): boolean;
+  matchDomain(domain: string): boolean;
+  matchPath(path: string): boolean;
+}
+
+export default class CookieJar extends EventEmitter {
+  cookies: Cookie[];
+
+  add(
+    name: string,
+    value: string,
+    expiry: string | number,
+    path?: string,
+    domain?: string,
+    httponly?: boolean,
+    callback?: (error?: Error, cookie?: Cookie) => void,
+  ): this;
+  remove(
+    name?: string,
+    domain?: string,
+    callback?: (error?: Error, cookiesRemoved?: Cookie[]) => void,
+  ): Cookie[];
+  get(name?: string, domain?: string, callback?: (error?: Error, cookies?: Cookie[]) => void): Cookie[];
+  getAsHeader(name?: string, domain?: string, callback?: (error?: Error, cookies?: Cookie[]) => void): string[];
+  addFromHeaders(headers: string | string[], callback?: (error?: Error) => void): this;
+  toString(): string;
+}

types/simplecrawler/crawler.d.ts

@@ -0,0 +1,143 @@
+import { EventEmitter } from 'events';
+import { Agent as HTTPAgent, IncomingMessage } from 'http';
+import { Agent as HTTPSAgent } from 'https';
+
+import Cache from './cache';
+import CookieJar from './cookies';
+import FetchQueue, { QueueItem } from './queue';
+
+type HTTPMethods = 'GET' | 'POST' | 'HEAD' | 'PUT' | 'DELETE' | 'CONNECT' | 'OPTIONS' | 'TRACE' | 'PATCH';
+
+interface RequestOptions {
+    method: HTTPMethods;
+    host: string;
+    port?: number;
+    agent: HTTPAgent | HTTPSAgent;
+    headers: {
+        Accept: string;
+        'User-Agent': string;
+        Host: string;
+        'Accept-Encoding'?: string;
+        Referer?: string;
+        cookie?: string;
+        Authorization?: string;
+        'Proxy-Authorization'?: string;
+        [key: string]: any;
+    };
+}
+
+declare class Crawler extends EventEmitter {
+    initialURL: string;
+    host: string;
+    interval: number;
+    maxConcurrency: number;
+    timeout: number;
+    listenerTTL: number;
+    userAgent: string;
+    queue: FetchQueue;
+    respectRobotsTxt: boolean;
+    allowInitialDomainChange: boolean;
+    decompressResponses: boolean;
+    decodeResponses: boolean;
+    filterByDomain: boolean;
+    scanSubdomains: boolean;
+    ignoreWWWDomain: boolean;
+    stripWWWDomain: boolean;
+    cache?: Cache;
+    useProxy: boolean;
+    proxyHostname: string;
+    proxyPort: number;
+    proxyUser?: string;
+    proxyPass?: string;
+    needsAuth: boolean;
+    authUser?: string;
+    authPass?: string;
+    acceptCookies: boolean;
+    cookies: CookieJar;
+    customHeaders: object;
+    domainWhitelist: string[];
+    allowedProtocols: RegExp[];
+    maxResourceSize: number;
+    supportedMimeTypes: Array<string | RegExp>;
+    downloadUnsupported: boolean;
+    urlEncoding: string;
+    stripQuerystring: boolean;
+    sortQueryParameters: boolean;
+    discoverRegex: Array<(() => void) | RegExp>;
+    parseHTMLComments: boolean;
+    parseScriptTags: boolean;
+    maxDepth: number;
+    ignoreInvalidSSL: boolean;
+    httpAgent: HTTPAgent;
+    httpsAgent: HTTPSAgent;
+
+    constructor(initialURL: string);
+
+    start(): this;
+
+    urlIsAllowed(url: string): boolean;
+
+    getRequestOptions(queueItem: QueueItem): RequestOptions;
+
+    getRobotsTxt(url: string, callback: (error?: Error, href?: string, responseBody?: string) => void): this;
+
+    protocolSupported(url: string): boolean;
+
+    mimeTypeSupported(mimetype: string): boolean;
+
+    processURL(url: string, referer?: QueueItem): QueueItem;
+
+    cleanURL(url: string, queueItem: QueueItem): string;
+
+    cleanExpandResources(urlMatch: string[], queueItem: QueueItem): string[];
+
+    discoverResources(resourceText: string): string[];
+
+    domainValid(host: string): boolean;
+
+    queueLinkedItems(resourceData: string | Buffer, queueItem: QueueItem): this;
+
+    queueURL(url: string, referrer?: QueueItem, force?: boolean): boolean;
+
+    fetchQueueItem(queueItem: QueueItem): this;
+
+    decodeBuffer(buffer: Buffer, contentTypeHeader?: string): string;
+
+    handleResponse(queueItem: QueueItem, response: IncomingMessage, timeCommenced?: Date): string;
+
+    /**
+     * @deprecated
+     */
+    crawl(): never;
+
+    stop(abortRequestsInFlight?: boolean): this;
+
+    wait(): () => void;
+
+    addDownloadCondition(callback: (queueItem: QueueItem, response: IncomingMessage) => void): number;
+
+    removeDownloadCondition(id: number | (() => void)): boolean;
+
+    addFetchCondition(callback: (queueItem: QueueItem, referrerQueueItem: QueueItem) => void): number;
+
+    removeFetchCondition(id: number | (() => void)): boolean;
+
+    on(event: 'crawlstart' | 'complete', listener: () => void): this;
+    on(event: 'discoverycomplete', listener: (queueItem: QueueItem, resources: string[]) => void): this;
+    on(event: 'invaliddomain' | 'fetchdisallowed' | 'queueduplicate', listener: (queueItem: QueueItem) => void): this;
+    on(event: 'fetchconditionerror' | 'downloadconditionerror', listener: (queueItem: QueueItem, error: any) => void): this;
+    on(event: 'fetchprevented', listener: (queueItem: QueueItem, fetchCondition: () => boolean) => void): this;
+    on(event: 'queueerror', listener: (error?: QueueItem, queueItem?: QueueItem) => void): this;
+    on(event: 'queueadd', listener: (queueItem?: QueueItem, referrer?: QueueItem) => void): this;
+    on(event: 'fetchtimeout', listener: (queueItem: QueueItem, timeout: number) => void): this;
+    on(event: 'fetchclienterror', listener: (queueItem: QueueItem, error?: object) => void): this;
+    on(event: 'fetchstart', listener: (queueItem: QueueItem, requestOptions: RequestOptions) => void): this;
+    on(event: 'cookieerror', listener: (queueItem: QueueItem, error: Error, cookie: string) => void): this;
+    on(event: 'fetchheaders' | 'downloadprevented' | 'fetch404' | 'fetch410' | 'fetcherror' | 'fetchdataerror', listener: (queueItem: QueueItem, response: IncomingMessage) => void): this;
+    on(event: 'notmodified', listener: (queueItem: QueueItem, response: IncomingMessage, cacheObject: Cache) => void): this;
+    on(event: 'fetchredirect', listener: (queueItem: QueueItem, redirectQueueItem: QueueItem, response: IncomingMessage) => void): this;
+    on(event: 'fetchcomplete' | 'gziperror', listener: (queueItem: QueueItem, responseBody: string | Buffer, response: IncomingMessage) => void): this;
+    on(event: 'robotstxterror', listener: (error: Error) => void): this;
+}
+
+export = Crawler;

types/simplecrawler/index.d.ts

@@ -0,0 +1,11 @@
+// Type definitions for simplecrawler 1.1
+// Project: https://github.com/simplecrawler/simplecrawler
+// Definitions by: Pedro Mutter <https://github.com/MutterPedro>
+// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
+// TypeScript Version: 2.2
+
+export as namespace simplecrawler;
+
+import Crawler = require('./crawler');
+
+export = Crawler;

types/simplecrawler/queue.d.ts

@@ -0,0 +1,41 @@
+export type QueueItemStatus = 'queued' | 'spooled' | 'headers' | 'downloaded' | 'redirected' | 'notfound' | 'failed';
+export interface QueueItem {
+  id: number;
+  url: string;
+  protocol: string;
+  host: string;
+  port: number;
+  path: string;
+  uriPath: string;
+  depth: number;
+  referrer: string;
+  fetched: boolean;
+  status: QueueItemStatus;
+  stateData: {
+    requestLatency: number;
+    requestTime: number;
+    downloadTime: number;
+    contentLength: number;
+    contentType: string;
+    code: number;
+    headers: object;
+    actualDataSize: number;
+    sentIncorrectSize: boolean;
+  };
+}
+
+export default class FetchQueue extends Array {
+  add(queueItem: QueueItem, force?: boolean, callback?: (error?: Error, queueItem?: QueueItem) => void): void;
+  exists(url: string, callback: (error: Error | null, exists: 0 | 1) => void): void;
+  get(index: number, callback?: (error?: Error, queueItem?: QueueItem) => void): void;
+  update(id: number, updates: object, callback?: (error?: Error, queueItem?: QueueItem) => void): void;
+  oldestUnfetchedItem(callback?: (error?: Error, queueItem?: QueueItem) => void): void;
+  max(statisticName: string, callback?: (error?: Error, maximum?: number) => void): void;
+  min(statisticName: string, callback?: (error?: Error, minimum?: number) => void): void;
+  avg(statisticName: string, callback?: (error?: Error, average?: number) => void): void;
+  countItems(comparator: object, callback?: (error?: Error, count?: number) => void): void;
+  filterItems(comparator: object, callback?: (error?: Error, items?: QueueItem[]) => void): void;
+  getLength(callback?: (error?: Error, length?: number) => void): void;
+  freeze(filename: string, callback?: (error?: Error) => void): void;
+  defrost(filename: string, callback?: (error?: Error) => void): void;
+}

types/simplecrawler/simplecrawler-tests.ts

@@ -0,0 +1,52 @@
+import Crawler = require('simplecrawler');
+
+const crawler = new Crawler('https://github.com/');
+crawler.initialURL; // $ExpectType string
+crawler.host; // $ExpectType string
+crawler.interval; // $ExpectType number
+crawler.maxConcurrency; // $ExpectType number
+crawler.timeout; // $ExpectType number
+crawler.listenerTTL; // $ExpectType number
+crawler.userAgent; // $ExpectType string
+crawler.queue; // $ExpectType FetchQueue
+crawler.respectRobotsTxt; // $ExpectType boolean
+crawler.allowInitialDomainChange; // $ExpectType boolean
+crawler.decompressResponses; // $ExpectType boolean
+crawler.decodeResponses; // $ExpectType boolean
+crawler.filterByDomain; // $ExpectType boolean
+crawler.scanSubdomains; // $ExpectType boolean
+crawler.ignoreWWWDomain; // $ExpectType boolean
+crawler.stripWWWDomain; // $ExpectType boolean
+crawler.cache; // $ExpectType Cache | undefined
+crawler.useProxy; // $ExpectType boolean
+crawler.proxyHostname; // $ExpectType string
+crawler.proxyPort; // $ExpectType number
+crawler.proxyUser; // $ExpectType string | undefined
+crawler.proxyPass; // $ExpectType string | undefined
+crawler.needsAuth; // $ExpectType boolean
+crawler.authUser; // $ExpectType string | undefined
+crawler.authPass; // $ExpectType string | undefined
+crawler.acceptCookies; // $ExpectType boolean
+crawler.cookies; // $ExpectType CookieJar
+crawler.customHeaders; // $ExpectType object
+crawler.domainWhitelist; // $ExpectType string[]
+crawler.allowedProtocols; // $ExpectType RegExp[]
+crawler.maxResourceSize; // $ExpectType number
+crawler.supportedMimeTypes; // $ExpectType (string | RegExp)[]
+crawler.downloadUnsupported; // $ExpectType boolean
+crawler.urlEncoding; // $ExpectType string
+crawler.stripQuerystring; // $ExpectType boolean
+crawler.sortQueryParameters; // $ExpectType boolean
+crawler.discoverRegex; // $ExpectType (RegExp | (() => void))[]
+crawler.parseHTMLComments; // $ExpectType boolean
+crawler.parseScriptTags; // $ExpectType boolean
+crawler.maxDepth; // $ExpectType number
+crawler.ignoreInvalidSSL; // $ExpectType boolean
+crawler.httpAgent; // $ExpectType Agent
+crawler.httpsAgent; // $ExpectType Agent
+crawler.start();
+crawler.on('fetchcomplete', (queueItem, responseBody, response) => {
+    queueItem; // $ExpectType QueueItem
+    responseBody; // $ExpectType string | Buffer
+    response; // $ExpectType IncomingMessage
+});

types/simplecrawler/tsconfig.json

@@ -0,0 +1,30 @@
+{
+    "compilerOptions": {
+        "module": "commonjs",
+        "lib": [
+            "es6"
+        ],
+        "noImplicitAny": true,
+        "noImplicitThis": true,
+        "strictNullChecks": true,
+        "strictFunctionTypes": true,
+        "baseUrl": "../",
+        "typeRoots": [
+            "../"
+        ],
+        "types": [],
+        "noEmit": true,
+        "forceConsistentCasingInFileNames": true,
+        "esModuleInterop": true
+    },
+    "files": [
+        "index.d.ts",
+        "simplecrawler-tests.ts",
+        "cache.d.ts",
+        "cookies.d.ts",
+        "queue.d.ts",
+        "cache-backend-fs.d.ts",
+        "crawler.d.ts",
+        "../node/v7/index.d.ts"
+    ]
+}

types/simplecrawler/tslint.json

@@ -0,0 +1 @@
+{ "extends": "dtslint/dt.json" }