diff --git a/types/simplecrawler/cache-backend-fs.d.ts b/types/simplecrawler/cache-backend-fs.d.ts new file mode 100644 index 0000000000..d30c750451 --- /dev/null +++ b/types/simplecrawler/cache-backend-fs.d.ts @@ -0,0 +1,16 @@ +import { QueueItem } from './queue'; + +export default function(loadParameter?: string): FSBackend; + +export class FSBackend { + laoded: boolean; + index: number[]; + location: string; + + fileExists(location: string): boolean; + isDirectory(location: string): boolean; + load(): void; + saveCache(callback?: (err: NodeJS.ErrnoException | null) => void): void; + setItem(queueObject: QueueItem, data: any, callback?: () => void): void; + getItem(queueObject: QueueItem, callback?: (error?: Error, data?: any) => void): false; +} diff --git a/types/simplecrawler/cache.d.ts b/types/simplecrawler/cache.d.ts new file mode 100644 index 0000000000..9ef677e67d --- /dev/null +++ b/types/simplecrawler/cache.d.ts @@ -0,0 +1,14 @@ +import { EventEmitter } from 'events'; +import { QueueItem } from './queue'; +import FilesystemBackend, { FSBackend } from './cache-backend-fs'; + +export default class Cache extends EventEmitter { + datastore: FSBackend; + constructor(cacheLoadParameter: string[], cacheBackend: () => void); + + setCacheData(queueObject: QueueItem, data: any, callback?: () => void): void; + getCacheData(queueObject: QueueItem, callback?: (error?: Error, data?: any) => void): void; + saveCache(): void; +} + +export { FilesystemBackend, Cache }; diff --git a/types/simplecrawler/cookies.d.ts b/types/simplecrawler/cookies.d.ts new file mode 100644 index 0000000000..177b416cc4 --- /dev/null +++ b/types/simplecrawler/cookies.d.ts @@ -0,0 +1,50 @@ +import { EventEmitter } from 'events'; + +export class Cookie { + static fromString(content: string): Cookie; + + name: string; + value: string; + expires: string | number; + path?: string; + domain?: string; + httponly?: boolean; + + constructor( + name: string, + value: string, + expires: string | number, + path?: string, + domain?: string, + httponly?: boolean, + ); + + toOutboundString(): string; + toString(includeHeader?: boolean): string; + isExpired(): boolean; + matchDomain(domain: string): boolean; + matchPath(path: string): boolean; +} + +export default class CookieJar extends EventEmitter { + cookies: Cookie[]; + + add( + name: string, + value: string, + expiry: string | number, + path?: string, + domain?: string, + httponly?: boolean, + callback?: (error?: Error, cookie?: Cookie) => void, + ): this; + remove( + name?: string, + domain?: string, + callback?: (error?: Error, cookiesRemoved?: Cookie[]) => void, + ): Cookie[]; + get(name?: string, domain?: string, callback?: (error?: Error, cookies?: Cookie[]) => void): Cookie[]; + getAsHeader(name?: string, domain?: string, callback?: (error?: Error, cookies?: Cookie[]) => void): string[]; + addFromHeaders(headers: string | string[], callback?: (error?: Error) => void): this; + toString(): string; +} diff --git a/types/simplecrawler/crawler.d.ts b/types/simplecrawler/crawler.d.ts new file mode 100644 index 0000000000..98b82b13a9 --- /dev/null +++ b/types/simplecrawler/crawler.d.ts @@ -0,0 +1,143 @@ +import { EventEmitter } from 'events'; +import { Agent as HTTPAgent, IncomingMessage } from 'http'; +import { Agent as HTTPSAgent } from 'https'; + +import Cache from './cache'; +import CookieJar from './cookies'; +import FetchQueue, { QueueItem } from './queue'; + +type HTTPMethods = 'GET' | 'POST' | 'HEAD' | 'PUT' | 'DELETE' | 'CONNECT' | 'OPTIONS' | 'TRACE' | 'PATCH'; + +interface RequestOptions { + method: HTTPMethods; + host: string; + port?: number; + agent: HTTPAgent | HTTPSAgent; + headers: { + Accept: string; + 'User-Agent': string; + Host: string; + 'Accept-Encoding'?: string; + Referer?: string; + cookie?: string; + Authorization?: string; + 'Proxy-Authorization'?: string; + [key: string]: any; + }; +} + +declare class Crawler extends EventEmitter { + initialURL: string; + host: string; + interval: number; + maxConcurrency: number; + timeout: number; + listenerTTL: number; + userAgent: string; + queue: FetchQueue; + respectRobotsTxt: boolean; + allowInitialDomainChange: boolean; + decompressResponses: boolean; + decodeResponses: boolean; + filterByDomain: boolean; + scanSubdomains: boolean; + ignoreWWWDomain: boolean; + stripWWWDomain: boolean; + cache?: Cache; + useProxy: boolean; + proxyHostname: string; + proxyPort: number; + proxyUser?: string; + proxyPass?: string; + needsAuth: boolean; + authUser?: string; + authPass?: string; + acceptCookies: boolean; + cookies: CookieJar; + customHeaders: object; + domainWhitelist: string[]; + allowedProtocols: RegExp[]; + maxResourceSize: number; + supportedMimeTypes: Array; + downloadUnsupported: boolean; + urlEncoding: string; + stripQuerystring: boolean; + sortQueryParameters: boolean; + discoverRegex: Array<(() => void) | RegExp>; + parseHTMLComments: boolean; + parseScriptTags: boolean; + maxDepth: number; + ignoreInvalidSSL: boolean; + httpAgent: HTTPAgent; + httpsAgent: HTTPSAgent; + + constructor(initialURL: string); + + start(): this; + + urlIsAllowed(url: string): boolean; + + getRequestOptions(queueItem: QueueItem): RequestOptions; + + getRobotsTxt(url: string, callback: (error?: Error, href?: string, responseBody?: string) => void): this; + + protocolSupported(url: string): boolean; + + mimeTypeSupported(mimetype: string): boolean; + + processURL(url: string, referer?: QueueItem): QueueItem; + + cleanURL(url: string, queueItem: QueueItem): string; + + cleanExpandResources(urlMatch: string[], queueItem: QueueItem): string[]; + + discoverResources(resourceText: string): string[]; + + domainValid(host: string): boolean; + + queueLinkedItems(resourceData: string | Buffer, queueItem: QueueItem): this; + + queueURL(url: string, referrer?: QueueItem, force?: boolean): boolean; + + fetchQueueItem(queueItem: QueueItem): this; + + decodeBuffer(buffer: Buffer, contentTypeHeader?: string): string; + + handleResponse(queueItem: QueueItem, response: IncomingMessage, timeCommenced?: Date): string; + + /** + * @deprecated + */ + crawl(): never; + + stop(abortRequestsInFlight?: boolean): this; + + wait(): () => void; + + addDownloadCondition(callback: (queueItem: QueueItem, response: IncomingMessage) => void): number; + + removeDownloadCondition(id: number | (() => void)): boolean; + + addFetchCondition(callback: (queueItem: QueueItem, referrerQueueItem: QueueItem) => void): number; + + removeFetchCondition(id: number | (() => void)): boolean; + + on(event: 'crawlstart' | 'complete', listener: () => void): this; + on(event: 'discoverycomplete', listener: (queueItem: QueueItem, resources: string[]) => void): this; + on(event: 'invaliddomain' | 'fetchdisallowed' | 'queueduplicate', listener: (queueItem: QueueItem) => void): this; + on(event: 'fetchconditionerror' | 'downloadconditionerror', listener: (queueItem: QueueItem, error: any) => void): this; + on(event: 'fetchprevented', listener: (queueItem: QueueItem, fetchCondition: () => boolean) => void): this; + on(event: 'queueerror', listener: (error?: QueueItem, queueItem?: QueueItem) => void): this; + on(event: 'queueadd', listener: (queueItem?: QueueItem, referrer?: QueueItem) => void): this; + on(event: 'fetchtimeout', listener: (queueItem: QueueItem, timeout: number) => void): this; + on(event: 'fetchclienterror', listener: (queueItem: QueueItem, error?: object) => void): this; + on(event: 'fetchstart', listener: (queueItem: QueueItem, requestOptions: RequestOptions) => void): this; + on(event: 'cookieerror', listener: (queueItem: QueueItem, error: Error, cookie: string) => void): this; + on(event: 'fetchheaders' | 'downloadprevented' | 'fetch404' | 'fetch410' | 'fetcherror' | 'fetchdataerror', listener: (queueItem: QueueItem, response: IncomingMessage) => void): this; + on(event: 'notmodified', listener: (queueItem: QueueItem, response: IncomingMessage, cacheObject: Cache) => void): this; + on(event: 'fetchredirect', listener: (queueItem: QueueItem, redirectQueueItem: QueueItem, response: IncomingMessage) => void): this; + on(event: 'fetchcomplete' | 'gziperror', listener: (queueItem: QueueItem, responseBody: string | Buffer, response: IncomingMessage) => void): this; + on(event: 'robotstxterror', listener: (error: Error) => void): this; +} + +export = Crawler; diff --git a/types/simplecrawler/index.d.ts b/types/simplecrawler/index.d.ts new file mode 100644 index 0000000000..980f242647 --- /dev/null +++ b/types/simplecrawler/index.d.ts @@ -0,0 +1,11 @@ +// Type definitions for simplecrawler 1.1 +// Project: https://github.com/simplecrawler/simplecrawler +// Definitions by: Pedro Mutter +// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped +// TypeScript Version: 2.2 + +export as namespace simplecrawler; + +import Crawler = require('./crawler'); + +export = Crawler; diff --git a/types/simplecrawler/queue.d.ts b/types/simplecrawler/queue.d.ts new file mode 100644 index 0000000000..8182f3dfda --- /dev/null +++ b/types/simplecrawler/queue.d.ts @@ -0,0 +1,41 @@ +export type QueueItemStatus = 'queued' | 'spooled' | 'headers' | 'downloaded' | 'redirected' | 'notfound' | 'failed'; +export interface QueueItem { + id: number; + url: string; + protocol: string; + host: string; + port: number; + path: string; + uriPath: string; + depth: number; + referrer: string; + fetched: boolean; + status: QueueItemStatus; + stateData: { + requestLatency: number; + requestTime: number; + downloadTime: number; + contentLength: number; + contentType: string; + code: number; + headers: object; + actualDataSize: number; + sentIncorrectSize: boolean; + }; +} + +export default class FetchQueue extends Array { + add(queueItem: QueueItem, force?: boolean, callback?: (error?: Error, queueItem?: QueueItem) => void): void; + exists(url: string, callback: (error: Error | null, exists: 0 | 1) => void): void; + get(index: number, callback?: (error?: Error, queueItem?: QueueItem) => void): void; + update(id: number, updates: object, callback?: (error?: Error, queueItem?: QueueItem) => void): void; + oldestUnfetchedItem(callback?: (error?: Error, queueItem?: QueueItem) => void): void; + max(statisticName: string, callback?: (error?: Error, maximum?: number) => void): void; + min(statisticName: string, callback?: (error?: Error, minimum?: number) => void): void; + avg(statisticName: string, callback?: (error?: Error, average?: number) => void): void; + countItems(comparator: object, callback?: (error?: Error, count?: number) => void): void; + filterItems(comparator: object, callback?: (error?: Error, items?: QueueItem[]) => void): void; + getLength(callback?: (error?: Error, length?: number) => void): void; + freeze(filename: string, callback?: (error?: Error) => void): void; + defrost(filename: string, callback?: (error?: Error) => void): void; +} diff --git a/types/simplecrawler/simplecrawler-tests.ts b/types/simplecrawler/simplecrawler-tests.ts new file mode 100644 index 0000000000..4089ca1ce3 --- /dev/null +++ b/types/simplecrawler/simplecrawler-tests.ts @@ -0,0 +1,52 @@ +import Crawler = require('simplecrawler'); + +const crawler = new Crawler('https://github.com/'); +crawler.initialURL; // $ExpectType string +crawler.host; // $ExpectType string +crawler.interval; // $ExpectType number +crawler.maxConcurrency; // $ExpectType number +crawler.timeout; // $ExpectType number +crawler.listenerTTL; // $ExpectType number +crawler.userAgent; // $ExpectType string +crawler.queue; // $ExpectType FetchQueue +crawler.respectRobotsTxt; // $ExpectType boolean +crawler.allowInitialDomainChange; // $ExpectType boolean +crawler.decompressResponses; // $ExpectType boolean +crawler.decodeResponses; // $ExpectType boolean +crawler.filterByDomain; // $ExpectType boolean +crawler.scanSubdomains; // $ExpectType boolean +crawler.ignoreWWWDomain; // $ExpectType boolean +crawler.stripWWWDomain; // $ExpectType boolean +crawler.cache; // $ExpectType Cache | undefined +crawler.useProxy; // $ExpectType boolean +crawler.proxyHostname; // $ExpectType string +crawler.proxyPort; // $ExpectType number +crawler.proxyUser; // $ExpectType string | undefined +crawler.proxyPass; // $ExpectType string | undefined +crawler.needsAuth; // $ExpectType boolean +crawler.authUser; // $ExpectType string | undefined +crawler.authPass; // $ExpectType string | undefined +crawler.acceptCookies; // $ExpectType boolean +crawler.cookies; // $ExpectType CookieJar +crawler.customHeaders; // $ExpectType object +crawler.domainWhitelist; // $ExpectType string[] +crawler.allowedProtocols; // $ExpectType RegExp[] +crawler.maxResourceSize; // $ExpectType number +crawler.supportedMimeTypes; // $ExpectType (string | RegExp)[] +crawler.downloadUnsupported; // $ExpectType boolean +crawler.urlEncoding; // $ExpectType string +crawler.stripQuerystring; // $ExpectType boolean +crawler.sortQueryParameters; // $ExpectType boolean +crawler.discoverRegex; // $ExpectType (RegExp | (() => void))[] +crawler.parseHTMLComments; // $ExpectType boolean +crawler.parseScriptTags; // $ExpectType boolean +crawler.maxDepth; // $ExpectType number +crawler.ignoreInvalidSSL; // $ExpectType boolean +crawler.httpAgent; // $ExpectType Agent +crawler.httpsAgent; // $ExpectType Agent +crawler.start(); +crawler.on('fetchcomplete', (queueItem, responseBody, response) => { + queueItem; // $ExpectType QueueItem + responseBody; // $ExpectType string | Buffer + response; // $ExpectType IncomingMessage +}); diff --git a/types/simplecrawler/tsconfig.json b/types/simplecrawler/tsconfig.json new file mode 100644 index 0000000000..2c5d1dfd17 --- /dev/null +++ b/types/simplecrawler/tsconfig.json @@ -0,0 +1,30 @@ +{ + "compilerOptions": { + "module": "commonjs", + "lib": [ + "es6" + ], + "noImplicitAny": true, + "noImplicitThis": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "baseUrl": "../", + "typeRoots": [ + "../" + ], + "types": [], + "noEmit": true, + "forceConsistentCasingInFileNames": true, + "esModuleInterop": true + }, + "files": [ + "index.d.ts", + "simplecrawler-tests.ts", + "cache.d.ts", + "cookies.d.ts", + "queue.d.ts", + "cache-backend-fs.d.ts", + "crawler.d.ts", + "../node/v7/index.d.ts" + ] +} diff --git a/types/simplecrawler/tslint.json b/types/simplecrawler/tslint.json new file mode 100644 index 0000000000..3db14f85ea --- /dev/null +++ b/types/simplecrawler/tslint.json @@ -0,0 +1 @@ +{ "extends": "dtslint/dt.json" }