foomo/walker
1
0
Fork 0
mirror of https://github.com/foomo/walker.git synced 2025-10-16 12:45:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

disable SSL certificate verification on crawler

Browse Source
This commit is contained in:
Milos Pejanovic 2022-09-27 19:14:47 +02:00
parent 5c50c10c9e
commit 6b2b0d344c
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapeloop.go
View File

@ -2,6 +2,7 @@ package walker
import ( import (
"context" "context"
"crypto/tls"
"errors" "errors"
"fmt" "fmt"
"net" "net"
@ -55,6 +56,10 @@ func newClientPool(concurrency int, agent string, useCookies bool) *clientPool {
IdleConnTimeout: 15 * time.Second, IdleConnTimeout: 15 * time.Second,
TLSHandshakeTimeout: 5 * time.Second, TLSHandshakeTimeout: 5 * time.Second,
ExpectContinueTimeout: 1 * time.Second, ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: &tls.Config{
// disable SSL certificate verification
InsecureSkipVerify: true,
},
}, },
CheckRedirect: func(req *http.Request, via []*http.Request) error { CheckRedirect: func(req *http.Request, via []*http.Request) error {