From 6b2b0d344cb3fb537c7fa2341e07179672b0927c Mon Sep 17 00:00:00 2001
From: Milos Pejanovic <pejan123@gmail.com>
Date: Tue, 27 Sep 2022 19:14:47 +0200
Subject: [PATCH] disable SSL certificate verification on crawler

---
 scrapeloop.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scrapeloop.go b/scrapeloop.go
index 3c9d8fe..4630035 100644
--- a/scrapeloop.go
+++ b/scrapeloop.go
@@ -2,6 +2,7 @@ package walker
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"net"
@@ -55,6 +56,10 @@ func newClientPool(concurrency int, agent string, useCookies bool) *clientPool {
 				IdleConnTimeout:       15 * time.Second,
 				TLSHandshakeTimeout:   5 * time.Second,
 				ExpectContinueTimeout: 1 * time.Second,
+				TLSClientConfig: &tls.Config{
+					// disable SSL certificate verification
+					InsecureSkipVerify: true,
+				},
 			},
 
 			CheckRedirect: func(req *http.Request, via []*http.Request) error {