diff --git a/colly.go b/colly.go index ae74b7c3..f0c321e5 100644 --- a/colly.go +++ b/colly.go @@ -481,7 +481,7 @@ func (c *Collector) Init() { c.wg = &sync.WaitGroup{} c.lock = &sync.RWMutex{} c.robotsMap = make(map[string]*robotstxt.RobotsData) - c.IgnoreRobotsTxt = true + c.IgnoreRobotsTxt = false c.ID = atomic.AddUint32(&collectorCounter, 1) c.TraceHTTP = false c.Context = context.Background() @@ -1468,7 +1468,6 @@ func createMultipartReader(boundary string, data map[string][]byte) io.Reader { } buffer.WriteString(dashBoundary + "--\n\n") return bytes.NewReader(buffer.Bytes()) - } // randomBoundary was borrowed from