maraudir/common/net.go

package common

import (
	"crypto/tls"
	"net"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/valyala/fasthttp"
)

var (
	// constants
	Paths     = []string{"/", "/files", "/ftp", "/backup", "/backups", "/config", "/logs", "/data", "/uploads", "/temp", "/tmp", "/static"}
	Patterns  = []string{"index of", "directory listing for"}
	Ignore    = []string{"..", ".", "../", "./", "parent directory", "last modified", "name", "size", "description"}
	UserAgent = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
)

func ValidRange(r string) bool {
	if _, _, err := net.ParseCIDR(r); err != nil {
		return false
	}
	return true
}

func Entries(doc *goquery.Document) []string {
	var list []string
	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		if len(list) >= 100 {
			return
		}

		text := strings.TrimSpace(s.Text())

		for _, ig := range Ignore {
			if text == ig {
				return
			}
		}

		list = append(list, text)
	})

	return list
}

func Checktitle(doc *goquery.Document) bool {
	if title := doc.Find("title").Text(); len(title) > 0 {
		for _, pattern := range Patterns {
			if strings.Contains(title, pattern) {
				return true
			}
		}
	}
	return false
}

func MkClient() *fasthttp.Client {
	tmout := time.Duration(Conf.Tmout) * time.Millisecond
	dialer := &fasthttp.TCPDialer{
		Concurrency: 0,
	}

	return &fasthttp.Client{
		MaxResponseBodySize:           10 * 1024 * 1024, // 10mb
		ReadTimeout:                   5 * time.Second,
		WriteTimeout:                  5 * time.Second,
		MaxIdleConnDuration:           5 * time.Second,
		MaxConnsPerHost:               Conf.Threads,
		NoDefaultUserAgentHeader:      true,
		DisableHeaderNamesNormalizing: true,
		DisablePathNormalizing:        true,
		TLSConfig:                     &tls.Config{InsecureSkipVerify: true},
		DialTimeout: func(addr string, timeout time.Duration) (net.Conn, error) {
			return dialer.DialTimeout(addr, tmout)
		},
	}
}

func Hit(c *fasthttp.Client, url string) (*goquery.Document, string, error) {
	req := fasthttp.AcquireRequest()
	req.SetRequestURI(url)
	req.Header.SetMethod(fasthttp.MethodGet)
	req.Header.SetUserAgent(UserAgent)
	defer fasthttp.ReleaseRequest(req)

	resp := fasthttp.AcquireResponse()
	defer fasthttp.ReleaseResponse(resp)

	// goes to stderr for verbosity, not included when stdout is piped
	if !Conf.Silent {
		println("->", url)
	}

	var err error
	if err = c.DoTimeout(req, resp, time.Duration(Conf.Tmout)*time.Millisecond); err == nil {
		// check for redirect
		if resp.StatusCode() >= 300 && resp.StatusCode() < 400 {
			if location := resp.Header.Peek("Location"); len(location) > 0 {
				redir := string(location)
				if strings.HasPrefix(redir, "http") {
					return nil, redir, nil
				}
			}
		}

		if body := strings.ToLower(string(resp.Body())); len(body) > 0 {
			doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
			return doc, "", err
		}
	}

	return nil, "", err
}