package common import ( "net" "strings" "time" "github.com/PuerkitoBio/goquery" "github.com/valyala/fasthttp" ) var ( // constants Paths = []string{"/", "/files/", "/ftp/", "/backup/", "/backups/", "/config/", "/logs/", "/data/", "/uploads/", "/temp/", "/tmp/", "/static/"} Patterns = []string{"index of", "directory listing for"} Ignore = []string{"..", ".", "../", "./", "parent directory", "last modified", "name", "size", "description"} UserAgent = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" ) func ValidRange(r string) bool { if _, _, err := net.ParseCIDR(r); err != nil { return false } return true } func Entries(doc *goquery.Document) []string { var list []string doc.Find("a").Each(func(i int, s *goquery.Selection) { if len(list) >= 100 { return } text := strings.TrimSpace(s.Text()) for _, ig := range Ignore { if text == ig { return } } list = append(list, text) }) return list } func Checktitle(doc *goquery.Document) bool { if title := doc.Find("title").Text(); len(title) > 0 { for _, pattern := range Patterns { if strings.Contains(title, pattern) { return true } } } return false } func MkClient() *fasthttp.Client { tmout := time.Duration(Conf.Tmout) * time.Millisecond dialer := &fasthttp.TCPDialer{ Concurrency: 0, } return &fasthttp.Client{ MaxResponseBodySize: 10 * 1024 * 1024, // 10mb ReadTimeout: 5 * time.Second, WriteTimeout: tmout, MaxIdleConnDuration: 5 * time.Second, MaxConnsPerHost: Conf.Threads, NoDefaultUserAgentHeader: true, DisableHeaderNamesNormalizing: true, DisablePathNormalizing: true, DialTimeout: func(addr string, timeout time.Duration) (net.Conn, error) { return dialer.DialTimeout(addr, tmout) }, } } func Hit(c *fasthttp.Client, url string) (*goquery.Document, error) { req := fasthttp.AcquireRequest() req.SetRequestURI(url) req.Header.SetMethod(fasthttp.MethodGet) req.Header.SetUserAgent(UserAgent) defer fasthttp.ReleaseRequest(req) resp := fasthttp.AcquireResponse() defer fasthttp.ReleaseResponse(resp) // goes to stderr for verbosity, not included when stdout is piped if !Conf.Silent { println("->", url) } var err error if err = c.DoTimeout(req, resp, time.Duration(Conf.Tmout)*time.Millisecond); err == nil { if body := strings.ToLower(string(resp.Body())); len(body) > 0 { doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) if err != nil { return nil, nil } return doc, nil } } return nil, err }