109 lines
2.6 KiB
Go
109 lines
2.6 KiB
Go
|
package common
|
||
|
|
|
||
|
|
import (
|
||
|
|
"net"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/PuerkitoBio/goquery"
|
||
|
|
"github.com/valyala/fasthttp"
|
||
|
|
)
|
||
|
|
|
||
|
|
var (
|
||
|
|
// constants
|
||
|
|
Paths = []string{"/", "/files/", "/ftp/", "/backup/", "/backups/", "/config/", "/logs/", "/data/", "/uploads/", "/temp/", "/tmp/", "/static/"}
|
||
|
|
Patterns = []string{"index of", "directory listing for"}
|
||
|
|
Ignore = []string{"..", ".", "../", "./", "parent directory", "last modified", "name", "size", "description"}
|
||
|
|
UserAgent = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
|
||
|
|
)
|
||
|
|
|
||
|
|
func ValidRange(r string) bool {
|
||
|
|
if _, _, err := net.ParseCIDR(r); err != nil {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
|
||
|
|
func Entries(doc *goquery.Document) []string {
|
||
|
|
var list []string
|
||
|
|
doc.Find("a").Each(func(i int, s *goquery.Selection) {
|
||
|
|
if len(list) >= 100 {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
text := strings.TrimSpace(s.Text())
|
||
|
|
|
||
|
|
for _, ig := range Ignore {
|
||
|
|
if text == ig {
|
||
|
|
return
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
list = append(list, text)
|
||
|
|
})
|
||
|
|
|
||
|
|
return list
|
||
|
|
}
|
||
|
|
|
||
|
|
func Checktitle(doc *goquery.Document) bool {
|
||
|
|
if title := doc.Find("title").Text(); len(title) > 0 {
|
||
|
|
for _, pattern := range Patterns {
|
||
|
|
if strings.Contains(title, pattern) {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
|
||
|
|
func MkClient() *fasthttp.Client {
|
||
|
|
tmout := time.Duration(Conf.Tmout) * time.Millisecond
|
||
|
|
dialer := &fasthttp.TCPDialer{
|
||
|
|
Concurrency: 0,
|
||
|
|
}
|
||
|
|
|
||
|
|
return &fasthttp.Client{
|
||
|
|
MaxResponseBodySize: 10 * 1024 * 1024, // 10mb
|
||
|
|
ReadTimeout: 5 * time.Second,
|
||
|
|
WriteTimeout: tmout,
|
||
|
|
MaxIdleConnDuration: 5 * time.Second,
|
||
|
|
MaxConnsPerHost: Conf.Threads,
|
||
|
|
NoDefaultUserAgentHeader: true,
|
||
|
|
DisableHeaderNamesNormalizing: true,
|
||
|
|
DisablePathNormalizing: true,
|
||
|
|
DialTimeout: func(addr string, timeout time.Duration) (net.Conn, error) {
|
||
|
|
return dialer.DialTimeout(addr, tmout)
|
||
|
|
},
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func Hit(c *fasthttp.Client, url string) (*goquery.Document, error) {
|
||
|
|
req := fasthttp.AcquireRequest()
|
||
|
|
req.SetRequestURI(url)
|
||
|
|
req.Header.SetMethod(fasthttp.MethodGet)
|
||
|
|
req.Header.SetUserAgent(UserAgent)
|
||
|
|
defer fasthttp.ReleaseRequest(req)
|
||
|
|
|
||
|
|
resp := fasthttp.AcquireResponse()
|
||
|
|
defer fasthttp.ReleaseResponse(resp)
|
||
|
|
|
||
|
|
// goes to stderr for verbosity, not included when stdout is piped
|
||
|
|
if !Conf.Silent {
|
||
|
|
println("->", url)
|
||
|
|
}
|
||
|
|
|
||
|
|
var err error
|
||
|
|
if err = c.DoTimeout(req, resp, time.Duration(Conf.Tmout)*time.Millisecond); err == nil {
|
||
|
|
if body := strings.ToLower(string(resp.Body())); len(body) > 0 {
|
||
|
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
|
||
|
|
if err != nil {
|
||
|
|
return nil, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
return doc, nil
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return nil, err
|
||
|
|
}
|