package common import ( "crypto/tls" "log/slog" "net" "strings" "time" "github.com/PuerkitoBio/goquery" "github.com/valyala/fasthttp" ) var ( // constants Patterns = []string{"index of", "directory listing for"} Ignore = []string{"..", ".", "../", "./", "parent directory", "last modified", "name", "size", "description"} ) func ValidRange(r string) bool { if _, _, err := net.ParseCIDR(r); err != nil { return false } return true } func Entries(doc *goquery.Document) []string { var list []string doc.Find("a").Each(func(i int, s *goquery.Selection) { if len(list) >= 100 { return } text := strings.TrimSpace(s.Text()) for _, ig := range Ignore { if text == ig { return } } list = append(list, text) }) return list } func Checktitle(doc *goquery.Document) bool { if title := doc.Find("title").Text(); len(title) > 0 { for _, pattern := range Patterns { if strings.Contains(title, pattern) { return true } } } return false } func MkClient() *fasthttp.Client { tmout := time.Duration(Conf.Tmout) * time.Millisecond dialer := &fasthttp.TCPDialer{ Concurrency: 0, } return &fasthttp.Client{ MaxResponseBodySize: 10 * 1024 * 1024, // 10mb ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second, MaxIdleConnDuration: 5 * time.Second, MaxConnsPerHost: Conf.Threads, NoDefaultUserAgentHeader: true, DisableHeaderNamesNormalizing: true, DisablePathNormalizing: true, TLSConfig: &tls.Config{InsecureSkipVerify: true}, DialTimeout: func(addr string, timeout time.Duration) (net.Conn, error) { return dialer.DialTimeout(addr, tmout) }, } } func Hit(l *slog.Logger, c *fasthttp.Client, url string) error { fallback, redirect := true, true for { if doc, redir, err := query(c, url); err == nil { if doc != nil { if Checktitle(doc) { entries := Entries(doc) l.Info("opendir", "url", url, "entries", entries) } } else if redir != "" && redirect { url = redir if !strings.HasPrefix(url, "https") { fallback = false } redirect = false continue } break } else { if !fallback { return err } url = "http://" + url[8:] fallback = false } } return nil } func query(c *fasthttp.Client, url string) (*goquery.Document, string, error) { req := fasthttp.AcquireRequest() req.SetRequestURI(url) req.Header.SetMethod(fasthttp.MethodGet) req.Header.SetUserAgent(Conf.UserAgent) defer fasthttp.ReleaseRequest(req) resp := fasthttp.AcquireResponse() defer fasthttp.ReleaseResponse(resp) // goes to stderr for verbosity, not included when stdout is piped if !Conf.Silent { println("->", url) } var err error if err = c.DoTimeout(req, resp, time.Duration(Conf.Tmout)*time.Millisecond); err == nil { // check for redirect if resp.StatusCode() >= 300 && resp.StatusCode() < 400 { if location := resp.Header.Peek("Location"); len(location) > 0 { redir := string(location) if strings.HasPrefix(redir, "http") { return nil, redir, nil } } } if body := strings.ToLower(string(resp.Body())); len(body) > 0 { doc, err := goquery.NewDocumentFromReader(strings.NewReader(body)) return doc, "", err } } return nil, "", err }