maraudir/common/net.go

package common

import (
	"crypto/tls"
	"log/slog"
	"net"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/valyala/fasthttp"
)

var (
	// constants
	Patterns = []string{"index of", "directory listing for"}
	Ignore   = []string{"..", ".", "../", "./", "parent directory", "last modified", "name", "size", "description"}
)

func ValidRange(r string) bool {
	if _, _, err := net.ParseCIDR(r); err != nil {
		return false
	}
	return true
}

func Entries(doc *goquery.Document) []string {
	var list []string
	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		if len(list) >= 100 {
			return
		}

		text := strings.TrimSpace(s.Text())

		for _, ig := range Ignore {
			if text == ig {
				return
			}
		}

		list = append(list, text)
	})

	return list
}

func Checktitle(doc *goquery.Document) bool {
	if title := doc.Find("title").Text(); len(title) > 0 {
		for _, pattern := range Patterns {
			if strings.Contains(title, pattern) {
				return true
			}
		}
	}
	return false
}

func MkClient() *fasthttp.Client {
	tmout := time.Duration(Conf.Tmout) * time.Millisecond
	dialer := &fasthttp.TCPDialer{
		Concurrency: 0,
	}

	return &fasthttp.Client{
		MaxResponseBodySize:           10 * 1024 * 1024, // 10mb
		ReadTimeout:                   5 * time.Second,
		WriteTimeout:                  5 * time.Second,
		MaxIdleConnDuration:           5 * time.Second,
		MaxConnsPerHost:               Conf.Threads,
		NoDefaultUserAgentHeader:      true,
		DisableHeaderNamesNormalizing: true,
		DisablePathNormalizing:        true,
		TLSConfig:                     &tls.Config{InsecureSkipVerify: true},
		DialTimeout: func(addr string, timeout time.Duration) (net.Conn, error) {
			return dialer.DialTimeout(addr, tmout)
		},
	}
}

func Hit(l *slog.Logger, c *fasthttp.Client, url string) error {
	fallback, redirect := true, true
	for {
		if doc, redir, err := query(c, url); err == nil {
			if doc != nil {
				if Checktitle(doc) {
					entries := Entries(doc)
					l.Info("opendir", "url", url, "entries", entries)
				}
			} else if redir != "" && redirect {
				url = redir
				if !strings.HasPrefix(url, "https") {
					fallback = false
				}
				redirect = false
				continue
			}
			break
		} else {
			if !fallback {
				return err
			}
			url = "http://" + url[8:]
			fallback = false
		}
	}
	return nil
}

func query(c *fasthttp.Client, url string) (*goquery.Document, string, error) {
	req := fasthttp.AcquireRequest()
	req.SetRequestURI(url)
	req.Header.SetMethod(fasthttp.MethodGet)
	req.Header.SetUserAgent(Conf.UserAgent)
	defer fasthttp.ReleaseRequest(req)

	resp := fasthttp.AcquireResponse()
	defer fasthttp.ReleaseResponse(resp)

	// goes to stderr for verbosity, not included when stdout is piped
	if !Conf.Silent {
		println("->", url)
	}

	var err error
	if err = c.DoTimeout(req, resp, time.Duration(Conf.Tmout)*time.Millisecond); err == nil {
		// check for redirect
		if resp.StatusCode() >= 300 && resp.StatusCode() < 400 {
			if location := resp.Header.Peek("Location"); len(location) > 0 {
				redir := string(location)
				if strings.HasPrefix(redir, "http") {
					return nil, redir, nil
				}
			}
		}

		if body := strings.ToLower(string(resp.Body())); len(body) > 0 {
			doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
			return doc, "", err
		}
	}

	return nil, "", err
}