configurable paths, useragents, other improvements

README.md

@@ -10,12 +10,14 @@
 ## usage
 ```
 maraudir
-    -l <ip/domain list>
-    -r <cidr range>
-    -t <threads>          [50]
-    -timeout <ms>         [500]
-    -delay <ms>           [200]
-    -s <silent>           [false]
+    -l, --list <ip/domain list>
+    -r, --range <cidr range>
+    -p, --pathlist <uri path list>
+    -t, --threads <threads>          [100]
+        --timeout <ms>               [1000]
+        --delay <ms>                 [200]
+    -s, --silent                     [false]
+        --useragent <useragent>      [a googlebot one]
 
 results are written to stdout and can be piped accordingly,
 verbosity is directed to stderr when silent is not set

cmd/maraudir/main.go

@@ -6,5 +6,10 @@ import (
 
 func main() {
 	common.LoadParams()
+	if paths, err := common.ReadPaths(); err != nil {
+		common.Fatal("error reading uri paths:" + err.Error())
+	} else {
+		common.Paths = paths
+	}
 	common.Takeoff()
 }

common/args.go

@@ -1,37 +1,43 @@
 package common
 
 import (
-	"flag"
+	flag "github.com/spf13/pflag"
 )
 
 type Params struct {
-	List    string
-	Cidr    string
-	Threads int
-	Tmout   int
-	Delay   int
-	Silent  bool
+	List      string
+	Cidr      string
+	Pathlist  string
+	UserAgent string
+	Threads   int
+	Tmout     int
+	Delay     int
+	Silent    bool
 }
 
 var (
-	list    = flag.String("l", "", "")
-	cidr    = flag.String("r", "", "")
-	threads = flag.Int("t", 100, "")
-	tmout   = flag.Int("timeout", 1000, "")
-	delay   = flag.Int("delay", 200, "")
-	silent  = flag.Bool("s", false, "")
-	Conf    Params
+	list     = flag.StringP("list", "l", "", "")
+	cidr     = flag.StringP("range", "r", "", "")
+	pathlist = flag.StringP("pathlist", "p", "", "")
+	ua       = flag.StringP("useragent", "u", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "")
+	threads  = flag.IntP("threads", "t", 100, "") // concurrent workers (green threads) not full threads. semantic preference.
+	tmout    = flag.Int("timeout", 1000, "")
+	delay    = flag.Int("delay", 200, "")
+	silent   = flag.BoolP("silent", "s", false, "")
+	Conf     Params
 )
 
 func LoadParams() {
 	flag.Usage = Usage
 	flag.Parse()
 	Conf = Params{
-		List:    *list,
-		Cidr:    *cidr,
-		Threads: *threads,
-		Tmout:   *tmout,
-		Delay:   *delay,
-		Silent:  *silent,
+		List:      *list,
+		Cidr:      *cidr,
+		Pathlist:  *pathlist,
+		UserAgent: *ua,
+		Threads:   *threads,
+		Tmout:     *tmout,
+		Delay:     *delay,
+		Silent:    *silent,
 	}
 }

common/exec.go

@@ -1,42 +1,19 @@
 package common
 
 import (
-	"fmt"
 	"log/slog"
 	"os"
-	"strings"
 )
 
 func thread(l *slog.Logger, dests <-chan string, tab chan<- interface{}) {
 	c := MkClient()
+
 	for dest := range dests {
 	uriloop:
 		for _, uri := range Paths {
-			url := fmt.Sprintf("https://%s%s", dest, uri)
-			fallback, redirect := true, true
-			for {
-				if doc, redir, err := Hit(c, url); err == nil {
-					if doc != nil {
-						if Checktitle(doc) {
-							entries := Entries(doc)
-							l.Info("opendir", "url", url, "entries", entries)
-						}
-					} else if redir != "" && redirect {
-						url = redir
-						if !strings.HasPrefix(url, "https") {
-							fallback = false
-						}
-						redirect = false
-						continue
-					}
-					break
-				} else {
-					if !fallback {
-						break uriloop
-					}
-					url = "http://" + url[8:]
-					fallback = false
-				}
+			url := "https://" + dest + uri
+			if err := Hit(l, c, url); err != nil {
+				break uriloop
 			}
 		}
 	}

common/io.go

@@ -13,12 +13,14 @@ func Fatal(msg string) {
 func Usage() {
 	fmt.Fprintf(os.Stderr, `
 maraudir
-    -l <ip/domain list>
-    -r <cidr range>
-    -t <threads>          [50]
-    -timeout <ms>         [500]
-    -delay <ms>           [200]
-    -s <silent>           [false]
+    -l, --list <ip/domain list>
+    -r, --range <cidr range>
+    -p, --pathlist <uri path list>
+    -t, --threads <threads>          [100]
+        --timeout <ms>               [1000]
+        --delay <ms>                 [200]
+    -s, --silent                     [false]
+        --useragent <useragent>      [a googlebot one]
 
 results are written to stdout and can be piped accordingly,
 verbosity is directed to stderr when silent is not set

common/net.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"crypto/tls"
+	"log/slog"
 	"net"
 	"strings"
 	"time"
@@ -12,10 +13,8 @@ import (
 
 var (
 	// constants
-	Paths     = []string{"/", "/files", "/ftp", "/backup", "/backups", "/config", "/logs", "/data", "/uploads", "/temp", "/tmp", "/static"}
-	Patterns  = []string{"index of", "directory listing for"}
-	Ignore    = []string{"..", ".", "../", "./", "parent directory", "last modified", "name", "size", "description"}
-	UserAgent = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+	Patterns = []string{"index of", "directory listing for"}
+	Ignore   = []string{"..", ".", "../", "./", "parent directory", "last modified", "name", "size", "description"}
 )
 
 func ValidRange(r string) bool {
@@ -79,11 +78,40 @@ func MkClient() *fasthttp.Client {
 	}
 }
 
-func Hit(c *fasthttp.Client, url string) (*goquery.Document, string, error) {
+func Hit(l *slog.Logger, c *fasthttp.Client, url string) error {
+	fallback, redirect := true, true
+	for {
+		if doc, redir, err := query(c, url); err == nil {
+			if doc != nil {
+				if Checktitle(doc) {
+					entries := Entries(doc)
+					l.Info("opendir", "url", url, "entries", entries)
+				}
+			} else if redir != "" && redirect {
+				url = redir
+				if !strings.HasPrefix(url, "https") {
+					fallback = false
+				}
+				redirect = false
+				continue
+			}
+			break
+		} else {
+			if !fallback {
+				return err
+			}
+			url = "http://" + url[8:]
+			fallback = false
+		}
+	}
+	return nil
+}
+
+func query(c *fasthttp.Client, url string) (*goquery.Document, string, error) {
 	req := fasthttp.AcquireRequest()
 	req.SetRequestURI(url)
 	req.Header.SetMethod(fasthttp.MethodGet)
-	req.Header.SetUserAgent(UserAgent)
+	req.Header.SetUserAgent(Conf.UserAgent)
 	defer fasthttp.ReleaseRequest(req)
 
 	resp := fasthttp.AcquireResponse()

common/read.go

@@ -6,6 +6,10 @@ import (
 	"strings"
 )
 
+var (
+	Paths []string
+)
+
 func Readfile(filename string, out chan<- string) error {
 	file, err := os.Open(filename)
 	if err != nil {
@@ -21,3 +25,24 @@ func Readfile(filename string, out chan<- string) error {
 
 	return scanner.Err()
 }
+
+func ReadPaths() ([]string, error) {
+	if Conf.Pathlist == "" {
+		// default fallback paths, some of the most common opendir uris
+		return []string{"/", "/files", "/ftp", "/backup", "/backups", "/config", "/dev", "/archive", "/old", "/scripts", "/logs", "/data", "/uploads", "/temp", "/tmp", "/static"}, nil
+	}
+	var paths []string
+
+	file, err := os.Open(Conf.Pathlist)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		paths = append(paths, strings.TrimSpace(scanner.Text()))
+	}
+
+	return paths, scanner.Err()
+}

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	golang.org/x/net v0.30.0 // indirect
 )

go.sum

@@ -6,6 +6,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
 github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.57.0 h1:Xw8SjWGEP/+wAAgyy5XTvgrWlOD1+TxbbvNADYCm1Tg=