43 lines
746 B
Go
43 lines
746 B
Go
|
package proxy
|
||
|
|
|
||
|
|
import (
|
||
|
|
"io"
|
||
|
|
"net/http"
|
||
|
|
|
||
|
|
"github.com/rs/zerolog"
|
||
|
|
)
|
||
|
|
|
||
|
|
// Scrape actions
|
||
|
|
type Scrape struct {
|
||
|
|
Domain string // Domain name
|
||
|
|
Proxies chan string // Proxy channel
|
||
|
|
Logger zerolog.Logger // Global logger
|
||
|
|
Verbose bool // Verbose mode
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s *Scrape) Fetch() {
|
||
|
|
// Send GET request
|
||
|
|
resp, err := http.Get(s.Domain)
|
||
|
|
if err != nil {
|
||
|
|
if s.Verbose {
|
||
|
|
s.Logger.Debug().Msg(err.Error())
|
||
|
|
}
|
||
|
|
s.Proxies <- ""
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// Read HTTP body
|
||
|
|
body, err := io.ReadAll(resp.Body)
|
||
|
|
if err != nil {
|
||
|
|
if s.Verbose {
|
||
|
|
s.Logger.Debug().Msg(err.Error())
|
||
|
|
}
|
||
|
|
s.Proxies <- ""
|
||
|
|
return
|
||
|
|
}
|
||
|
|
|
||
|
|
// Send proxies & log it
|
||
|
|
s.Proxies <- string(body)
|
||
|
|
s.Logger.Info().Str("domain", s.Domain).Msg("Fetched proxies from")
|
||
|
|
}
|