improved title parsing, misc formatting, readme

This commit is contained in:
delorean 2024-05-02 19:55:44 -05:00
parent 4578bfe896
commit 9ebe835278
6 changed files with 48 additions and 23 deletions

View File

@ -1,3 +1,9 @@
# speedboat <p align="center">
<img src="https://i.imgur.com/uQiapHN.gif" width="1000" title="speedboat demo">
</p>
lightweight web service aggregator, offering performance/reliability that httpX lacks for protracted scans <center>
<h1 style="color: hotpink;"><span style="color: cyan;">speed</span>boat</h1>
lightweight web service aggregator, offering performance/reliability that httpX lacks for protracted scans
</center>

View File

@ -10,7 +10,7 @@ pub struct Params {
#[derive(Parser, Default)] #[derive(Parser, Default)]
#[clap( #[clap(
author = "tommy touchdown", author = "tommy touchdown",
about = "speedboat - lightweight web content aggregator", about = "speedboat - lightweight web service aggregator",
version = VERSION version = VERSION
)] )]
pub struct Config { pub struct Config {

View File

@ -18,18 +18,34 @@ pub fn fmtcode(code: u16) -> ColoredString {
} }
} }
pub fn parsehit(sc: u16, url: String, title: &str) -> String { pub fn parsetitle(s: String) -> String {
let mut out = String::with_capacity(s.len());
s.split_whitespace().for_each(|w| {
if !out.is_empty() {
out.push(' ');
}
out.push_str(w);
});
out
}
pub fn parsehit(sc: u16, url: String, title: String) -> String {
format!( format!(
"{} {} {} {}{}{}", "{} {} {} {}{}{}",
fmtcode(sc), fmtcode(sc),
"|".black().bold(), "|".black().bold(),
url.white().underline(), url.white().underline(),
"[".black(), "[".black().bold(),
title.trim_matches(['\n', '\t', '\r']).bright_cyan().bold(), parsetitle(title).bright_cyan().bold(),
"]".black() "]".black().bold()
) )
} }
pub fn banner() { pub fn banner() {
eprintln!("{}{} {}", "speed".bright_cyan().bold(), "boat".bright_magenta().bold(), VERSION.black()) eprintln!(
"{}{} {}",
"speed".bright_cyan().bold(),
"boat".bright_magenta().bold(),
VERSION.black()
);
} }

View File

@ -1,5 +1,8 @@
use futures::{stream, StreamExt}; use futures::{stream, StreamExt};
use std::{fs::File, io::{BufReader, BufRead}}; use std::{
fs::File,
io::{BufRead, BufReader},
};
use super::{ use super::{
conf::{Config, Params}, conf::{Config, Params},
@ -11,19 +14,24 @@ pub async fn takeoff(args: Config, params: Params) {
let c = mkclient(args.follow).unwrap_or_else(|_| fatal("error instantiating http client")); let c = mkclient(args.follow).unwrap_or_else(|_| fatal("error instantiating http client"));
let file = File::open(args.list) let file = File::open(args.list)
.unwrap_or_else(|e| fatal(format!("unable to read file: {e}").as_str())); .unwrap_or_else(|e| fatal(format!("unable to read file: {e}").as_str()));
// Create a buffered reader. // Create a buffered reader.
let buf = BufReader::new(file); let buf = BufReader::new(file);
stream::iter(buf.lines()) stream::iter(buf.lines())
.for_each_concurrent(args.threads, |line| { .for_each_concurrent(args.threads, |line| {
// call request function
let wc = c.clone(); let wc = c.clone();
let scodes = params.statcodes.clone(); let scodes = params.statcodes.clone();
async move { async move {
let _ = query(wc, line.unwrap_or_else(|_| fatal("error attempting buffered read")).trim(), scodes, params.exclude).await; let _ = query(
wc,
line.unwrap_or_else(|_| fatal("error attempting buffered read"))
.trim(),
scodes,
params.exclude,
)
.await;
} }
}) })
.await; .await;

View File

@ -42,20 +42,15 @@ pub async fn query(
let url: String = response.url().to_string(); let url: String = response.url().to_string();
let body = response.text().await?; let body = response.text().await?;
// Parse the HTML document
let document = Document::from(body.as_str()); let document = Document::from(body.as_str());
// Use select to find the <title> tag
let title = document let title = document
.find(Name("title")) .find(Name("title"))
.next() .next()
.map(|n| n.text()) .map(|n| n.text())
.unwrap_or_else(|| "".to_string()); .unwrap_or_else(|| "".to_string());
println!( println!("{}", parsehit(sc, url, title));
"{}",
parsehit(sc, url, title.trim_matches(['\n', '\t', '\r']))
);
Ok(()) Ok(())
} }