improved title parsing, misc formatting, readme
This commit is contained in:
parent
4578bfe896
commit
9ebe835278
@ -1,3 +1,9 @@
|
|||||||
# speedboat
|
<p align="center">
|
||||||
|
<img src="https://i.imgur.com/uQiapHN.gif" width="1000" title="speedboat demo">
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<center>
|
||||||
|
<h1 style="color: hotpink;"><span style="color: cyan;">speed</span>boat</h1>
|
||||||
|
|
||||||
lightweight web service aggregator, offering performance/reliability that httpX lacks for protracted scans
|
lightweight web service aggregator, offering performance/reliability that httpX lacks for protracted scans
|
||||||
|
</center>
|
@ -10,7 +10,7 @@ pub struct Params {
|
|||||||
#[derive(Parser, Default)]
|
#[derive(Parser, Default)]
|
||||||
#[clap(
|
#[clap(
|
||||||
author = "tommy touchdown",
|
author = "tommy touchdown",
|
||||||
about = "speedboat - lightweight web content aggregator",
|
about = "speedboat - lightweight web service aggregator",
|
||||||
version = VERSION
|
version = VERSION
|
||||||
)]
|
)]
|
||||||
pub struct Config {
|
pub struct Config {
|
||||||
|
@ -18,18 +18,34 @@ pub fn fmtcode(code: u16) -> ColoredString {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn parsehit(sc: u16, url: String, title: &str) -> String {
|
pub fn parsetitle(s: String) -> String {
|
||||||
|
let mut out = String::with_capacity(s.len());
|
||||||
|
s.split_whitespace().for_each(|w| {
|
||||||
|
if !out.is_empty() {
|
||||||
|
out.push(' ');
|
||||||
|
}
|
||||||
|
out.push_str(w);
|
||||||
|
});
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn parsehit(sc: u16, url: String, title: String) -> String {
|
||||||
format!(
|
format!(
|
||||||
"{} {} {} {}{}{}",
|
"{} {} {} {}{}{}",
|
||||||
fmtcode(sc),
|
fmtcode(sc),
|
||||||
"|".black().bold(),
|
"|".black().bold(),
|
||||||
url.white().underline(),
|
url.white().underline(),
|
||||||
"[".black(),
|
"[".black().bold(),
|
||||||
title.trim_matches(['\n', '\t', '\r']).bright_cyan().bold(),
|
parsetitle(title).bright_cyan().bold(),
|
||||||
"]".black()
|
"]".black().bold()
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn banner() {
|
pub fn banner() {
|
||||||
eprintln!("{}{} {}", "speed".bright_cyan().bold(), "boat".bright_magenta().bold(), VERSION.black())
|
eprintln!(
|
||||||
|
"{}{} {}",
|
||||||
|
"speed".bright_cyan().bold(),
|
||||||
|
"boat".bright_magenta().bold(),
|
||||||
|
VERSION.black()
|
||||||
|
);
|
||||||
}
|
}
|
@ -1,5 +1,8 @@
|
|||||||
use futures::{stream, StreamExt};
|
use futures::{stream, StreamExt};
|
||||||
use std::{fs::File, io::{BufReader, BufRead}};
|
use std::{
|
||||||
|
fs::File,
|
||||||
|
io::{BufRead, BufReader},
|
||||||
|
};
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
conf::{Config, Params},
|
conf::{Config, Params},
|
||||||
@ -18,12 +21,17 @@ let buf = BufReader::new(file);
|
|||||||
|
|
||||||
stream::iter(buf.lines())
|
stream::iter(buf.lines())
|
||||||
.for_each_concurrent(args.threads, |line| {
|
.for_each_concurrent(args.threads, |line| {
|
||||||
// call request function
|
|
||||||
|
|
||||||
let wc = c.clone();
|
let wc = c.clone();
|
||||||
let scodes = params.statcodes.clone();
|
let scodes = params.statcodes.clone();
|
||||||
async move {
|
async move {
|
||||||
let _ = query(wc, line.unwrap_or_else(|_| fatal("error attempting buffered read")).trim(), scodes, params.exclude).await;
|
let _ = query(
|
||||||
|
wc,
|
||||||
|
line.unwrap_or_else(|_| fatal("error attempting buffered read"))
|
||||||
|
.trim(),
|
||||||
|
scodes,
|
||||||
|
params.exclude,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
|
@ -42,20 +42,15 @@ pub async fn query(
|
|||||||
let url: String = response.url().to_string();
|
let url: String = response.url().to_string();
|
||||||
let body = response.text().await?;
|
let body = response.text().await?;
|
||||||
|
|
||||||
// Parse the HTML document
|
|
||||||
let document = Document::from(body.as_str());
|
let document = Document::from(body.as_str());
|
||||||
|
|
||||||
// Use select to find the <title> tag
|
|
||||||
let title = document
|
let title = document
|
||||||
.find(Name("title"))
|
.find(Name("title"))
|
||||||
.next()
|
.next()
|
||||||
.map(|n| n.text())
|
.map(|n| n.text())
|
||||||
.unwrap_or_else(|| "".to_string());
|
.unwrap_or_else(|| "".to_string());
|
||||||
|
|
||||||
println!(
|
println!("{}", parsehit(sc, url, title));
|
||||||
"{}",
|
|
||||||
parsehit(sc, url, title.trim_matches(['\n', '\t', '\r']))
|
|
||||||
);
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user