Fix panic, preventing full crawling
parent
c141135847
commit
ed53ec320e
|
@ -48,6 +48,7 @@ async fn main() {
|
||||||
for res in results {
|
for res in results {
|
||||||
for t in res {
|
for t in res {
|
||||||
for url in t {
|
for url in t {
|
||||||
|
info!("pushing {}", &url.as_str());
|
||||||
to_crawl.push(url);
|
to_crawl.push(url);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -143,10 +144,16 @@ async fn find_links(html: &str, base: &Url, allow: Vec<Url>) -> Vec<Url> {
|
||||||
// if &url == base {
|
// if &url == base {
|
||||||
// break;
|
// break;
|
||||||
// }
|
// }
|
||||||
|
info!("Found url: {}", &url.as_str());
|
||||||
for x in &allow {
|
for x in &allow {
|
||||||
if &x.domain().unwrap() == &url.domain().unwrap() {
|
match &url.domain() {
|
||||||
links.push(url);
|
Some(d) => {
|
||||||
break;
|
if &x.domain().unwrap() == d {
|
||||||
|
links.push(url);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue