Fix panic, preventing full crawling

2023-07-25 20:11:29 -04:00 · 2023-07-25 20:11:29 -04:00 · ed53ec320e
parent c141135847
commit ed53ec320e
1 changed files with 10 additions and 3 deletions
--- a/crawler/src/main.rs
+++ b/crawler/src/main.rs
@ -48,6 +48,7 @@ async fn main() {
        for res in results {
            for t in res {
                for url in t {
+                    info!("pushing {}", &url.as_str());
                    to_crawl.push(url);
                }
            }
@ -143,10 +144,16 @@ async fn find_links(html: &str, base: &Url, allow: Vec<Url>) -> Vec<Url> {
        // if &url == base {
        //     break;
        // }
+        info!("Found url: {}", &url.as_str());
        for x in &allow {
-            if &x.domain().unwrap() == &url.domain().unwrap() {
-                links.push(url);
-                break;
+            match &url.domain() {
+                Some(d) => {
+                    if &x.domain().unwrap() == d {
+                        links.push(url);
+                        break;
+                    }
+                }
+                None => {}
            }
        }
    }