From ed53ec320ef2567056544bb60507d98f3cdcba92 Mon Sep 17 00:00:00 2001
From: Erin Nova <erin@the-system.eu.org>
Date: Tue, 25 Jul 2023 20:11:29 -0400
Subject: [PATCH] Fix panic, preventing full crawling

---
 crawler/src/main.rs | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/crawler/src/main.rs b/crawler/src/main.rs
index 5dd21d2..69ee7f3 100644
--- a/crawler/src/main.rs
+++ b/crawler/src/main.rs
@@ -48,6 +48,7 @@ async fn main() {
         for res in results {
             for t in res {
                 for url in t {
+                    info!("pushing {}", &url.as_str());
                     to_crawl.push(url);
                 }
             }
@@ -143,10 +144,16 @@ async fn find_links(html: &str, base: &Url, allow: Vec<Url>) -> Vec<Url> {
         // if &url == base {
         //     break;
         // }
+        info!("Found url: {}", &url.as_str());
         for x in &allow {
-            if &x.domain().unwrap() == &url.domain().unwrap() {
-                links.push(url);
-                break;
+            match &url.domain() {
+                Some(d) => {
+                    if &x.domain().unwrap() == d {
+                        links.push(url);
+                        break;
+                    }
+                }
+                None => {}
             }
         }
     }