From e1c3827d65a35d87e45cefe04135c1ff16374410 Mon Sep 17 00:00:00 2001
From: Alexander Kabui
Date: Thu, 11 Feb 2021 03:26:49 +0300
Subject: add links validator

---
 test/requests/links_scraper/genelinks.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'test')

diff --git a/test/requests/links_scraper/genelinks.py b/test/requests/links_scraper/genelinks.py
index 223a2c31..5dddcc47 100644
--- a/test/requests/links_scraper/genelinks.py
+++ b/test/requests/links_scraper/genelinks.py
@@ -7,11 +7,20 @@ import logging
 from urllib.request import urlopen as uReq
 from bs4 import BeautifulSoup as soup
 from urllib.parse import urljoin
+from urllib.parse import urlparse
 
 
 PORT = os.environ.get("PORT", "5004")
 
 
+def is_valid_link(url_link):
+    try:
+        result = urlparse(url_link)
+        return all([result.scheme, result.netloc, result.path])
+    except Exception as e:
+        return False
+
+
 def test_link(link, strict=True):
     print(f"link testing {link}")
     results = None
@@ -57,7 +66,8 @@ def fetch_html_links(parsed_page):
         if re.match(r"^/", link_url):
             full_path = urljoin('http://localhost:5004/', link_url)
 
-        elif re.match(r'^http://', link_url):
+        elif is_valid_link(link_url):
+            print(link_url)
             full_path = link_url
 
         if full_path is not None:
-- 
cgit v1.2.3