mirror of
https://github.com/soapingtime/diyhrt.git
synced 2026-11-01 23:14:51 +01:00
update hrtcafe.net + change domain in the useragents
This commit is contained in:
@@ -1,27 +0,0 @@
|
||||
import sys
|
||||
|
||||
def remove_trailing_spaces(line):
|
||||
return line.rstrip()
|
||||
|
||||
def remove_duplicate_lines(file_path):
|
||||
lines_seen = set()
|
||||
output_lines = []
|
||||
|
||||
with open(file_path, 'r') as file:
|
||||
for line in file:
|
||||
cleaned_line = remove_trailing_spaces(line)
|
||||
|
||||
if cleaned_line.startswith("https://diyhrt.cafe") and cleaned_line not in lines_seen:
|
||||
lines_seen.add(cleaned_line)
|
||||
output_lines.append(line)
|
||||
|
||||
with open(file_path, 'w') as file:
|
||||
file.writelines(output_lines)
|
||||
|
||||
# Usage example
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Please provide the file path as an argument.")
|
||||
else:
|
||||
file_path = sys.argv[1]
|
||||
remove_duplicate_lines(file_path)
|
||||
@@ -1,29 +0,0 @@
|
||||
from bs4 import BeautifulSoup
|
||||
from urllib.request import Request, urlopen
|
||||
import re
|
||||
import sys
|
||||
from urllib.parse import urljoin
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Please provide a URL.")
|
||||
sys.exit(1)
|
||||
|
||||
url = sys.argv[1]
|
||||
|
||||
headers = {'User-Agent': 'Mozilla/5.0'}
|
||||
|
||||
req = Request(url, headers=headers)
|
||||
|
||||
html_page = urlopen(req)
|
||||
|
||||
soup = BeautifulSoup(html_page, "html.parser")
|
||||
|
||||
links = []
|
||||
for link in soup.findAll('a'):
|
||||
href = link.get('href')
|
||||
if href:
|
||||
full_url = urljoin(url, href)
|
||||
links.append(full_url)
|
||||
|
||||
formatted_links = '\n'.join(links)
|
||||
print(formatted_links)
|
||||
Reference in New Issue
Block a user