violet/diyhrt
1
0
Fork 0
mirror of https://github.com/soapingtime/diyhrt.git synced 2026-09-01 02:20:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

update hrtcafe.net + change domain in the useragents

Browse Source
This commit is contained in:
soap
2024-03-06 13:09:21 -06:00
parent 7d5a3caf73
commit 8811d9c0fe
9 changed files with 5 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/diyhrt.market.yml
+1 -1
View File
@@ -24,7 +24,7 @@ jobs:
- name: download diyhrt.market
run: |
rm -rf diyhrt.market cdn.diyhrt.market
wget -mkpKEH --limit-rate 10m --random-wait --user-agent="Contact: hrt.boytw.ink/crawl_contact.txt" --domains=diyhrt.market,cdn.diyhrt.market https://diyhrt.market || true
wget -mkpKEH --limit-rate 10m --random-wait --user-agent="github.com/soapingtime/diyhrt" --domains=diyhrt.market,cdn.diyhrt.market https://diyhrt.market || true
find . -name "*.orig" -type f -delete
- name: commit changes
if: ${{ success() }}
.github/workflows/diyhrt.wiki.yml
+1 -1
View File
@@ -24,7 +24,7 @@ jobs:
- name: download diyhrt.wiki
run: |
rm -rf diyhrt.wiki
wget -mkxKE -e robots=off --limit-rate=10m --random-wait --user-agent="Contact: hrt.boytw.ink/crawl_contact.txt" --reject '*age-check*' https://diyhrt.wiki https://diyhrt.wiki/nav.html || true # downloads nav.html because it doesn't crawl properly without it
wget -mkxKE -e robots=off --limit-rate=10m --random-wait --user-agent="github.com/soapingtime/diyhrt" --reject '*age-check*' https://diyhrt.wiki https://diyhrt.wiki/nav.html || true # downloads nav.html because it doesn't crawl properly without it
cd diyhrt.wiki && rm *.orig && cd ..
- name: setup python
uses: actions/setup-python@v4
.github/workflows/hrt.coffee.yml
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
- name: Download hrt.coffee
run: |
rm -rf hrt.coffee
export USER_AGENT="Mosaic/9.0 (Commodore AmigaOS 4.0) AmiWeb/4.1.33 (KHTML, like Gecko) Version/22.1 Mozilla/5.0. Contact: https://boytw.ink/crawl_contact.txt"
export USER_AGENT="Mosaic/9.0 (Commodore AmigaOS 4.0) AmiWeb/4.1.33 (KHTML, like Gecko) Version/22.1 Mozilla/5.0. github.com/soapingtime/diyhrt"
wget --adjust-extension --convert-links -e robots=off --mirror --page-requisites --waitretry 5 --timeout 60 --tries 5 --wait 1 -U "$USER_AGENT" https://hrt.coffee
- name: Commit changes
if: ${{ success() }}
.github/workflows/diyhrt.cafe.yml → .github/workflows/hrtcafe.net.yml
+1 -1
View File
@@ -26,7 +26,7 @@ jobs:
run: |
rm -rf diyhrt.cafe
rm -rf hrtcafe.net
wget --convert-links --page-requisites --span-hosts --no-parent --reject 'rocket-loader.min.js' --reject '*UserLogin*' --reject '*visualEditor*' --reject '*CreateAccount*' --reject '*load.php*' --wait 3 --limit-rate 4m --continue --random-wait --adjust-extension -U 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.6) Gecko/20070802 SeaMonkey/1.1.4 Contact: hrt.boytw.ink/crawl_contact.txt' -e robots=off -i scripts/cafe.txt || true
wget -mkpKEH --limit-rate 10m --random-wait --user-agent="github.com/soapingtime/diyhrt" --domains=hrtcafe.net https://hrtcafe.net || true
- name: commit changes
if: ${{ success() }}
uses: EndBug/add-and-commit@v9
.github/workflows/transfemscience.org.yml
+1 -1
View File
@@ -24,7 +24,7 @@ jobs:
- name: download transfemscience
run: |
rm -rf transfemscience.org
wget -mkpKEH --limit-rate 10m --random-wait -e robots=off --user-agent="Contact: hrt.boytw.ink/crawl_contact.txt" --domains=transfemscience.org https://transfemscience.org || true
wget -mkpKEH --limit-rate 10m --random-wait -e robots=off --user-agent="github.com/soapingtime/diyhrt" --domains=transfemscience.org https://transfemscience.org || true
find . -name "*.orig" -type f -delete
- name: commit changes
if: ${{ success() }}
.github/workflows/update_cafe.yml
-32
View File
@@ -1,32 +0,0 @@
name: Update DIYHRT.cafe links
on:
workflow_dispatch:
schedule:
- cron: "30 23 1 * *" # every 1st of month
jobs:
update:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
token: ${{ secrets.PAT_TOKEN }}
- uses: actions/setup-python@v4
with:
python-version: pypy-3.7-v7.3.3
- name: Update links
if: always()
run: |
rm scripts/cafe.txt
pip install beautifulsoup4 bs4
python scripts/getlinks.py 'https://hrtcafe.net/index.php/Special:AllPages' > scripts/cafe.txt
cd scripts && python formatlinks.py cafe.txt
- name: Commit changes
if: ${{ success() }}
uses: EndBug/add-and-commit@v9
with:
add: "scripts/cafe.txt"
pull: "--rebase --autostash ."
message: "[automated] update diyhrt.cafe list"
default_author: github_actions