violet/diyhrt
1
0
Fork 0
mirror of https://github.com/soapingtime/diyhrt.git synced 2026-09-01 02:20:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

re-initialize and make some changes. repository got to around 2 gigabytes and needed to be more efficient

Browse Source
This commit is contained in:
soap
2023-09-29 12:56:26 -05:00
commit 234e4592fa
1441 changed files with 476627 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/codeberg_mirror.yml
+38
View File
@@ -0,0 +1,38 @@
name: Mirror to Codeberg
on:
workflow_dispatch:
push:
jobs:
codeberg:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
token: ${{ secrets.PAT_TOKEN }}
- name: check for 404
run: |
url="https://codeberg.org/diyhrt/pages"
status=$(curl -o /dev/null -s -w "%{http_code}" $url)
if [ $status -eq 404 ]; then
echo "Got 404. Exiting."
curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
exit 1
fi
- uses: pixta-dev/repository-mirroring-action@v1.1.1
with:
target_repo_url: "git@codeberg.org:diyhrt/pages.git"
ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}
secret_mirror:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
token: ${{ secrets.PAT_TOKEN }}
- uses: pixta-dev/repository-mirroring-action@v1.1.1
with:
target_repo_url: "${{ secrets.SECRET_MIRROR_URL }}"
ssh_private_key: ${{ secrets.SECRET_SSH_PRIVATE_KEY }}
.github/workflows/disroot_mirror.yml
+28
View File
@@ -0,0 +1,28 @@
name: Mirror to Disroot
on:
workflow_dispatch:
schedule:
- cron: "0 0 1,15 * *"
jobs:
disroot:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
token: ${{ secrets.PAT_TOKEN }}
- name: check for 404
run: |
url="https://git.disroot.org/diyhrt/mirrors"
status=$(curl -o /dev/null -s -w "%{http_code}" $url)
if [ $status -eq 404 ]; then
echo "Got 404. Exiting."
curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
exit 1
fi
- uses: pixta-dev/repository-mirroring-action@v1.1.1
with:
target_repo_url: "git@git.disroot.org:diyhrt/mirrors.git"
ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}
.github/workflows/diyhrt.cafe.yml
+36
View File
@@ -0,0 +1,36 @@
name: DIYHRT.cafe
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * 0"
jobs:
download:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
token: ${{ secrets.PAT_TOKEN }}
- name: check for 404
run: |
url="https://diyhrt.cafe"
status=$(curl -o /dev/null -s -w "%{http_code}" $url)
if [ $status -eq 404 ]; then
echo "Got 404. Exiting."
curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
exit 1
fi
- name: download .cafe
if: always()
run: |
rm -rf diyhrt.cafe
wget --convert-links --page-requisites --span-hosts --no-parent --reject 'rocket-loader.min.js' --reject '*UserLogin*' --reject '*visualEditor*' --reject '*CreateAccount*' --reject '*load.php*' --wait 3 --limit-rate 4m --continue --random-wait --adjust-extension -U 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.6) Gecko/20070802 SeaMonkey/1.1.4 Contact: hrt.boytw.ink/crawl_contact.txt' -e robots=off -i scripts/cafe.txt || true
- name: commit changes
if: ${{ success() }}
uses: EndBug/add-and-commit@v9
with:
add: "diyhrt.cafe"
pull: "--rebase --autostash ."
message: "[automated] update diyhrt.cafe"
default_author: github_actions
.github/workflows/diyhrt.market.yml
+36
View File
@@ -0,0 +1,36 @@
name: DIYHRT.market
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * 1"
jobs:
archive:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
token: ${{ secrets.PAT_TOKEN }}
- name: check for 404
run: |
url="https://diyhrt.market"
status=$(curl -o /dev/null -s -w "%{http_code}" $url)
if [ $status -eq 404 ]; then
echo "Got 404. Exiting."
curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
exit 1
fi
- name: download diyhrt.market
run: |
rm -rf diyhrt.market cdn.diyhrt.market
wget -mkpKEH --limit-rate 10m --random-wait --user-agent="Contact: hrt.boytw.ink/crawl_contact.txt" --domains=diyhrt.market,cdn.diyhrt.market https://diyhrt.market || true
find . -name "*.orig" -type f -delete
- name: commit changes
if: ${{ success() }}
uses: EndBug/add-and-commit@v9
with:
add: "['diyhrt.market', 'cdn.diyhrt.market']"
pull: "--rebase --autostash ."
message: "[automated] update diyhrt.market"
default_author: github_actions
.github/workflows/diyhrt.wiki.yml
+42
View File
@@ -0,0 +1,42 @@
name: DIYHRT.wiki
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * 2"
jobs:
archive:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
token: ${{ secrets.PAT_TOKEN }}
- name: check for 404
run: |
url="https://diyhrt.wiki"
status=$(curl -o /dev/null -s -w "%{http_code}" $url)
if [ $status -eq 404 ]; then
echo "Got 404. Exiting."
curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
exit 1
fi
- name: download diyhrt.wiki
run: |
rm -rf diyhrt.wiki
wget -mkxKE -e robots=off --limit-rate=10m --random-wait --user-agent="Contact: hrt.boytw.ink/crawl_contact.txt" --reject '*age-check*' https://diyhrt.wiki https://diyhrt.wiki/nav.html || true # downloads nav.html because it doesn't crawl properly without it
cd diyhrt.wiki && rm *.orig && cd ..
- name: setup python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: run agebypass.py # removes the age-check.js script because it was causing problems
run: python scripts/agebypass.py
- name: commit changes
if: ${{ success() }}
uses: EndBug/add-and-commit@v9
with:
add: "diyhrt.wiki"
pull: "--rebase --autostash ."
message: "[automated] update diyhrt.wiki"
default_author: github_actions
.github/workflows/groups.io.yml
+26
View File
@@ -0,0 +1,26 @@
name: groups.io wiki
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * 3"
jobs:
archive:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
token: ${{ secrets.PAT_TOKEN }}
- name: download groups.io wiki
run: |
rm -rf groups.io
wget -e robots=off --mirror --limit-rate=10m --convert-links --adjust-extension --page-requisites --random-wait --no-parent https://groups.io/g/MTFHRT/wiki/ https://groups.io/g/MTFHRT/topic/80225302/ https://groups.io/g/MTFHRT/topic/79606271/ || true
- name: commit changes
if: ${{ success() }}
uses: EndBug/add-and-commit@v9
with:
add: "groups.io"
pull: --rebase --autostash .
message: "[automated] update groups.io wiki"
default_author: github_actions
.github/workflows/hrt.coffee.yml
+40
View File
@@ -0,0 +1,40 @@
name: HRT.coffee
on:
workflow_dispatch:
schedule:
- cron: '0 0 * * 4'
jobs:
archive:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
token: ${{ secrets.PAT_TOKEN }}
- name: Check for 404
run: |
url="https://hrt.coffee"
status=$(curl -o /dev/null -s -w "%{http_code}" $url)
if [ $status -eq 404 ]; then
echo "Got 404. Exiting."
curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
exit 1
fi
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install wget
- name: Download hrt.coffee
run: |
rm -rf hrt.coffee
export USER_AGENT="Mosaic/9.0 (Commodore AmigaOS 4.0) AmiWeb/4.1.33 (KHTML, like Gecko) Version/22.1 Mozilla/5.0. Contact: https://boytw.ink/crawl_contact.txt"
wget --adjust-extension --convert-links -e robots=off --mirror --page-requisites --waitretry 5 --timeout 60 --tries 5 --wait 1 -U "$USER_AGENT" https://hrt.coffee
- name: Commit changes
if: ${{ success() }}
uses: EndBug/add-and-commit@v9
with:
add: hrt.coffee
pull: "--rebase --autostash ."
message: "[automated] update hrt.coffee"
default_author: github_actions
.github/workflows/md_to_html.yml
+32
View File
@@ -0,0 +1,32 @@
name: Convert markdown to html
on:
workflow_dispatch:
push:
paths:
- 'README.md'
- 'DISCLAIMER.md'
jobs:
convert:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
token: ${{ secrets.PAT_TOKEN }}
- uses: actions/setup-python@v4
with:
python-version: pypy-3.7-v7.3.3
- name: Convert to html
run: |
pip install markdown bleach
python scripts/md_to_html.py README.md readme.html
python scripts/md_to_html.py DISCLAIMER.md disclaimer.html
- name: Commit changes
if: ${{ success() }}
uses: EndBug/add-and-commit@v9
with:
add: "['readme.html', 'disclaimer.html']"
pull: "--rebase --autostash ."
message: "[automated] update markdown files"
default_author: github_actions
.github/workflows/reddit.yml
+40
View File
@@ -0,0 +1,40 @@
name: Reddit Wikis
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * 5"
jobs:
archive:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
token: ${{ secrets.PAT_TOKEN }}
- name: check for 404
run: |
urls=("https://old.reddit.com/r/TransDIY/wiki/pharmacies" "https://old.reddit.com/r/TransDIY/wiki" "https://old.reddit.com/r/TransSurgeriesWiki/wiki/index/" "https://old.reddit.com/r/FTM/wiki")
failed=""
for url in "${urls[@]}"; do
status=$(curl -o /dev/null -s -w "%{http_code}" $url)
if [ $status -eq 404 ]; then
failed=$url
echo "Got 404 on $failed. Exiting."
curl -H "Content-Type: application/json" -d "{\"content\": \"${{ secrets.DISCORD_USER_ID }} $url returned 404\"}" ${{ secrets.WEBHOOK_URL }}
fi
done
- name: Download Wikis
run: |
rm -rf old.reddit.com && rm -rf reddit
wget --no-cookies --header "Cookie: over18=1" -E -H -k -p -r -l1 --domains=old.reddit.com https://old.reddit.com/r/TransDIY/wiki https://old.reddit.com/r/TransDIY/wiki/pharmacies https://old.reddit.com/r/TransSurgeriesWiki/wiki/index/ https://old.reddit.com/r/FTM/wiki || true
mv old.reddit.com reddit
cd reddit && rm robots.txt
- name: Commit Changes
if: ${{ success() }}
uses: EndBug/add-and-commit@v9
with:
add: reddit
pull: --rebase --autostash .
message: "[automated] update /r/TransDIY wiki"
default_author: github_actions
.github/workflows/transfemscience.org.yml
+36
View File
@@ -0,0 +1,36 @@
name: transfemscience.org
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * 6"
jobs:
archive:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
token: ${{ secrets.PAT_TOKEN }}
- name: check for 404
run: |
url="https://transfemscience.org"
status=$(curl -o /dev/null -s -w "%{http_code}" $url)
if [ $status -eq 404 ]; then
echo "Got 404. Exiting."
curl -H "Content-Type: application/json" -d '{"content": "${{ secrets.DISCORD_USER_ID }} $URL returned 404"}' ${{ secrets.WEBHOOK_URL }}
exit 1
fi
- name: download transfemscience
run: |
rm -rf transfemscience.org
wget -mkpKEH --limit-rate 10m --random-wait -e robots=off --user-agent="Contact: hrt.boytw.ink/crawl_contact.txt" --domains=transfemscience.org,files.transfemscience.org https://transfemscience.org || true
find . -name "*.orig" -type f -delete
- name: commit changes
if: ${{ success() }}
uses: EndBug/add-and-commit@v9
with:
add: "['files.transfemscience.org', 'transfemscience.org']"
pull: "--rebase --autostash ."
message: "[automated] update transfemscience"
default_author: github_actions
.github/workflows/update_cafe.yml
+32
View File
@@ -0,0 +1,32 @@
name: Update DIYHRT.cafe links
on:
workflow_dispatch:
schedule:
- cron: "30 23 1 * *" # every 1st of month
jobs:
update:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
token: ${{ secrets.PAT_TOKEN }}
- uses: actions/setup-python@v4
with:
python-version: pypy-3.7-v7.3.3
- name: Update links
if: always()
run: |
rm scripts/cafe.txt
pip install beautifulsoup4 bs4
python scripts/getlinks.py 'https://diyhrt.cafe/index.php/Special:AllPages' > scripts/cafe.txt
cd scripts && python formatlinks.py cafe.txt
- name: Commit changes
if: ${{ success() }}
uses: EndBug/add-and-commit@v9
with:
add: "scripts/cafe.txt"
pull: "--rebase --autostash ."
message: "[automated] update diyhrt.cafe list"
default_author: github_actions