#!/bin/bash

DOMAIN="www.lhasa-apso.org"
START_URL="https://www.lhasa-apso.org/"
OUTDIR="$HOME/lhasa-apso-offline"

mkdir -p "$OUTDIR"
cd "$OUTDIR" || exit 1

touch urls.txt
echo "$START_URL" > urls.txt

PASS=1
MAXPASS=10

while [ $PASS -le $MAXPASS ]
do
    echo "===== PASS $PASS ====="

    cp urls.txt urls_old.txt

    while read -r URL
    do
        echo "Scanning $URL"

        wget \
            --spider \
            --force-html \
            --recursive \
            --level=1 \
            --domains="$DOMAIN" \
            "$URL" \
            2>&1 |
        grep -oE 'https?://[^ ]+' >> discovered.txt

    done < urls_old.txt

    sort -u discovered.txt urls.txt | \
        grep "$DOMAIN" | \
        sort -u > urls_new.txt

    mv urls_new.txt urls.txt

    OLDCOUNT=$(wc -l < urls_old.txt)
    NEWCOUNT=$(wc -l < urls.txt)

    echo "URLs known: $NEWCOUNT"

    if [ "$OLDCOUNT" -eq "$NEWCOUNT" ]; then
        echo "No new URLs found."
        break
    fi

    PASS=$((PASS + 1))
done

echo "===== DOWNLOADING ====="

wget \
    --input-file=urls.txt \
    --mirror \
    --convert-links \
    --adjust-extension \
    --page-requisites \
    --directory-prefix=site

echo "Done."