Alyssa Smith revised this gist . Go to revision
1 file changed, 23 insertions, 9 deletions
digitalocean-torrentdl.sh
| @@ -3,6 +3,15 @@ | |||
| 3 | 3 | # Prints estimated USD cost of transaction when complete | |
| 4 | 4 | # uses your user's rclone.conf, so set up the remote locally and make sure it works | |
| 5 | 5 | # torrentdl.sh https://releases.ubuntu.com/20.04/ubuntu-20.04.1-live-server-amd64.iso.torrent b2:my-linux-isos/ubuntu/ | |
| 6 | + | ||
| 7 | + | # Default values | |
| 8 | + | IMAGE="ubuntu-20-04-x64" | |
| 9 | + | REGION="sgp1" # overridden by ping check | |
| 10 | + | HOSTNAME="torrent-dl" | |
| 11 | + | URLREGEX='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' | |
| 12 | + | TORRENT="$1" | |
| 13 | + | DESTREMOTE="$2" | |
| 14 | + | ||
| 6 | 15 | echo "Checking dependencies (wget, installed doctl, configured rclone)" | |
| 7 | 16 | [ -s "$HOME/.config/rclone/rclone.conf" ] || (echo "Set up rclone (from rclone.org, not from package manager) with a remote, or write to ~/.config/rclone/rclone.conf, then run this script again"; exit) | |
| 8 | 17 | ||
| @@ -28,12 +37,16 @@ else | |||
| 28 | 37 | doctl auth init | |
| 29 | 38 | fi | |
| 30 | 39 | ||
| 31 | - | IMAGE="ubuntu-20-04-x64" | |
| 32 | - | REGION="sgp1" | |
| 33 | - | HOSTNAME="torrent-dl" | |
| 34 | - | URLREGEX='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' | |
| 35 | - | TORRENT="$1" | |
| 36 | - | DESTREMOTE="$2" | |
| 40 | + | if [ -f $HOME/.config/doctl/closest-dc ]; then | |
| 41 | + | REGION=`cat $HOME/.config/doctl/closest-dc` | |
| 42 | + | else | |
| 43 | + | echo "Finding your closest datacenter..." | |
| 44 | + | REGION=`doctl compute region list --no-header | grep -v false | awk '{print $1}' | \ | |
| 45 | + | while read p; do ping -c4 speedtest-$p.digitalocean.com | grep --color=never rtt | awk -v p=" $p" -F'/' '{print $5 p}'; done | \ | |
| 46 | + | sort | head -n1 | awk '{print $2}'` | |
| 47 | + | echo $REGION > $HOME/.config/doctl/closest-dc | |
| 48 | + | fi | |
| 49 | + | ||
| 37 | 50 | if [[ $TORRENT =~ $URLREGEX ]]; then | |
| 38 | 51 | if ! type wget >/dev/null 2>&1; then | |
| 39 | 52 | echo wget not found for downloading torrent file from url, installing from apt using sudo. | |
| @@ -56,13 +69,13 @@ else # up to 300GB | |||
| 56 | 69 | SIZE="s-8vcpu-16gb" | |
| 57 | 70 | fi | |
| 58 | 71 | ||
| 59 | - | if [ -v $1 ]; then | |
| 72 | + | if [ -v "$1" ]; then | |
| 60 | 73 | echo Usage: $0 https://releases.ubuntu.com/20.04/ubuntu-20.04.1-live-server-amd64.iso.torrent b2:my-linux-isos/ubuntu/ | |
| 61 | 74 | exit | |
| 62 | 75 | fi | |
| 63 | 76 | ||
| 64 | 77 | echo "Downloading '$TORRENT' on $REGION:$IMAGE:$SIZE with hostname $HOSTNAME, then uploading to '$DESTREMOTE'" | |
| 65 | - | read -p "Press enter to continue, ctrl-c to abort..." | |
| 78 | + | #read -p "Press enter to continue, ctrl-c to abort..." | |
| 66 | 79 | ||
| 67 | 80 | echo "Started at `date`" | |
| 68 | 81 | export STARTTIME=`date +%s` | |
| @@ -84,7 +97,8 @@ scp ~/.config/rclone/rclone.conf root@$IP:/root/.config/rclone/rclone.conf | |||
| 84 | 97 | scp "$TORRENT" root@$IP:/root/t/ | |
| 85 | 98 | ||
| 86 | 99 | echo Installing dependencies... | |
| 87 | - | ssh -t root@$IP "apt-get update; apt-get install -y aria2; curl https://rclone.org/install.sh | sudo bash" | |
| 100 | + | ssh -t root@$IP "rm /var/lib/man-db/auto-update; until (apt-get update; apt-get install -y aria2); do sleep 1; done; curl https://rclone.org/install.sh | sudo bash" | |
| 101 | + | #ssh -t root@$IP "apt-get update; apt-get install -y aria2; curl https://rclone.org/install.sh | sudo bash" | |
| 88 | 102 | echo Downloading torrent... | |
| 89 | 103 | ssh -t root@$IP aria2c -d "/root/t/" --enable-dht=false --enable-peer-exchange=false --seed-time=0 "/root/t/*.torrent" | |
| 90 | 104 | echo Uploading torrent to rclone remote... | |
Alyssa Smith revised this gist . Go to revision
1 file changed, 30 insertions, 1 deletion
digitalocean-torrentdl.sh
| @@ -3,14 +3,43 @@ | |||
| 3 | 3 | # Prints estimated USD cost of transaction when complete | |
| 4 | 4 | # uses your user's rclone.conf, so set up the remote locally and make sure it works | |
| 5 | 5 | # torrentdl.sh https://releases.ubuntu.com/20.04/ubuntu-20.04.1-live-server-amd64.iso.torrent b2:my-linux-isos/ubuntu/ | |
| 6 | + | echo "Checking dependencies (wget, installed doctl, configured rclone)" | |
| 7 | + | [ -s "$HOME/.config/rclone/rclone.conf" ] || (echo "Set up rclone (from rclone.org, not from package manager) with a remote, or write to ~/.config/rclone/rclone.conf, then run this script again"; exit) | |
| 8 | + | ||
| 9 | + | if type doctl >/dev/null 2>&1; then | |
| 10 | + | if ! doctl compute droplet list >/dev/null 2>&1; then | |
| 11 | + | echo doctl not authenticated, running doctl auth init | |
| 12 | + | doctl auth init | |
| 13 | + | fi | |
| 14 | + | else | |
| 15 | + | echo 'doctl not found, attempting to install with "go get github.com/digitalocean/doctl/cmd/doctl"' | |
| 16 | + | if ! go version >/dev/null 2>&1; then | |
| 17 | + | echo golang-go not found, installing gvm and go1.15.6 to ~/.gvm in 5 seconds. Ctrl-C to abort | |
| 18 | + | sleep 5 | |
| 19 | + | bash < <(curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/master/binscripts/gvm-installer) | |
| 20 | + | source ~/.gvm/scripts/gvm | |
| 21 | + | gvm install go1.4 -B | |
| 22 | + | gvm use go1.4 | |
| 23 | + | export GOROOT_BOOTSTRAP=$GOROOT | |
| 24 | + | gvm install go1.15.6 | |
| 25 | + | gvm use --default go1.15.6 | |
| 26 | + | fi | |
| 27 | + | go get github.com/digitalocean/doctl/cmd/doctl | |
| 28 | + | doctl auth init | |
| 29 | + | fi | |
| 30 | + | ||
| 6 | 31 | IMAGE="ubuntu-20-04-x64" | |
| 7 | 32 | REGION="sgp1" | |
| 8 | - | SIZE="s-4vcpu-8gb" | |
| 9 | 33 | HOSTNAME="torrent-dl" | |
| 10 | 34 | URLREGEX='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' | |
| 11 | 35 | TORRENT="$1" | |
| 12 | 36 | DESTREMOTE="$2" | |
| 13 | 37 | if [[ $TORRENT =~ $URLREGEX ]]; then | |
| 38 | + | if ! type wget >/dev/null 2>&1; then | |
| 39 | + | echo wget not found for downloading torrent file from url, installing from apt using sudo. | |
| 40 | + | echo "If this isn't right for your environment, install wget before running script again" | |
| 41 | + | sudo apt-get install -y wget || exit | |
| 42 | + | fi | |
| 14 | 43 | echo Downloading torrent from url... | |
| 15 | 44 | wget "$TORRENT" -c | |
| 16 | 45 | export TORRENT=`basename "$TORRENT"` | |
Alyssa Smith revised this gist . Go to revision
1 file changed, 24 insertions, 9 deletions
digitalocean-torrentdl.sh
| @@ -3,20 +3,35 @@ | |||
| 3 | 3 | # Prints estimated USD cost of transaction when complete | |
| 4 | 4 | # uses your user's rclone.conf, so set up the remote locally and make sure it works | |
| 5 | 5 | # torrentdl.sh https://releases.ubuntu.com/20.04/ubuntu-20.04.1-live-server-amd64.iso.torrent b2:my-linux-isos/ubuntu/ | |
| 6 | - | export IMAGE="ubuntu-20-04-x64" | |
| 7 | - | export REGION="sgp1" | |
| 8 | - | export SIZE="s-4vcpu-8gb" | |
| 9 | - | export HOSTNAME="torrent-dl" | |
| 10 | - | export TORRENT="$1" | |
| 11 | - | export DESTREMOTE="$2" | |
| 12 | - | export URLREGEX='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' | |
| 13 | - | ||
| 6 | + | IMAGE="ubuntu-20-04-x64" | |
| 7 | + | REGION="sgp1" | |
| 8 | + | SIZE="s-4vcpu-8gb" | |
| 9 | + | HOSTNAME="torrent-dl" | |
| 10 | + | URLREGEX='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' | |
| 11 | + | TORRENT="$1" | |
| 12 | + | DESTREMOTE="$2" | |
| 14 | 13 | if [[ $TORRENT =~ $URLREGEX ]]; then | |
| 15 | 14 | echo Downloading torrent from url... | |
| 16 | - | wget "$TORRENT" | |
| 15 | + | wget "$TORRENT" -c | |
| 17 | 16 | export TORRENT=`basename "$TORRENT"` | |
| 18 | 17 | fi | |
| 19 | 18 | ||
| 19 | + | TORRENTSIZE=`head -1 "$TORRENT" | grep -aoE '6:lengthi[0-9]+' | cut -di -f2 | awk '{s+=$1} END {print s}'` | |
| 20 | + | if (($TORRENTSIZE<=30000000000)); then # 30GB | |
| 21 | + | SIZE="s-1vcpu-2gb" | |
| 22 | + | elif (($TORRENTSIZE<=60000000000)); then # 60GB | |
| 23 | + | SIZE="s-2vcpu-4gb" | |
| 24 | + | elif (($TORRENTSIZE<=140000000000)); then # 140GB | |
| 25 | + | SIZE="s-4vcpu-8gb" | |
| 26 | + | else # up to 300GB | |
| 27 | + | SIZE="s-8vcpu-16gb" | |
| 28 | + | fi | |
| 29 | + | ||
| 30 | + | if [ -v $1 ]; then | |
| 31 | + | echo Usage: $0 https://releases.ubuntu.com/20.04/ubuntu-20.04.1-live-server-amd64.iso.torrent b2:my-linux-isos/ubuntu/ | |
| 32 | + | exit | |
| 33 | + | fi | |
| 34 | + | ||
| 20 | 35 | echo "Downloading '$TORRENT' on $REGION:$IMAGE:$SIZE with hostname $HOSTNAME, then uploading to '$DESTREMOTE'" | |
| 21 | 36 | read -p "Press enter to continue, ctrl-c to abort..." | |
| 22 | 37 | ||
Alyssa Smith revised this gist . Go to revision
1 file changed, 20 insertions, 8 deletions
digitalocean-torrentdl.sh
| @@ -2,40 +2,52 @@ | |||
| 2 | 2 | # Provisions a droplet on DigitalOcean to download a given torrent file, then reupload to a given rclone remote | |
| 3 | 3 | # Prints estimated USD cost of transaction when complete | |
| 4 | 4 | # uses your user's rclone.conf, so set up the remote locally and make sure it works | |
| 5 | - | # Torrent filename should be a local file, not a URL | |
| 6 | - | # torrentdl.sh ubuntu-20.04-server-amd64.iso.torrent b2:linux-isos/ubuntu/ | |
| 5 | + | # torrentdl.sh https://releases.ubuntu.com/20.04/ubuntu-20.04.1-live-server-amd64.iso.torrent b2:my-linux-isos/ubuntu/ | |
| 7 | 6 | export IMAGE="ubuntu-20-04-x64" | |
| 8 | 7 | export REGION="sgp1" | |
| 9 | 8 | export SIZE="s-4vcpu-8gb" | |
| 10 | 9 | export HOSTNAME="torrent-dl" | |
| 11 | 10 | export TORRENT="$1" | |
| 12 | 11 | export DESTREMOTE="$2" | |
| 12 | + | export URLREGEX='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]$' | |
| 13 | 13 | ||
| 14 | - | echo "Downloading '$1' on $REGION:$IMAGE:$SIZE with hostname $HOSTNAME, then uploading to '$2'" | |
| 14 | + | if [[ $TORRENT =~ $URLREGEX ]]; then | |
| 15 | + | echo Downloading torrent from url... | |
| 16 | + | wget "$TORRENT" | |
| 17 | + | export TORRENT=`basename "$TORRENT"` | |
| 18 | + | fi | |
| 19 | + | ||
| 20 | + | echo "Downloading '$TORRENT' on $REGION:$IMAGE:$SIZE with hostname $HOSTNAME, then uploading to '$DESTREMOTE'" | |
| 15 | 21 | read -p "Press enter to continue, ctrl-c to abort..." | |
| 22 | + | ||
| 16 | 23 | echo "Started at `date`" | |
| 17 | 24 | export STARTTIME=`date +%s` | |
| 25 | + | ||
| 18 | 26 | echo "Provisioning droplet..." | |
| 19 | 27 | export SSHKEYS=`doctl compute ssh-key list --no-header --format ID | tr '\n' ',' | sed 's/,$/\n/'` | |
| 20 | 28 | export IP=$( set -x; doctl compute droplet create $HOSTNAME --ssh-keys $SSHKEYS --region $REGION --size $SIZE --image $IMAGE --wait --no-header --format PublicIPv4 ) | |
| 29 | + | ||
| 21 | 30 | echo -n "Droplet at $IP, waiting for connection" | |
| 22 | 31 | until (ssh -o StrictHostKeyChecking=no root@$IP mkdir -p /root/.config/rclone /root/t 2>/dev/null >/dev/null); do | |
| 23 | 32 | echo -n "." | |
| 24 | 33 | sleep 1 | |
| 25 | 34 | done | |
| 26 | - | ssh-keyscan -H $IP >> ~/.ssh/known_hosts | |
| 35 | + | ssh-keyscan -H $IP 2>/dev/null >> ~/.ssh/known_hosts | |
| 27 | 36 | echo | |
| 28 | - | echo Installing dependencies on droplet... | |
| 29 | - | ssh -t root@$IP "apt-get update; apt-get install -y aria2; curl https://rclone.org/install.sh | sudo bash" | |
| 37 | + | ||
| 30 | 38 | echo Copying rclone conf and torrent to droplet... | |
| 31 | - | scp ~/.config/rclone/rclone.conf root@$IP:/root/.config/rclone/rclone.conf 2>/dev/null >/dev/null | |
| 32 | - | scp "$1" root@$IP:/root/t/ 2>/dev/null >/dev/null | |
| 39 | + | scp ~/.config/rclone/rclone.conf root@$IP:/root/.config/rclone/rclone.conf | |
| 40 | + | scp "$TORRENT" root@$IP:/root/t/ | |
| 41 | + | ||
| 42 | + | echo Installing dependencies... | |
| 43 | + | ssh -t root@$IP "apt-get update; apt-get install -y aria2; curl https://rclone.org/install.sh | sudo bash" | |
| 33 | 44 | echo Downloading torrent... | |
| 34 | 45 | ssh -t root@$IP aria2c -d "/root/t/" --enable-dht=false --enable-peer-exchange=false --seed-time=0 "/root/t/*.torrent" | |
| 35 | 46 | echo Uploading torrent to rclone remote... | |
| 36 | 47 | ssh -t root@$IP rclone copy /root/t/ "$DESTREMOTE" -P --fast-list --transfers=$(($LINES-5)) | |
| 37 | 48 | echo Job complete, deleting droplet | |
| 38 | 49 | $(set -x; doctl compute droplet delete -f $HOSTNAME) | |
| 50 | + | ||
| 39 | 51 | echo "Finished at `date`" | |
| 40 | 52 | export ENDTIME=`date +%s` | |
| 41 | 53 | echo "Total run time: $(($ENDTIME-$STARTTIME))" | |
Alyssa Smith revised this gist . Go to revision
1 file changed, 1 insertion, 1 deletion
digitalocean-torrentdl.sh
| @@ -26,7 +26,7 @@ done | |||
| 26 | 26 | ssh-keyscan -H $IP >> ~/.ssh/known_hosts | |
| 27 | 27 | echo | |
| 28 | 28 | echo Installing dependencies on droplet... | |
| 29 | - | ssh -t root@$IP "apt-get update 2>&1 >/dev/null && apt-get install -y aria2; curl https://rclone.org/install.sh | sudo bash" | |
| 29 | + | ssh -t root@$IP "apt-get update; apt-get install -y aria2; curl https://rclone.org/install.sh | sudo bash" | |
| 30 | 30 | echo Copying rclone conf and torrent to droplet... | |
| 31 | 31 | scp ~/.config/rclone/rclone.conf root@$IP:/root/.config/rclone/rclone.conf 2>/dev/null >/dev/null | |
| 32 | 32 | scp "$1" root@$IP:/root/t/ 2>/dev/null >/dev/null | |
Alyssa Smith revised this gist . Go to revision
1 file changed, 43 insertions
digitalocean-torrentdl.sh(file created)
| @@ -0,0 +1,43 @@ | |||
| 1 | + | #!/bin/bash | |
| 2 | + | # Provisions a droplet on DigitalOcean to download a given torrent file, then reupload to a given rclone remote | |
| 3 | + | # Prints estimated USD cost of transaction when complete | |
| 4 | + | # uses your user's rclone.conf, so set up the remote locally and make sure it works | |
| 5 | + | # Torrent filename should be a local file, not a URL | |
| 6 | + | # torrentdl.sh ubuntu-20.04-server-amd64.iso.torrent b2:linux-isos/ubuntu/ | |
| 7 | + | export IMAGE="ubuntu-20-04-x64" | |
| 8 | + | export REGION="sgp1" | |
| 9 | + | export SIZE="s-4vcpu-8gb" | |
| 10 | + | export HOSTNAME="torrent-dl" | |
| 11 | + | export TORRENT="$1" | |
| 12 | + | export DESTREMOTE="$2" | |
| 13 | + | ||
| 14 | + | echo "Downloading '$1' on $REGION:$IMAGE:$SIZE with hostname $HOSTNAME, then uploading to '$2'" | |
| 15 | + | read -p "Press enter to continue, ctrl-c to abort..." | |
| 16 | + | echo "Started at `date`" | |
| 17 | + | export STARTTIME=`date +%s` | |
| 18 | + | echo "Provisioning droplet..." | |
| 19 | + | export SSHKEYS=`doctl compute ssh-key list --no-header --format ID | tr '\n' ',' | sed 's/,$/\n/'` | |
| 20 | + | export IP=$( set -x; doctl compute droplet create $HOSTNAME --ssh-keys $SSHKEYS --region $REGION --size $SIZE --image $IMAGE --wait --no-header --format PublicIPv4 ) | |
| 21 | + | echo -n "Droplet at $IP, waiting for connection" | |
| 22 | + | until (ssh -o StrictHostKeyChecking=no root@$IP mkdir -p /root/.config/rclone /root/t 2>/dev/null >/dev/null); do | |
| 23 | + | echo -n "." | |
| 24 | + | sleep 1 | |
| 25 | + | done | |
| 26 | + | ssh-keyscan -H $IP >> ~/.ssh/known_hosts | |
| 27 | + | echo | |
| 28 | + | echo Installing dependencies on droplet... | |
| 29 | + | ssh -t root@$IP "apt-get update 2>&1 >/dev/null && apt-get install -y aria2; curl https://rclone.org/install.sh | sudo bash" | |
| 30 | + | echo Copying rclone conf and torrent to droplet... | |
| 31 | + | scp ~/.config/rclone/rclone.conf root@$IP:/root/.config/rclone/rclone.conf 2>/dev/null >/dev/null | |
| 32 | + | scp "$1" root@$IP:/root/t/ 2>/dev/null >/dev/null | |
| 33 | + | echo Downloading torrent... | |
| 34 | + | ssh -t root@$IP aria2c -d "/root/t/" --enable-dht=false --enable-peer-exchange=false --seed-time=0 "/root/t/*.torrent" | |
| 35 | + | echo Uploading torrent to rclone remote... | |
| 36 | + | ssh -t root@$IP rclone copy /root/t/ "$DESTREMOTE" -P --fast-list --transfers=$(($LINES-5)) | |
| 37 | + | echo Job complete, deleting droplet | |
| 38 | + | $(set -x; doctl compute droplet delete -f $HOSTNAME) | |
| 39 | + | echo "Finished at `date`" | |
| 40 | + | export ENDTIME=`date +%s` | |
| 41 | + | echo "Total run time: $(($ENDTIME-$STARTTIME))" | |
| 42 | + | echo -n "Total (estimated) cost: USD $" | |
| 43 | + | doctl compute size list | grep $SIZE | awk '{print $5}' | while read p; do python3 -c "print(($p/2592000)*($ENDTIME-$STARTTIME))"; done | |
Newer
Older