Support multiple mirrors for each source file
If multiple URLs are entered in a sources listing for the same file, each will be tried in turn, until either one succeeds, or we fail having run out of mirrors.
This commit is contained in:
parent
86e1a5e7f6
commit
89a4d18ff0
|
@ -15,8 +15,19 @@ download_source() {
|
||||||
local dest_path="${distfiles}/${fname}"
|
local dest_path="${distfiles}/${fname}"
|
||||||
if ! [ -e "${dest_path}" ]; then
|
if ! [ -e "${dest_path}" ]; then
|
||||||
echo "Downloading ${fname}"
|
echo "Downloading ${fname}"
|
||||||
curl --fail --location "${url}" --output "${dest_path}"
|
curl --fail --location "${url}" --output "${dest_path}" || true
|
||||||
fi
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_source() {
|
||||||
|
local distfiles="${1}"
|
||||||
|
local url="${2}"
|
||||||
|
local checksum="${3}"
|
||||||
|
local fname="${4}"
|
||||||
|
# Default to basename of url if not given
|
||||||
|
fname="${fname:-$(basename "${url}")}"
|
||||||
|
|
||||||
|
local dest_path="${distfiles}/${fname}"
|
||||||
echo "${checksum} ${dest_path}" | sha256sum -c
|
echo "${checksum} ${dest_path}" | sha256sum -c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -25,6 +36,7 @@ set -e
|
||||||
cd "$(dirname "$(readlink -f "$0")")"
|
cd "$(dirname "$(readlink -f "$0")")"
|
||||||
mkdir -p distfiles
|
mkdir -p distfiles
|
||||||
|
|
||||||
|
# First, try to download anything missing - ignore failing mirrors
|
||||||
for entry in steps/*; do
|
for entry in steps/*; do
|
||||||
[ -e "${entry}/sources" ] || continue
|
[ -e "${entry}/sources" ] || continue
|
||||||
|
|
||||||
|
@ -35,3 +47,15 @@ for entry in steps/*; do
|
||||||
download_source distfiles ${line}
|
download_source distfiles ${line}
|
||||||
done < "${entry}/sources"
|
done < "${entry}/sources"
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# Then, check if everything has been obtained at least once
|
||||||
|
for entry in steps/*; do
|
||||||
|
[ -e "${entry}/sources" ] || continue
|
||||||
|
|
||||||
|
# shellcheck disable=SC2162
|
||||||
|
while read line; do
|
||||||
|
# This is intentional - we want to split out ${line} into separate arguments.
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
check_source distfiles ${line}
|
||||||
|
done < "${entry}/sources"
|
||||||
|
done
|
||||||
|
|
|
@ -12,6 +12,7 @@ import hashlib
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import tarfile
|
import tarfile
|
||||||
|
import traceback
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
# pylint: disable=too-many-instance-attributes
|
# pylint: disable=too-many-instance-attributes
|
||||||
|
@ -305,7 +306,12 @@ this script the next time")
|
||||||
def get_packages(self):
|
def get_packages(self):
|
||||||
"""Prepare remaining sources"""
|
"""Prepare remaining sources"""
|
||||||
for line in self.source_manifest:
|
for line in self.source_manifest:
|
||||||
path = self.download_file(line[2], line[1], line[3])
|
try:
|
||||||
|
path = self.download_file(line[2], line[1], line[3])
|
||||||
|
except requests.HTTPError:
|
||||||
|
print(traceback.format_exc())
|
||||||
|
for line in self.source_manifest:
|
||||||
|
path = os.path.join(line[1], line[3])
|
||||||
self.check_file(path, line[0])
|
self.check_file(path, line[0])
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|
|
@ -257,14 +257,25 @@ build() {
|
||||||
unset extract
|
unset extract
|
||||||
}
|
}
|
||||||
|
|
||||||
interpret_source_line() {
|
download_source_line() {
|
||||||
url="${1}"
|
url="${1}"
|
||||||
checksum="${2}"
|
checksum="${2}"
|
||||||
fname="${3}"
|
fname="${3}"
|
||||||
# Default to basename of url if not given
|
# Default to basename of url if not given
|
||||||
fname="${fname:-$(basename "${url}")}"
|
fname="${fname:-$(basename "${url}")}"
|
||||||
if ! [ -e "${fname}" ]; then
|
if ! [ -e "${fname}" ]; then
|
||||||
curl --fail --retry 5 --location "${url}" --output "${fname}"
|
curl --fail --retry 5 --location "${url}" --output "${fname}" || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_source_line() {
|
||||||
|
url="${1}"
|
||||||
|
checksum="${2}"
|
||||||
|
fname="${3}"
|
||||||
|
# Default to basename of url if not given
|
||||||
|
fname="${fname:-$(basename "${url}")}"
|
||||||
|
if ! [ -e "${fname}" ]; then
|
||||||
|
false
|
||||||
fi
|
fi
|
||||||
echo "${checksum} ${fname}" > "${fname}.sum"
|
echo "${checksum} ${fname}" > "${fname}.sum"
|
||||||
sha256sum -c "${fname}.sum"
|
sha256sum -c "${fname}.sum"
|
||||||
|
@ -279,7 +290,13 @@ default_src_get() {
|
||||||
while read line; do
|
while read line; do
|
||||||
# This is intentional - we want to split out ${line} into separate arguments.
|
# This is intentional - we want to split out ${line} into separate arguments.
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
interpret_source_line ${line}
|
download_source_line ${line}
|
||||||
|
done < "${base_dir}/sources"
|
||||||
|
# shellcheck disable=SC2162
|
||||||
|
while read line; do
|
||||||
|
# This is intentional - we want to split out ${line} into separate arguments.
|
||||||
|
# shellcheck disable=SC2086
|
||||||
|
check_source_line ${line}
|
||||||
done < "${base_dir}/sources"
|
done < "${base_dir}/sources"
|
||||||
cd -
|
cd -
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue