Merge pull request #454 from Googulator/source-mirrors
Support multiple mirrors for each source file
This commit is contained in:
commit
29a029947c
|
@ -15,8 +15,19 @@ download_source() {
|
|||
local dest_path="${distfiles}/${fname}"
|
||||
if ! [ -e "${dest_path}" ]; then
|
||||
echo "Downloading ${fname}"
|
||||
curl --fail --location "${url}" --output "${dest_path}"
|
||||
curl --fail --location "${url}" --output "${dest_path}" || true
|
||||
fi
|
||||
}
|
||||
|
||||
check_source() {
|
||||
local distfiles="${1}"
|
||||
local url="${2}"
|
||||
local checksum="${3}"
|
||||
local fname="${4}"
|
||||
# Default to basename of url if not given
|
||||
fname="${fname:-$(basename "${url}")}"
|
||||
|
||||
local dest_path="${distfiles}/${fname}"
|
||||
echo "${checksum} ${dest_path}" | sha256sum -c
|
||||
}
|
||||
|
||||
|
@ -25,6 +36,7 @@ set -e
|
|||
cd "$(dirname "$(readlink -f "$0")")"
|
||||
mkdir -p distfiles
|
||||
|
||||
# First, try to download anything missing - ignore failing mirrors
|
||||
for entry in steps/*; do
|
||||
[ -e "${entry}/sources" ] || continue
|
||||
|
||||
|
@ -35,3 +47,15 @@ for entry in steps/*; do
|
|||
download_source distfiles ${line}
|
||||
done < "${entry}/sources"
|
||||
done
|
||||
|
||||
# Then, check if everything has been obtained at least once
|
||||
for entry in steps/*; do
|
||||
[ -e "${entry}/sources" ] || continue
|
||||
|
||||
# shellcheck disable=SC2162
|
||||
while read line; do
|
||||
# This is intentional - we want to split out ${line} into separate arguments.
|
||||
# shellcheck disable=SC2086
|
||||
check_source distfiles ${line}
|
||||
done < "${entry}/sources"
|
||||
done
|
||||
|
|
|
@ -12,6 +12,7 @@ import hashlib
|
|||
import os
|
||||
import shutil
|
||||
import tarfile
|
||||
import traceback
|
||||
import requests
|
||||
|
||||
# pylint: disable=too-many-instance-attributes
|
||||
|
@ -305,7 +306,12 @@ this script the next time")
|
|||
def get_packages(self):
|
||||
"""Prepare remaining sources"""
|
||||
for line in self.source_manifest:
|
||||
path = self.download_file(line[2], line[1], line[3])
|
||||
try:
|
||||
path = self.download_file(line[2], line[1], line[3])
|
||||
except requests.HTTPError:
|
||||
print(traceback.format_exc())
|
||||
for line in self.source_manifest:
|
||||
path = os.path.join(line[1], line[3])
|
||||
self.check_file(path, line[0])
|
||||
|
||||
@classmethod
|
||||
|
|
|
@ -257,14 +257,25 @@ build() {
|
|||
unset extract
|
||||
}
|
||||
|
||||
interpret_source_line() {
|
||||
download_source_line() {
|
||||
url="${1}"
|
||||
checksum="${2}"
|
||||
fname="${3}"
|
||||
# Default to basename of url if not given
|
||||
fname="${fname:-$(basename "${url}")}"
|
||||
if ! [ -e "${fname}" ]; then
|
||||
curl --fail --retry 5 --location "${url}" --output "${fname}"
|
||||
curl --fail --retry 5 --location "${url}" --output "${fname}" || true
|
||||
fi
|
||||
}
|
||||
|
||||
check_source_line() {
|
||||
url="${1}"
|
||||
checksum="${2}"
|
||||
fname="${3}"
|
||||
# Default to basename of url if not given
|
||||
fname="${fname:-$(basename "${url}")}"
|
||||
if ! [ -e "${fname}" ]; then
|
||||
false
|
||||
fi
|
||||
echo "${checksum} ${fname}" > "${fname}.sum"
|
||||
sha256sum -c "${fname}.sum"
|
||||
|
@ -279,7 +290,13 @@ default_src_get() {
|
|||
while read line; do
|
||||
# This is intentional - we want to split out ${line} into separate arguments.
|
||||
# shellcheck disable=SC2086
|
||||
interpret_source_line ${line}
|
||||
download_source_line ${line}
|
||||
done < "${base_dir}/sources"
|
||||
# shellcheck disable=SC2162
|
||||
while read line; do
|
||||
# This is intentional - we want to split out ${line} into separate arguments.
|
||||
# shellcheck disable=SC2086
|
||||
check_source_line ${line}
|
||||
done < "${base_dir}/sources"
|
||||
cd -
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue