diff --git a/download-distfiles.sh b/download-distfiles.sh index e4c12b9..0287b72 100755 --- a/download-distfiles.sh +++ b/download-distfiles.sh @@ -15,8 +15,19 @@ download_source() { local dest_path="${distfiles}/${fname}" if ! [ -e "${dest_path}" ]; then echo "Downloading ${fname}" - curl --fail --location "${url}" --output "${dest_path}" + curl --fail --location "${url}" --output "${dest_path}" || true fi +} + +check_source() { + local distfiles="${1}" + local url="${2}" + local checksum="${3}" + local fname="${4}" + # Default to basename of url if not given + fname="${fname:-$(basename "${url}")}" + + local dest_path="${distfiles}/${fname}" echo "${checksum} ${dest_path}" | sha256sum -c } @@ -25,6 +36,7 @@ set -e cd "$(dirname "$(readlink -f "$0")")" mkdir -p distfiles +# First, try to download anything missing - ignore failing mirrors for entry in steps/*; do [ -e "${entry}/sources" ] || continue @@ -35,3 +47,15 @@ for entry in steps/*; do download_source distfiles ${line} done < "${entry}/sources" done + +# Then, check if everything has been obtained at least once +for entry in steps/*; do + [ -e "${entry}/sources" ] || continue + + # shellcheck disable=SC2162 + while read line; do + # This is intentional - we want to split out ${line} into separate arguments. + # shellcheck disable=SC2086 + check_source distfiles ${line} + done < "${entry}/sources" +done diff --git a/lib/generator.py b/lib/generator.py index 69f9913..ba16eac 100755 --- a/lib/generator.py +++ b/lib/generator.py @@ -12,6 +12,7 @@ import hashlib import os import shutil import tarfile +import traceback import requests # pylint: disable=too-many-instance-attributes @@ -305,7 +306,12 @@ this script the next time") def get_packages(self): """Prepare remaining sources""" for line in self.source_manifest: - path = self.download_file(line[2], line[1], line[3]) + try: + path = self.download_file(line[2], line[1], line[3]) + except requests.HTTPError: + print(traceback.format_exc()) + for line in self.source_manifest: + path = os.path.join(line[1], line[3]) self.check_file(path, line[0]) @classmethod diff --git a/steps/helpers.sh b/steps/helpers.sh index 9694e66..42fbdb4 100755 --- a/steps/helpers.sh +++ b/steps/helpers.sh @@ -257,14 +257,25 @@ build() { unset extract } -interpret_source_line() { +download_source_line() { url="${1}" checksum="${2}" fname="${3}" # Default to basename of url if not given fname="${fname:-$(basename "${url}")}" if ! [ -e "${fname}" ]; then - curl --fail --retry 5 --location "${url}" --output "${fname}" + curl --fail --retry 5 --location "${url}" --output "${fname}" || true + fi +} + +check_source_line() { + url="${1}" + checksum="${2}" + fname="${3}" + # Default to basename of url if not given + fname="${fname:-$(basename "${url}")}" + if ! [ -e "${fname}" ]; then + false fi echo "${checksum} ${fname}" > "${fname}.sum" sha256sum -c "${fname}.sum" @@ -279,7 +290,13 @@ default_src_get() { while read line; do # This is intentional - we want to split out ${line} into separate arguments. # shellcheck disable=SC2086 - interpret_source_line ${line} + download_source_line ${line} + done < "${base_dir}/sources" + # shellcheck disable=SC2162 + while read line; do + # This is intentional - we want to split out ${line} into separate arguments. + # shellcheck disable=SC2086 + check_source_line ${line} done < "${base_dir}/sources" cd - }