diff options
Diffstat (limited to 'infra/libkookie/nixpkgs/pkgs/common-updater')
3 files changed, 150 insertions, 67 deletions
diff --git a/infra/libkookie/nixpkgs/pkgs/common-updater/scripts/mark-broken b/infra/libkookie/nixpkgs/pkgs/common-updater/scripts/mark-broken index d128d0d458ba..f132c86c5e8b 100755 --- a/infra/libkookie/nixpkgs/pkgs/common-updater/scripts/mark-broken +++ b/infra/libkookie/nixpkgs/pkgs/common-updater/scripts/mark-broken @@ -1,86 +1,106 @@ #!/usr/bin/env bash -set -e + +# This script is meant to be used to mark failing hydra builds as broken in the meta attrs +# To use the script, you should pass the list of failing attrs as arguments to the script. +# +# Example: `cat failing-attrs | xargs ./pkgs/common-updater/scripts/mark-broken` +# +# Generating a list of failing attrs: (this should be improved at a later date) +# - Go to the most recent hydra evaluation with all builds completed +# - Select the "builds still failing" tab +# - Highlight and select all packages, should be prefixed with `nixpkgs.` +# - Use regex and editor foo to leave only the attr names +# - Use the above example command to then execute the script +# +# OTHER NOTES: +# - The `denyFileList` and `denyAttrList` will likely need to be updated slightly +# to align with the conventions used in nixpkgs at execution time +# - Any attrs which failed for any reason will be written to `failed-marks.txt`. +# Those attrs will likely need manual attention as disablement will likely be conditional. scriptName=mark-broken # do not use the .wrapped name -die() { - echo "$scriptName: error: $1" >&2 - exit 1 +failMark() { + local attr=$1 + shift 1 + + echo "$attr: $@" >&2 + echo $attr >> failed-marks.txt } usage() { - echo "Usage: $scriptName <attr> [--new-value=<new-value>]" + echo "Usage: $scriptName <attrs>" } -args=() - -for arg in "$@"; do - case $arg in - --new-value=*) - newValue="${arg#*=}" - ;; - --help) - usage - exit 0 - ;; - --*) - echo "$scriptName: Unknown argument: $arg" - usage - exit 1 - ;; - *) - args["${#args[*]}"]=$arg - ;; - esac -done - -attr=${args[0]} - -if (( "${#args[*]}" < 1 )); then +if (( "${#@}" < 1 )); then echo "$scriptName: Too few arguments" usage exit 1 fi -if (( "${#args[*]}" > 1 )); then - echo "$scriptName: Too many arguments" - usage - exit 1 -fi +# in case we resolve to an auto-generated file, just skip these entries +denyFileList=( + node-packages.nix # node, it will mark all node packages as broken + generic-builder.nix # haskell, it will mark all haskell packages as broken +) -if [ -z $newValue ]; then - newValue="true" -fi +# ignore older versions of parameterized packages sets, these likely need +# to be conditionally disabled +denyAttrList=( + python27Packages + python37Packages + libsForQt512 + linuxPackages_ + rubyPackages_ +) -nixFile=$(nix-instantiate --eval --json -E "with import ./. {}; (builtins.unsafeGetAttrPos \"description\" $attr.meta).file" | jq -r .) -if [[ ! -f "$nixFile" ]]; then - die "Couldn't evaluate 'builtins.unsafeGetAttrPos \"description\" $attr.meta' to locate the .nix file!" -fi +function attemptToMarkBroken() { + local attr=$1 -# Insert broken attribute -sed -i.bak "$nixFile" -r \ - -e "/^\s*broken\s*=.*$/d" \ - -e "s/(\s*)meta\s*=.*\{/&\n\1 broken = $newValue;/" + # skip likely to be noisy attrs + for badAttr in ${denyAttrList[@]};do + if [[ $attr =~ $badAttr ]]; then + failMark $attr "attr contained $badAttr, skipped." + return + fi + done -if cmp -s "$nixFile" "$nixFile.bak"; then - mv "$nixFile.bak" "$nixFile" - die "Failed to mark the package as broken! Does it have a meta attribute?" -fi + nixFile=$(nix-instantiate --eval --json -E "with import ./. {}; (builtins.unsafeGetAttrPos \"description\" $attr.meta).file" 2>/dev/null | jq -r .) + if [[ ! -f "$nixFile" ]]; then + failMark $attr "Couldn't locate correct file" + return + fi -if [[ "$newValue" == "true" ]]; then - # broken should evaluate to true in any case now - markedSuccessfully=$(nix-instantiate --eval -E "with import ./. {}; $attr.meta.broken" || true) - if [[ ! "$markedSuccessfully" == "true" ]]; then - mv "$nixFile.bak" "$nixFile" - die "Couldn't verify the broken attribute to be set correctly, restoring backup!" - fi -else - # we can not check if broken evaluates to the correct value, but we can check that it does evaluate - if ! nix-instantiate --eval -E "with import ./. {}; $attr.meta.broken" >/dev/null; then - mv "$nixFile.bak" "$nixFile" - die "Couldn't verify the broken attribute to be set correctly, restoring backup!" - fi -fi + # skip files which are auto-generated + for filename in ${denyFileList[@]};do + if [[ "$filename" == $(basename $nixFile) ]]; then + failMark $attr "filename matched $filename, skipped." + return + fi + done + + # Insert broken attribute + sed -i.bak "$nixFile" -r \ + -e "/^\s*broken\s*=.*$/d" \ + -e "s/(\s*)meta\s*=.*\{/&\n\1 broken = true;/" + + if cmp -s "$nixFile" "$nixFile.bak"; then + mv "$nixFile.bak" "$nixFile" + failMark $attr "Does it have a meta attribute?" + return + fi -rm -f "$nixFile.bak" -rm -f "$attr.fetchlog" + # broken should evaluate to true in any case now + markedSuccessfully=$(nix-instantiate --eval -E "with import ./. {}; $attr.meta.broken") + if [[ "$markedSuccessfully" != "true" ]]; then + mv "$nixFile.bak" "$nixFile" + failMark $attr "$attr.meta.broken doesn't evaluate to true." + return + fi + + rm -f "$nixFile.bak" +} + +for attr in $@; do + attemptToMarkBroken $attr +done diff --git a/infra/libkookie/nixpkgs/pkgs/common-updater/scripts/update-source-version b/infra/libkookie/nixpkgs/pkgs/common-updater/scripts/update-source-version index ba628fd2a445..181561242fc3 100755 --- a/infra/libkookie/nixpkgs/pkgs/common-updater/scripts/update-source-version +++ b/infra/libkookie/nixpkgs/pkgs/common-updater/scripts/update-source-version @@ -10,7 +10,7 @@ die() { usage() { echo "Usage: $scriptName <attr> <version> [<new-source-hash>] [<new-source-url>]" - echo " [--version-key=<version-key>] [--system=<system>] [--file=<file-to-update>]" + echo " [--version-key=<version-key>] [--system=<system>] [--file=<file-to-update>] [--rev=<revision>]" echo " [--ignore-same-hash] [--print-changes]" } @@ -30,6 +30,9 @@ for arg in "$@"; do die "Could not find provided file $nixFile" fi ;; + --rev=*) + newRevision="${arg#*=}" + ;; --ignore-same-hash) ignoreSameHash="true" ;; @@ -111,6 +114,13 @@ if [[ "$oldVersion" = "$newVersion" ]]; then exit 0 fi +if [[ -n "$newRevision" ]]; then + oldRevision=$(nix-instantiate $systemArg --eval -E "with import ./. {}; $attr.src.rev" | tr -d '"') + if [[ -z "$oldRevision" ]]; then + die "Couldn't evaluate source revision from '$attr.src'!" + fi +fi + # Escape regex metacharacter that are allowed in store path names oldVersionEscaped=$(echo "$oldVersion" | sed -re 's|[.+]|\\&|g') oldUrlEscaped=$(echo "$oldUrl" | sed -re 's|[${}.+]|\\&|g') @@ -174,6 +184,15 @@ if cmp -s "$nixFile" "$nixFile.bak"; then die "Failed to replace source hash of '$attr' to a temporary hash!" fi +# Replace new revision, if given +if [[ -n "$newRevision" ]]; then + sed -i "$nixFile" -re "s|\"$oldRevision\"|\"$newRevision\"|" + + if cmp -s "$nixFile" "$nixFile.bak"; then + die "Failed to replace source revision '$oldRevision' to '$newRevision' in '$attr'!" + fi +fi + # If new hash not given on the command line, recalculate it ourselves. if [[ -z "$newHash" ]]; then nix-build $systemArg --no-out-link -A "$attr.src" 2>"$attr.fetchlog" >/dev/null || true diff --git a/infra/libkookie/nixpkgs/pkgs/common-updater/unstable-updater.nix b/infra/libkookie/nixpkgs/pkgs/common-updater/unstable-updater.nix new file mode 100644 index 000000000000..94cd33b9a26b --- /dev/null +++ b/infra/libkookie/nixpkgs/pkgs/common-updater/unstable-updater.nix @@ -0,0 +1,44 @@ +{ stdenv +, writeShellScript +, coreutils +, git +, nix +, common-updater-scripts +}: + +# This is an updater for unstable packages that should always use the latest +# commit. +{ url ? null # The git url, if empty it will be set to src.url +}: + +let + updateScript = writeShellScript "unstable-update-script.sh" '' + set -ex + + url="$1" + + # By default we set url to src.url + if [[ -z "$url" ]]; then + url="$(${nix}/bin/nix-instantiate $systemArg --eval -E \ + "with import ./. {}; $UPDATE_NIX_ATTR_PATH.src.url" \ + | tr -d '"')" + fi + + # Get info about HEAD from a shallow git clone + tmpdir="$(${coreutils}/bin/mktemp -d)" + ${git}/bin/git clone --bare --depth=1 "$url" "$tmpdir" + pushd "$tmpdir" + commit_date="$(${git}/bin/git show -s --pretty='format:%cs')" + commit_sha="$(${git}/bin/git show -s --pretty='format:%H')" + popd + ${coreutils}/bin/rm -rf "$tmpdir" + + # update the nix expression + ${common-updater-scripts}/bin/update-source-version \ + "$UPDATE_NIX_ATTR_PATH" \ + "unstable-$commit_date" \ + --rev="$commit_sha" + ''; + +in [ updateScript url ] + |