diff --git a/doc/stdenv/stdenv.xml b/doc/stdenv/stdenv.xml index f97c2a145af5..ccf5bb440d36 100644 --- a/doc/stdenv/stdenv.xml +++ b/doc/stdenv/stdenv.xml @@ -475,10 +475,48 @@ passthru.updateScript = writeScript "update-zoom-us" '' passthru.updateScript = [ ../../update.sh pname "--requested-release=unstable" ]; + Finally, the attribute can be an attribute set, listing the extra supported features among other things. + +passthru.updateScript = { + command = [ ../../update.sh pname ]; + supportedFeatures = [ "commit" ]; +}; + + + + The script will be usually run from the root of the Nixpkgs repository but you should not rely on that. Also note that the update scripts will be run in parallel by default; you should avoid running git commit or any other commands that cannot handle that. + + - - The script will be usually run from the root of the Nixpkgs repository but you should not rely on that. Also note that the update scripts will be run in parallel by default; you should avoid running git commit or any other commands that cannot handle that. - + + Supported features + + + commit + + + + Whenever the update script exits with 0 return + status, it is expected to print a JSON list containing an object for + each updated attribute. Empty list can be returned when the script did + not update any files: for example, when the attribute is already the + latest version. The required keys can be seen below: + +[ + { + "attrPath": "volume_key", + "oldVersion": "0.3.11", + "newVersion": "0.3.12", + "files": [ + "/path/to/nixpkgs/pkgs/development/libraries/volume-key/default.nix" + ] + } +] + + + + + For information about how to run the updates, execute nix-shell maintainers/scripts/update.nix. diff --git a/maintainers/scripts/update.nix b/maintainers/scripts/update.nix index 9568c6cbbccd..e11e2450bd0a 100755 --- a/maintainers/scripts/update.nix +++ b/maintainers/scripts/update.nix @@ -4,6 +4,7 @@ , max-workers ? null , include-overlays ? false , keep-going ? null +, commit ? null }: # TODO: add assert statements @@ -132,19 +133,26 @@ let --argstr keep-going true to continue running when a single update fails. + + You can also make the updater automatically commit on your behalf from updateScripts + that support it by adding + + --argstr commit true ''; packageData = package: { name = package.name; pname = lib.getName package; - updateScript = map builtins.toString (lib.toList package.updateScript); + updateScript = map builtins.toString (lib.toList (package.updateScript.command or package.updateScript)); + supportedFeatures = package.updateScript.supportedFeatures or []; }; packagesJson = pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages)); optionalArgs = lib.optional (max-workers != null) "--max-workers=${max-workers}" - ++ lib.optional (keep-going == "true") "--keep-going"; + ++ lib.optional (keep-going == "true") "--keep-going" + ++ lib.optional (commit == "true") "--commit"; args = [ packagesJson ] ++ optionalArgs; diff --git a/maintainers/scripts/update.py b/maintainers/scripts/update.py index b9e17736bed3..b440f9defe43 100644 --- a/maintainers/scripts/update.py +++ b/maintainers/scripts/update.py @@ -1,22 +1,45 @@ import argparse +import contextlib import concurrent.futures import json import os import subprocess import sys +import tempfile +import threading updates = {} +thread_name_prefix='UpdateScriptThread' + def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) -def run_update_script(package): +def run_update_script(package, commit): + if commit and 'commit' in package['supportedFeatures']: + thread_name = threading.current_thread().name + worktree, _branch, lock = temp_dirs[thread_name] + lock.acquire() + package['thread'] = thread_name + else: + worktree = None + eprint(f" - {package['name']}: UPDATING ...") - subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) + return subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, cwd=worktree) +@contextlib.contextmanager +def make_worktree(): + with tempfile.TemporaryDirectory() as wt: + branch_name = f'update-{os.path.basename(wt)}' + target_directory = f'{wt}/nixpkgs' -def main(max_workers, keep_going, packages): + subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory], check=True) + yield (target_directory, branch_name) + subprocess.run(['git', 'worktree', 'remove', target_directory], check=True) + subprocess.run(['git', 'branch', '-D', branch_name], check=True) + +def main(max_workers, keep_going, commit, packages): with open(sys.argv[1]) as f: packages = json.load(f) @@ -31,15 +54,29 @@ def main(max_workers, keep_going, packages): eprint() eprint('Running update for:') - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + with contextlib.ExitStack() as stack, concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix=thread_name_prefix) as executor: + global temp_dirs + + if commit: + temp_dirs = {f'{thread_name_prefix}_{str(i)}': (*stack.enter_context(make_worktree()), threading.Lock()) for i in range(max_workers)} + for package in packages: - updates[executor.submit(run_update_script, package)] = package + updates[executor.submit(run_update_script, package, commit)] = package for future in concurrent.futures.as_completed(updates): package = updates[future] try: - future.result() + p = future.result() + if commit and 'commit' in package['supportedFeatures']: + thread_name = package['thread'] + worktree, branch, lock = temp_dirs[thread_name] + changes = json.loads(p.stdout) + for change in changes: + subprocess.run(['git', 'add'] + change['files'], check=True, cwd=worktree) + commit_message = '{attrPath}: {oldVersion} → {newVersion}'.format(**change) + subprocess.run(['git', 'commit', '-m', commit_message], check=True, cwd=worktree) + subprocess.run(['git', 'cherry-pick', branch], check=True) eprint(f" - {package['name']}: DONE.") except subprocess.CalledProcessError as e: eprint(f" - {package['name']}: ERROR") @@ -54,6 +91,9 @@ def main(max_workers, keep_going, packages): if not keep_going: sys.exit(1) + finally: + if commit and 'commit' in package['supportedFeatures']: + lock.release() eprint() eprint('Packages updated!') @@ -65,13 +105,14 @@ def main(max_workers, keep_going, packages): parser = argparse.ArgumentParser(description='Update packages') parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4) parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure') +parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes') parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts') if __name__ == '__main__': args = parser.parse_args() try: - main(args.max_workers, args.keep_going, args.packages) + main(args.max_workers, args.keep_going, args.commit, args.packages) except (KeyboardInterrupt, SystemExit) as e: for update in updates: update.cancel()