1efc042d92
Update scripts can now declare features using
passthru.updateScript = {
command = [ ../../update.sh pname ];
supportedFeatures = [ "commit" ];
};
A `commit` feature means that when the update script finishes successfully,
it will print a JSON list like the following:
[
{
"attrPath": "volume_key",
"oldVersion": "0.3.11",
"newVersion": "0.3.12",
"files": [
"/path/to/nixpkgs/pkgs/development/libraries/volume-key/default.nix"
]
}
]
and data from that will be used when update.nix is run with --argstr commit true
to create commits.
We will create a new git worktree for each thread in the pool and run the update
script there. Then we will commit the change and cherry pick it in the main repo,
releasing the worktree for a next change.
120 lines
4.8 KiB
Python
120 lines
4.8 KiB
Python
import argparse
|
|
import contextlib
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
|
|
updates = {}
|
|
|
|
thread_name_prefix='UpdateScriptThread'
|
|
|
|
def eprint(*args, **kwargs):
|
|
print(*args, file=sys.stderr, **kwargs)
|
|
|
|
def run_update_script(package, commit):
|
|
if commit and 'commit' in package['supportedFeatures']:
|
|
thread_name = threading.current_thread().name
|
|
worktree, _branch, lock = temp_dirs[thread_name]
|
|
lock.acquire()
|
|
package['thread'] = thread_name
|
|
else:
|
|
worktree = None
|
|
|
|
eprint(f" - {package['name']}: UPDATING ...")
|
|
|
|
return subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, cwd=worktree)
|
|
|
|
@contextlib.contextmanager
|
|
def make_worktree():
|
|
with tempfile.TemporaryDirectory() as wt:
|
|
branch_name = f'update-{os.path.basename(wt)}'
|
|
target_directory = f'{wt}/nixpkgs'
|
|
|
|
subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory], check=True)
|
|
yield (target_directory, branch_name)
|
|
subprocess.run(['git', 'worktree', 'remove', target_directory], check=True)
|
|
subprocess.run(['git', 'branch', '-D', branch_name], check=True)
|
|
|
|
def main(max_workers, keep_going, commit, packages):
|
|
with open(sys.argv[1]) as f:
|
|
packages = json.load(f)
|
|
|
|
eprint()
|
|
eprint('Going to be running update for following packages:')
|
|
for package in packages:
|
|
eprint(f" - {package['name']}")
|
|
eprint()
|
|
|
|
confirm = input('Press Enter key to continue...')
|
|
if confirm == '':
|
|
eprint()
|
|
eprint('Running update for:')
|
|
|
|
with contextlib.ExitStack() as stack, concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix=thread_name_prefix) as executor:
|
|
global temp_dirs
|
|
|
|
if commit:
|
|
temp_dirs = {f'{thread_name_prefix}_{str(i)}': (*stack.enter_context(make_worktree()), threading.Lock()) for i in range(max_workers)}
|
|
|
|
for package in packages:
|
|
updates[executor.submit(run_update_script, package, commit)] = package
|
|
|
|
for future in concurrent.futures.as_completed(updates):
|
|
package = updates[future]
|
|
|
|
try:
|
|
p = future.result()
|
|
if commit and 'commit' in package['supportedFeatures']:
|
|
thread_name = package['thread']
|
|
worktree, branch, lock = temp_dirs[thread_name]
|
|
changes = json.loads(p.stdout)
|
|
for change in changes:
|
|
subprocess.run(['git', 'add'] + change['files'], check=True, cwd=worktree)
|
|
commit_message = '{attrPath}: {oldVersion} → {newVersion}'.format(**change)
|
|
subprocess.run(['git', 'commit', '-m', commit_message], check=True, cwd=worktree)
|
|
subprocess.run(['git', 'cherry-pick', branch], check=True)
|
|
eprint(f" - {package['name']}: DONE.")
|
|
except subprocess.CalledProcessError as e:
|
|
eprint(f" - {package['name']}: ERROR")
|
|
eprint()
|
|
eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
|
|
eprint()
|
|
eprint(e.stdout.decode('utf-8'))
|
|
with open(f"{package['pname']}.log", 'wb') as f:
|
|
f.write(e.stdout)
|
|
eprint()
|
|
eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
|
|
|
|
if not keep_going:
|
|
sys.exit(1)
|
|
finally:
|
|
if commit and 'commit' in package['supportedFeatures']:
|
|
lock.release()
|
|
|
|
eprint()
|
|
eprint('Packages updated!')
|
|
sys.exit()
|
|
else:
|
|
eprint('Aborting!')
|
|
sys.exit(130)
|
|
|
|
parser = argparse.ArgumentParser(description='Update packages')
|
|
parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
|
|
parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
|
|
parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes')
|
|
parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
|
|
|
|
if __name__ == '__main__':
|
|
args = parser.parse_args()
|
|
|
|
try:
|
|
main(args.max_workers, args.keep_going, args.commit, args.packages)
|
|
except (KeyboardInterrupt, SystemExit) as e:
|
|
for update in updates:
|
|
update.cancel()
|
|
|
|
sys.exit(e.code if isinstance(e, SystemExit) else 130)
|