BoomMicrophone/nixpkgs-suyu
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

maintainers/scripts/update.nix: Add support for auto-commiting changes

Browse source 
Update scripts can now declare features using

	passthru.updateScript = {
	  command = [ ../../update.sh pname ];
	  supportedFeatures = [ "commit" ];
	};

A `commit` feature means that when the update script finishes successfully,
it will print a JSON list like the following:

	[
	  {
	    "attrPath": "volume_key",
	    "oldVersion": "0.3.11",
	    "newVersion": "0.3.12",
	    "files": [
	      "/path/to/nixpkgs/pkgs/development/libraries/volume-key/default.nix"
	    ]
	  }
	]

and data from that will be used when update.nix is run with --argstr commit true
to create commits.

We will create a new git worktree for each thread in the pool and run the update
script there. Then we will commit the change and cherry pick it in the main repo,
releasing the worktree for a next change.
This commit is contained in:
Jan Tojnar 2019-04-12 19:32:44 +02:00
parent d351cea9f3
commit 1efc042d92
No known key found for this signature in database
GPG key ID: 7FAB2A15F7A607A4
3 changed files with 99 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
doc/stdenv/stdenv.xml
View file

@ -475,10 +475,48 @@ passthru.updateScript = writeScript "update-zoom-us" ''
<programlisting>
passthru.updateScript = [ ../../update.sh pname "--requested-release=unstable" ];
</programlisting>
Finally, the attribute can be an attribute set, listing the extra supported features among other things.
<programlisting>
passthru.updateScript = {
command = [ ../../update.sh pname ];
supportedFeatures = [ "commit" ];
};
</programlisting>
<note>
<para>
The script will be usually run from the root of the Nixpkgs repository but you should not rely on that. Also note that the update scripts will be run in parallel by default; you should avoid running <command>git commit</command> or any other commands that cannot handle that.
</para>
</note>
</para>
<para>
The script will be usually run from the root of the Nixpkgs repository but you should not rely on that. Also note that the update scripts will be run in parallel by default; you should avoid running <command>git commit</command> or any other commands that cannot handle that.
</para>
<variablelist>
<title>Supported features</title>
<varlistentry>
<term>
<varname>commit</varname>
</term>
<listitem>
<para>
Whenever the update script exits with <literal>0</literal> return
status, it is expected to print a JSON list containing an object for
each updated attribute. Empty list can be returned when the script did
not update any files: for example, when the attribute is already the
latest version. The required keys can be seen below:
<programlisting>
[
{
"attrPath": "volume_key",
"oldVersion": "0.3.11",
"newVersion": "0.3.12",
"files": [
"/path/to/nixpkgs/pkgs/development/libraries/volume-key/default.nix"
]
}
]
</programlisting>
</para>
</listitem>
</varlistentry>
</variablelist>
<para>
For information about how to run the updates, execute <command>nix-shell maintainers/scripts/update.nix</command>.
</para>

12
maintainers/scripts/update.nix
View file

@ -4,6 +4,7 @@
, max-workers ? null
, include-overlays ? false
, keep-going ? null
, commit ? null
}:
# TODO: add assert statements
@ -132,19 +133,26 @@ let
--argstr keep-going true
to continue running when a single update fails.
You can also make the updater automatically commit on your behalf from updateScripts
that support it by adding
--argstr commit true
'';
packageData = package: {
name = package.name;
pname = lib.getName package;
updateScript = map builtins.toString (lib.toList package.updateScript);
updateScript = map builtins.toString (lib.toList (package.updateScript.command or package.updateScript));
supportedFeatures = package.updateScript.supportedFeatures or [];
};
packagesJson = pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages));
optionalArgs =
lib.optional (max-workers != null) "--max-workers=${max-workers}"
++ lib.optional (keep-going == "true") "--keep-going";
++ lib.optional (keep-going == "true") "--keep-going"
++ lib.optional (commit == "true") "--commit";
args = [ packagesJson ] ++ optionalArgs;

55
maintainers/scripts/update.py
View file

@ -1,22 +1,45 @@
import argparse
import contextlib
import concurrent.futures
import json
import os
import subprocess
import sys
import tempfile
import threading
updates = {}
thread_name_prefix='UpdateScriptThread'
def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
def run_update_script(package):
def run_update_script(package, commit):
if commit and 'commit' in package['supportedFeatures']:
thread_name = threading.current_thread().name
worktree, _branch, lock = temp_dirs[thread_name]
lock.acquire()
package['thread'] = thread_name
else:
worktree = None
eprint(f" - {package['name']}: UPDATING ...")
subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
return subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, cwd=worktree)
@contextlib.contextmanager
def make_worktree():
with tempfile.TemporaryDirectory() as wt:
branch_name = f'update-{os.path.basename(wt)}'
target_directory = f'{wt}/nixpkgs'
def main(max_workers, keep_going, packages):
subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory], check=True)
yield (target_directory, branch_name)
subprocess.run(['git', 'worktree', 'remove', target_directory], check=True)
subprocess.run(['git', 'branch', '-D', branch_name], check=True)
def main(max_workers, keep_going, commit, packages):
with open(sys.argv[1]) as f:
packages = json.load(f)
@ -31,15 +54,29 @@ def main(max_workers, keep_going, packages):
eprint()
eprint('Running update for:')
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
with contextlib.ExitStack() as stack, concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix=thread_name_prefix) as executor:
global temp_dirs
if commit:
temp_dirs = {f'{thread_name_prefix}_{str(i)}': (*stack.enter_context(make_worktree()), threading.Lock()) for i in range(max_workers)}
for package in packages:
updates[executor.submit(run_update_script, package)] = package
updates[executor.submit(run_update_script, package, commit)] = package
for future in concurrent.futures.as_completed(updates):
package = updates[future]
try:
future.result()
p = future.result()
if commit and 'commit' in package['supportedFeatures']:
thread_name = package['thread']
worktree, branch, lock = temp_dirs[thread_name]
changes = json.loads(p.stdout)
for change in changes:
subprocess.run(['git', 'add'] + change['files'], check=True, cwd=worktree)
commit_message = '{attrPath}: {oldVersion} → {newVersion}'.format(**change)
subprocess.run(['git', 'commit', '-m', commit_message], check=True, cwd=worktree)
subprocess.run(['git', 'cherry-pick', branch], check=True)
eprint(f" - {package['name']}: DONE.")
except subprocess.CalledProcessError as e:
eprint(f" - {package['name']}: ERROR")
@ -54,6 +91,9 @@ def main(max_workers, keep_going, packages):
if not keep_going:
sys.exit(1)
finally:
if commit and 'commit' in package['supportedFeatures']:
lock.release()
eprint()
eprint('Packages updated!')
@ -65,13 +105,14 @@ def main(max_workers, keep_going, packages):
parser = argparse.ArgumentParser(description='Update packages')
parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes')
parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
if __name__ == '__main__':
args = parser.parse_args()
try:
main(args.max_workers, args.keep_going, args.packages)
main(args.max_workers, args.keep_going, args.commit, args.packages)
except (KeyboardInterrupt, SystemExit) as e:
for update in updates:
update.cancel()