Merge branch 'vmtools-windows-vm'.

This adds a new function in vmTools, called runInWindowsVM, which allows
to run a derivation within a Windows + Cygwin environment.

To use it, you need to pass a Windows ISO and product key, for example:

------------------------------------------------------
vmTools.runInWindowsVM (stdenv.mkDerivation {
  name = "hello-from-windows";

  windowsImage = {
    isoFile = /path/to/windows/image.iso;
    productKey = "ABCDE-FGHIJ-KLMNO-PQRST-UVWXY";
  };

  buildCommand = ''
    echo 'Look, I am running inside Windoze!'
    uname -a > "$out"
  '';
})
------------------------------------------------------

The derivation is then run within a special build process, which roughly
does something like this:

                ____________
               |            |
               | controller |
               |____________|
              /      |       \
  _________ /    ____|____     \___________      _______
 |         |    |         |    |           |    |       |
 | install | -> | suspend | -> | suspended | -> | build |
 |_________|    |_________|    |___________|    |_______|

There are three steps necessary to produce the builder, which in the end
is just a suspended Windows VM, running Cygwin and OpenSSH.

Those steps are essentially:

 * install: Install the base Windows VM with Cygwin and OpenSSH.
 * suspend: Run the installed VM and dump the memory into a state file.
 * suspended: Resume from the state file and execute the build.

Every build is based on the "suspended" step, which throws away all
changes except to the resulting output store path(s).

All of these steps are based on the controller, which is described in
greater detail in commit 276b72fb93.

The reason I'm merging this right in is because it actually adds a
feature that doesn't break existing functionality and only hooks into
vmTools with a single line.

To the contrary it even duplicates a bit of the code from vmTools, which
might be a good start for refactoring. I didn't do that within that
branch, because it otherwise *could* break existing functionality - VM
tests in particular.

Also, this implementation currently *only* supports Windows XP, because
the implementation was originally made for building a Software where the
majority of the users are using Windows XP and we need to do extensive
testing on that platform.

However, adding support for more recent versions is rather trivial. All
there needs to be done is adding a new unattended installation config in
unattended-image.nix.
This commit is contained in:
aszlig 2014-02-26 06:24:56 +01:00
commit b5de8156cb
No known key found for this signature in database
GPG key ID: D0EBD0EC8C2DC961
8 changed files with 679 additions and 2 deletions

View file

@ -1714,5 +1714,4 @@ rec {
};
};
}
} // import ./windows pkgs

View file

@ -0,0 +1,80 @@
{ stdenv, fetchurl, vmTools, writeScript, writeText, runCommand, makeInitrd
, python, perl, coreutils, dosfstools, gzip, mtools, netcat, openssh, qemu
, samba, socat, vde2, cdrkit, pathsFromGraph
}:
{ isoFile, productKey }:
with stdenv.lib;
let
controller = import ./controller {
inherit stdenv writeScript vmTools makeInitrd;
inherit samba vde2 openssh socat netcat coreutils gzip;
};
mkCygwinImage = import ./cygwin-iso {
inherit stdenv fetchurl runCommand python perl cdrkit pathsFromGraph;
};
installer = import ./install {
inherit controller mkCygwinImage;
inherit stdenv runCommand openssh qemu writeText dosfstools mtools;
};
in rec {
installedVM = installer {
inherit isoFile productKey;
};
runInVM = img: attrs: controller (attrs // {
inherit (installedVM) sshKey;
qemuArgs = attrs.qemuArgs or [] ++ [
"-boot order=c"
"-drive file=${img},index=0,media=disk"
];
});
runAndSuspend = let
drives = {
s = {
source = "nixstore";
target = "/nix/store";
};
x = {
source = "xchg";
target = "/tmp/xchg";
};
};
genDriveCmds = letter: { source, target }: [
"net use ${letter}: '\\\\192.168.0.2\\${source}' /persistent:yes"
"mkdir -p '${target}'"
"mount -o bind '/cygdrive/${letter}' '${target}'"
"echo '/cygdrive/${letter} ${target} none bind 0 0' >> /etc/fstab"
];
in runInVM "winvm.img" {
command = concatStringsSep " && " ([
"net config server /autodisconnect:-1"
] ++ concatLists (mapAttrsToList genDriveCmds drives));
suspendTo = "state.gz";
};
suspendedVM = stdenv.mkDerivation {
name = "cygwin-suspended-vm";
buildCommand = ''
${qemu}/bin/qemu-img create \
-b "${installedVM}/disk.img" \
-f qcow2 winvm.img
${runAndSuspend}
ensureDir "$out"
cp winvm.img "$out/disk.img"
cp state.gz "$out/state.gz"
'';
};
resumeAndRun = command: runInVM "${suspendedVM}/disk.img" {
resumeFrom = "${suspendedVM}/state.gz";
qemuArgs = singleton "-snapshot";
inherit command;
};
}

View file

@ -0,0 +1,229 @@
{ stdenv, writeScript, vmTools, makeInitrd
, samba, vde2, openssh, socat, netcat, coreutils, gzip
}:
{ sshKey
, qemuArgs ? []
, command ? "sync"
, suspendTo ? null
, resumeFrom ? null
, installMode ? false
}:
with stdenv.lib;
let
preInitScript = writeScript "preinit.sh" ''
#!${vmTools.initrdUtils}/bin/ash -e
export PATH=${vmTools.initrdUtils}/bin
mount -t proc none /proc
mount -t sysfs none /sys
for arg in $(cat /proc/cmdline); do
if [ "x''${arg#command=}" != "x$arg" ]; then
command="''${arg#command=}"
fi
done
for i in $(cat ${modulesClosure}/insmod-list); do
insmod $i
done
mkdir -p /dev /fs
mount -t tmpfs none /dev
mknod /dev/null c 1 3
mknod /dev/zero c 1 5
mknod /dev/random c 1 8
mknod /dev/urandom c 1 9
mknod /dev/tty c 5 0
ifconfig lo up
ifconfig eth0 up 192.168.0.2
mount -t tmpfs none /fs
mkdir -p /fs/nix/store /fs/xchg /fs/dev /fs/sys /fs/proc /fs/etc /fs/tmp
mount -o bind /dev /fs/dev
mount -t sysfs none /fs/sys
mount -t proc none /fs/proc
mount -t 9p \
-o trans=virtio,version=9p2000.L,msize=262144,cache=loose \
store /fs/nix/store
mount -t 9p \
-o trans=virtio,version=9p2000.L,msize=262144,cache=loose \
xchg /fs/xchg
echo root:x:0:0::/root:/bin/false > /fs/etc/passwd
set +e
chroot /fs $command $out
echo $? > /fs/xchg/in-vm-exit
poweroff -f
'';
initrd = makeInitrd {
contents = singleton {
object = preInitScript;
symlink = "/init";
};
};
shellEscape = x: "'${replaceChars ["'"] [("'\\'" + "'")] x}'";
loopForever = "while :; do ${coreutils}/bin/sleep 1; done";
initScript = writeScript "init.sh" (''
#!${stdenv.shell}
${coreutils}/bin/cp -L "${sshKey}" /ssh.key
${coreutils}/bin/chmod 600 /ssh.key
'' + (if installMode then ''
echo -n "Waiting for Windows installation to finish..."
while ! ${netcat}/bin/netcat -z 192.168.0.1 22; do
echo -n .
# Print a dot every 10 seconds only to shorten line length.
${coreutils}/bin/sleep 10
done
echo " success."
# Loop forever, because this VM is going to be killed.
${loopForever}
'' else ''
${coreutils}/bin/mkdir -p /etc/samba /etc/samba/private \
/var/lib/samba /var/log /var/run
${coreutils}/bin/cat > /etc/samba/smb.conf <<CONFIG
[global]
security = user
map to guest = Bad User
guest account = root
workgroup = cygwin
netbios name = controller
server string = %h
log level = 1
max log size = 1000
log file = /var/log/samba.log
[nixstore]
path = /nix/store
writable = yes
guest ok = yes
[xchg]
path = /xchg
writable = yes
guest ok = yes
CONFIG
${samba}/sbin/nmbd -D
${samba}/sbin/smbd -D
echo -n "Waiting for Windows VM to become available..."
while ! ${netcat}/bin/netcat -z 192.168.0.1 22; do
echo -n .
${coreutils}/bin/sleep 1
done
echo " success."
${openssh}/bin/ssh \
-o UserKnownHostsFile=/dev/null \
-o StrictHostKeyChecking=no \
-i /ssh.key \
-l Administrator \
192.168.0.1 -- ${shellEscape command}
'') + optionalString (suspendTo != null) ''
${coreutils}/bin/touch /xchg/suspend_now
${loopForever}
'');
kernelAppend = concatStringsSep " " [
"panic=1"
"loglevel=4"
"console=tty1"
"console=ttyS0"
"command=${initScript}"
];
controllerQemuArgs = concatStringsSep " " (maybeKvm64 ++ [
"-nographic"
"-no-reboot"
"-virtfs local,path=/nix/store,security_model=none,mount_tag=store"
"-virtfs local,path=$XCHG_DIR,security_model=none,mount_tag=xchg"
"-kernel ${modulesClosure.kernel}/bzImage"
"-initrd ${initrd}/initrd"
"-append \"${kernelAppend}\""
"-net nic,vlan=0,macaddr=52:54:00:12:01:02,model=virtio"
"-net vde,vlan=0,sock=$QEMU_VDE_SOCKET"
]);
maybeKvm64 = optional (stdenv.system == "x86_64-linux") "-cpu kvm64";
cygwinQemuArgs = concatStringsSep " " (maybeKvm64 ++ [
"-monitor unix:$MONITOR_SOCKET,server,nowait"
"-nographic"
"-net nic,vlan=0,macaddr=52:54:00:12:01:01"
"-net vde,vlan=0,sock=$QEMU_VDE_SOCKET"
"-rtc base=2010-01-01,clock=vm"
] ++ qemuArgs ++ optionals (resumeFrom != null) [
"-incoming 'exec: ${gzip}/bin/gzip -c -d \"${resumeFrom}\"'"
]);
modulesClosure = overrideDerivation vmTools.modulesClosure (o: {
rootModules = o.rootModules ++ singleton "virtio_net";
});
preVM = ''
(set; declare -p) > saved-env
XCHG_DIR="$(${coreutils}/bin/mktemp -d nix-vm.XXXXXXXXXX --tmpdir)"
${coreutils}/bin/mv saved-env "$XCHG_DIR/"
eval "$preVM"
QEMU_VDE_SOCKET="$(pwd)/vde.ctl"
MONITOR_SOCKET="$(pwd)/monitor"
${vde2}/bin/vde_switch -s "$QEMU_VDE_SOCKET" &
echo 'alive?' | ${socat}/bin/socat - \
UNIX-CONNECT:$QEMU_VDE_SOCKET/ctl,retry=20
'';
bgBoth = optionalString (suspendTo != null) " &";
vmExec = if installMode then ''
${vmTools.qemuProg} ${controllerQemuArgs} &
${vmTools.qemuProg} ${cygwinQemuArgs}${bgBoth}
'' else ''
${vmTools.qemuProg} ${cygwinQemuArgs} &
${vmTools.qemuProg} ${controllerQemuArgs}${bgBoth}
'';
postVM = if suspendTo != null then ''
while ! test -e "$XCHG_DIR/suspend_now"; do sleep 1; done
${socat}/bin/socat - UNIX-CONNECT:$MONITOR_SOCKET <<CMD
stop
migrate_set_speed 4095m
migrate "exec:${gzip}/bin/gzip -c > '${suspendTo}'"
quit
CMD
wait %-
eval "$postVM"
exit 0
'' else if installMode then ''
eval "$postVM"
exit 0
'' else ''
if ! test -e "$XCHG_DIR/in-vm-exit"; then
echo "Virtual machine didn't produce an exit code."
exit 1
fi
eval "$postVM"
exit $(< "$XCHG_DIR/in-vm-exit")
'';
in writeScript "run-cygwin-vm.sh" ''
#!${stdenv.shell} -e
${preVM}
${vmExec}
${postVM}
''

View file

@ -0,0 +1,46 @@
{ stdenv, fetchurl, runCommand, python, perl, cdrkit, pathsFromGraph }:
{ packages ? []
, mirror ? "http://ftp.gwdg.de/pub/linux/sources.redhat.com/cygwin"
, extraContents ? []
}:
let
cygPkgList = if stdenv.is64bit then fetchurl {
url = "${mirror}/x86_64/setup.ini";
sha256 = "142f8zyfwgi6s2djxv3z5wn0ysl94pxwa79z8rjfqz4kvnpgz120";
} else fetchurl {
url = "${mirror}/x86/setup.ini";
sha256 = "1v596lln2iip5h7wxjnig5rflzvqa21zzd2iyhx07zs28q5h76i9";
};
makeCygwinClosure = { packages, packageList }: let
expr = import (runCommand "cygwin.nix" { buildInputs = [ python ]; } ''
python ${./mkclosure.py} "${packages}" ${toString packageList} > "$out"
'');
gen = { url, md5 }: {
source = fetchurl {
url = "${mirror}/${url}";
inherit md5;
};
target = url;
};
in map gen expr;
in import <nixpkgs/nixos/lib/make-iso9660-image.nix> {
inherit (import <nixpkgs> {}) stdenv perl cdrkit pathsFromGraph;
contents = [
{ source = fetchurl {
url = "http://cygwin.com/setup-x86_64.exe";
sha256 = "1bjmq9h1p6mmiqp6f1kvmg94jbsdi1pxfa07a5l497zzv9dsfivm";
};
target = "setup.exe";
}
{ source = cygPkgList;
target = "setup.ini";
}
] ++ makeCygwinClosure {
packages = cygPkgList;
packageList = packages;
} ++ extraContents;
}

View file

@ -0,0 +1,78 @@
# Ugliest Python code I've ever written. -- aszlig
import sys
def get_plist(path):
in_pack = False
in_str = False
current_key = None
buf = ""
packages = {}
package_name = None
package_attrs = {}
with open(path, 'r') as setup:
for line in setup:
if in_str and line.rstrip().endswith('"'):
package_attrs[current_key] = buf + line.rstrip()[:-1]
in_str = False
continue
elif in_str:
buf += line
continue
if line.startswith('@'):
in_pack = True
package_name = line[1:].strip()
package_attrs = {}
elif in_pack and ':' in line:
key, value = line.split(':', 1)
if value.lstrip().startswith('"'):
if value.lstrip()[1:].rstrip().endswith('"'):
value = value.strip().strip('"')
else:
in_str = True
current_key = key.strip().lower()
buf = value.lstrip()[1:]
continue
package_attrs[key.strip().lower()] = value.strip()
elif in_pack:
in_pack = False
packages[package_name] = package_attrs
return packages
def main():
packages = get_plist(sys.argv[1])
to_include = set()
def traverse(package):
to_include.add(package)
attrs = packages.get(package, {})
deps = attrs.get('requires', '').split()
for new_dep in set(deps) - to_include:
traverse(new_dep)
map(traverse, sys.argv[2:])
sys.stdout.write('[\n')
for package, attrs in packages.iteritems():
if package not in to_include:
cats = [c.lower() for c in attrs.get('category', '').split()]
if 'base' not in cats:
continue
install_line = attrs.get('install')
if install_line is None:
continue
url, size, md5 = install_line.split(' ', 2)
pack = [
' {',
' url = "{0}";'.format(url),
' md5 = "{0}";'.format(md5),
' }',
];
sys.stdout.write('\n'.join(pack) + '\n')
sys.stdout.write(']\n')
if __name__ == '__main__':
main()

View file

@ -0,0 +1,48 @@
pkgs:
let
bootstrapper = import ./bootstrap.nix {
inherit (pkgs) stdenv vmTools writeScript writeText runCommand makeInitrd;
inherit (pkgs) coreutils dosfstools gzip mtools netcat openssh qemu samba;
inherit (pkgs) socat vde2 fetchurl python perl cdrkit pathsFromGraph;
};
builder = ''
source /tmp/xchg/saved-env 2> /dev/null || true
export NIX_STORE=/nix/store
export NIX_BUILD_TOP=/tmp
export TMPDIR=/tmp
export PATH=/empty
cd "$NIX_BUILD_TOP"
exec $origBuilder $origArgs
'';
in {
runInWindowsVM = drv: let
newDrv = drv.override {
stdenv = drv.stdenv.override {
shell = "/bin/sh";
};
};
in pkgs.lib.overrideDerivation drv (attrs: let
bootstrap = bootstrapper attrs.windowsImage;
in {
requiredSystemFeatures = [ "kvm" ];
buildur = "${pkgs.stdenv.shell}";
args = ["-e" (bootstrap.resumeAndRun builder)];
windowsImage = bootstrap.suspendedVM;
origArgs = attrs.args;
origBuilder = if attrs.builder == attrs.stdenv.shell
then "/bin/sh"
else attrs.builder;
postHook = ''
PATH=/usr/bin:/bin:/usr/sbin:/sbin
SHELL=/bin/sh
eval "$origPostHook"
'';
origPostHook = attrs.postHook or "";
fixupPhase = ":";
});
}

View file

@ -0,0 +1,74 @@
{ stdenv, runCommand, openssh, qemu, controller, mkCygwinImage
, writeText, dosfstools, mtools
}:
{ isoFile
, productKey
}:
let
bootstrapAfterLogin = runCommand "bootstrap.sh" {} ''
cat > "$out" <<EOF
mkdir -p ~/.ssh
cat > ~/.ssh/authorized_keys <<PUBKEY
$(cat "${cygwinSshKey}/key.pub")
PUBKEY
ssh-host-config -y -c 'binmode ntsec' -w dummy
cygrunsrv -S sshd
shutdown -s 5
EOF
'';
cygwinSshKey = stdenv.mkDerivation {
name = "snakeoil-ssh-cygwin";
buildCommand = ''
ensureDir "$out"
${openssh}/bin/ssh-keygen -t ecdsa -f "$out/key" -N ""
'';
};
sshKey = "${cygwinSshKey}/key";
packages = [ "openssh" "shutdown" ];
floppyCreator = import ./unattended-image.nix {
inherit stdenv writeText dosfstools mtools;
};
instfloppy = floppyCreator {
cygwinPackages = packages;
inherit productKey;
};
cygiso = mkCygwinImage {
inherit packages;
extraContents = stdenv.lib.singleton {
source = bootstrapAfterLogin;
target = "bootstrap.sh";
};
};
installController = controller {
inherit sshKey;
installMode = true;
qemuArgs = [
"-boot order=c,once=d"
"-drive file=${instfloppy},readonly,index=0,if=floppy"
"-drive file=winvm.img,index=0,media=disk"
"-drive file=${isoFile},index=1,media=cdrom"
"-drive file=${cygiso}/iso/cd.iso,index=2,media=cdrom"
];
};
in stdenv.mkDerivation {
name = "cygwin-base-vm";
buildCommand = ''
${qemu}/bin/qemu-img create -f qcow2 winvm.img 2G
${installController}
ensureDir "$out"
cp winvm.img "$out/disk.img"
'';
passthru = {
inherit sshKey;
};
}

View file

@ -0,0 +1,123 @@
{ stdenv, writeText, dosfstools, mtools }:
{ productKey
, shExecAfterwards ? "E:\\bootstrap.sh"
, cygwinRoot ? "C:\\cygwin"
, cygwinSetup ? "E:\\setup.exe"
, cygwinRepository ? "E:\\"
, cygwinPackages ? [ "openssh" ]
}:
let
afterSetup = [
cygwinSetup
"-L -n -q"
"-l ${cygwinRepository}"
"-R ${cygwinRoot}"
"-C base"
] ++ map (p: "-P ${p}") cygwinPackages;
winXpUnattended = writeText "winnt.sif" ''
[Data]
AutoPartition = 1
AutomaticUpdates = 0
MsDosInitiated = 0
UnattendedInstall = Yes
[Unattended]
DUDisable = Yes
DriverSigningPolicy = Ignore
Hibernation = No
OemPreinstall = No
OemSkipEula = Yes
Repartition = Yes
TargetPath = \WINDOWS
UnattendMode = FullUnattended
UnattendSwitch = Yes
WaitForReboot = No
[GuiUnattended]
AdminPassword = "nopasswd"
AutoLogon = Yes
AutoLogonCount = 1
OEMSkipRegional = 1
OemSkipWelcome = 1
ServerWelcome = No
TimeZone = 85
[UserData]
ComputerName = "cygwin"
FullName = "cygwin"
OrgName = ""
ProductKey = "${productKey}"
[Networking]
InstallDefaultComponents = Yes
[Identification]
JoinWorkgroup = cygwin
[NetAdapters]
PrimaryAdapter = params.PrimaryAdapter
[params.PrimaryAdapter]
InfID = *
[params.MS_MSClient]
[NetProtocols]
MS_TCPIP = params.MS_TCPIP
[params.MS_TCPIP]
AdapterSections=params.MS_TCPIP.PrimaryAdapter
[params.MS_TCPIP.PrimaryAdapter]
DHCP = No
IPAddress = 192.168.0.1
SpecificTo = PrimaryAdapter
SubnetMask = 255.255.255.0
WINS = No
; Turn off all components
[Components]
${stdenv.lib.concatMapStrings (comp: "${comp} = Off\n") [
"AccessOpt" "Appsrv_console" "Aspnet" "BitsServerExtensionsISAPI"
"BitsServerExtensionsManager" "Calc" "Certsrv" "Certsrv_client"
"Certsrv_server" "Charmap" "Chat" "Clipbook" "Cluster" "Complusnetwork"
"Deskpaper" "Dialer" "Dtcnetwork" "Fax" "Fp_extensions" "Fp_vdir_deploy"
"Freecell" "Hearts" "Hypertrm" "IEAccess" "IEHardenAdmin" "IEHardenUser"
"Iis_asp" "Iis_common" "Iis_ftp" "Iis_inetmgr" "Iis_internetdataconnector"
"Iis_nntp" "Iis_serversideincludes" "Iis_smtp" "Iis_webdav" "Iis_www"
"Indexsrv_system" "Inetprint" "Licenseserver" "Media_clips" "Media_utopia"
"Minesweeper" "Mousepoint" "Msmq_ADIntegrated" "Msmq_Core"
"Msmq_HTTPSupport" "Msmq_LocalStorage" "Msmq_MQDSService"
"Msmq_RoutingSupport" "Msmq_TriggersService" "Msnexplr" "Mswordpad"
"Netcis" "Netoc" "OEAccess" "Objectpkg" "Paint" "Pinball" "Pop3Admin"
"Pop3Service" "Pop3Srv" "Rec" "Reminst" "Rootautoupdate" "Rstorage" "SCW"
"Sakit_web" "Solitaire" "Spider" "TSWebClient" "Templates"
"TerminalServer" "UDDIAdmin" "UDDIDatabase" "UDDIWeb" "Vol" "WMAccess"
"WMPOCM" "WbemMSI" "Wms" "Wms_admin_asp" "Wms_admin_mmc" "Wms_isapi"
"Wms_server" "Zonegames"
]}
[WindowsFirewall]
Profiles = WindowsFirewall.TurnOffFirewall
[WindowsFirewall.TurnOffFirewall]
Mode = 0
[SetupParams]
UserExecute = "${stdenv.lib.concatStringsSep " " afterSetup}"
[GuiRunOnce]
Command0 = "${cygwinRoot}\bin\bash -l ${shExecAfterwards}"
'';
in stdenv.mkDerivation {
name = "unattended-floppy.img";
buildCommand = ''
dd if=/dev/zero of="$out" count=1440 bs=1024
${dosfstools}/sbin/mkfs.msdos "$out"
${mtools}/bin/mcopy -i "$out" "${winXpUnattended}" ::winnt.sif
'';
}