Merge pull request #176835 from pennae/syncserver

nixos/doc/manual/from_md/release-notes/rl-2211.section.xml

@@ -124,6 +124,13 @@
           <link xlink:href="options.html#opt-virtualisation.appvm.enable">virtualisation.appvm</link>.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://github.com/mozilla-services/syncstorage-rs">syncstorage-rs</link>,
+          a self-hostable sync server for Firefox. Available as
+          <link xlink:href="options.html#opt-services.firefox-syncserver.enable">services.firefox-syncserver</link>.
+        </para>
+      </listitem>
       <listitem>
         <para>
           <link xlink:href="https://dragonflydb.io/">dragonflydb</link>,

nixos/doc/manual/release-notes/rl-2211.section.md

@@ -55,6 +55,7 @@ In addition to numerous new and upgraded packages, this release has the followin
 ## New Services {#sec-release-22.11-new-services}
 
 - [appvm](https://github.com/jollheef/appvm), Nix based app VMs. Available as [virtualisation.appvm](options.html#opt-virtualisation.appvm.enable).
+- [syncstorage-rs](https://github.com/mozilla-services/syncstorage-rs), a self-hostable sync server for Firefox. Available as [services.firefox-syncserver](options.html#opt-services.firefox-syncserver.enable).
 
 - [dragonflydb](https://dragonflydb.io/), a modern replacement for Redis and Memcached. Available as [services.dragonflydb](#opt-services.dragonflydb.enable).
 

nixos/modules/module-list.nix

@@ -783,6 +783,7 @@
   ./services/networking/expressvpn.nix
   ./services/networking/fakeroute.nix
   ./services/networking/ferm.nix
+  ./services/networking/firefox-syncserver.nix
   ./services/networking/fireqos.nix
   ./services/networking/firewall.nix
   ./services/networking/flannel.nix

nixos/modules/services/networking/firefox-syncserver.md

@@ -0,0 +1,55 @@
+# Firefox Sync server {#module-services-firefox-syncserver}
+
+A storage server for Firefox Sync that you can easily host yourself.
+
+## Quickstart {#module-services-firefox-syncserver-quickstart}
+
+The absolute minimal configuration for the sync server looks like this:
+
+```nix
+services.mysql.package = pkgs.mariadb;
+
+services.firefox-syncserver = {
+  enable = true;
+  secrets = builtins.toFile "sync-secrets" ''
+    SYNC_MASTER_SECRET=this-secret-is-actually-leaked-to-/nix/store
+  '';
+  singleNode = {
+    enable = true;
+    hostname = "localhost";
+    url = "http://localhost:5000";
+  };
+};
+```
+
+This will start a sync server that is only accessible locally. Once the services is
+running you can navigate to `about:config` in your Firefox profile and set
+`identity.sync.tokenserver.uri` to `http://localhost:5000/1.0/sync/1.5`. Your browser
+will now use your local sync server for data storage.
+
+::: {.warning}
+This configuration should never be used in production. It is not encrypted and
+stores its secrets in a world-readable location.
+:::
+
+## More detailed setup {#module-services-firefox-syncserver-configuration}
+
+The `firefox-syncserver` service provides a number of options to make setting up
+small deployment easier. These are grouped under the `singleNode` element of the
+option tree and allow simple configuration of the most important parameters.
+
+Single node setup is split into two kinds of options: those that affect the sync
+server itself, and those that affect its surroundings. Options that affect the
+sync server are `capacity`, which configures how many accounts may be active on
+this instance, and `url`, which holds the URL under which the sync server can be
+accessed. The `url` can be configured automatically when using nginx.
+
+Options that affect the surroundings of the sync server are `enableNginx`,
+`enableTLS` and `hostnam`. If `enableNginx` is set the sync server module will
+automatically add an nginx virtual host to the system using `hostname` as the
+domain and set `url` accordingly. If `enableTLS` is set the module will also
+enable ACME certificates on the new virtual host and force all connections to
+be made via TLS.
+
+For actual deployment it is also recommended to store the `secrets` file in a
+secure location.

nixos/modules/services/networking/firefox-syncserver.nix

@@ -0,0 +1,328 @@
+{ config, pkgs, lib, options, ... }:
+
+let
+  cfg = config.services.firefox-syncserver;
+  opt = options.services.firefox-syncserver;
+  defaultDatabase = "firefox_syncserver";
+  defaultUser = "firefox-syncserver";
+
+  dbIsLocal = cfg.database.host == "localhost";
+  dbURL = "mysql://${cfg.database.user}@${cfg.database.host}/${cfg.database.name}";
+
+  format = pkgs.formats.toml {};
+  settings = {
+    database_url = dbURL;
+    human_logs = true;
+    tokenserver = {
+      node_type = "mysql";
+      database_url = dbURL;
+      fxa_email_domain = "api.accounts.firefox.com";
+      fxa_oauth_server_url = "https://oauth.accounts.firefox.com/v1";
+      run_migrations = true;
+    } // lib.optionalAttrs cfg.singleNode.enable {
+      # Single-node mode is likely to be used on small instances with little
+      # capacity. The default value (0.1) can only ever release capacity when
+      # accounts are removed if the total capacity is 10 or larger to begin
+      # with.
+      # https://github.com/mozilla-services/syncstorage-rs/issues/1313#issuecomment-1145293375
+      node_capacity_release_rate = 1;
+    };
+  };
+  configFile = format.generate "syncstorage.toml" (lib.recursiveUpdate settings cfg.settings);
+in
+
+{
+  options = {
+    services.firefox-syncserver = {
+      enable = lib.mkEnableOption ''
+        the Firefox Sync storage service.
+
+        Out of the box this will not be very useful unless you also configure at least
+        one service and one nodes by inserting them into the mysql database manually, e.g.
+        by running
+
+        <programlisting>
+          INSERT INTO `services` (`id`, `service`, `pattern`) VALUES ('1', 'sync-1.5', '{node}/1.5/{uid}');
+          INSERT INTO `nodes` (`id`, `service`, `node`, `available`, `current_load`,
+              `capacity`, `downed`, `backoff`)
+            VALUES ('1', '1', 'https://mydomain.tld', '1', '0', '10', '0', '0');
+        </programlisting>
+
+        <option>${opt.singleNode.enable}</option> does this automatically when enabled
+      '';
+
+      package = lib.mkOption {
+        type = lib.types.package;
+        default = pkgs.syncstorage-rs;
+        defaultText = lib.literalExpression "pkgs.syncstorage-rs";
+        description = ''
+          Package to use.
+        '';
+      };
+
+      database.name = lib.mkOption {
+        # the mysql module does not allow `-quoting without resorting to shell
+        # escaping, so we restrict db names for forward compaitiblity should this
+        # behavior ever change.
+        type = lib.types.strMatching "[a-z_][a-z0-9_]*";
+        default = defaultDatabase;
+        description = ''
+          Database to use for storage. Will be created automatically if it does not exist
+          and <literal>config.${opt.database.createLocally}</literal> is set.
+        '';
+      };
+
+      database.user = lib.mkOption {
+        type = lib.types.str;
+        default = defaultUser;
+        description = ''
+          Username for database connections.
+        '';
+      };
+
+      database.host = lib.mkOption {
+        type = lib.types.str;
+        default = "localhost";
+        description = ''
+          Database host name. <literal>localhost</literal> is treated specially and inserts
+          systemd dependencies, other hostnames or IP addresses of the local machine do not.
+        '';
+      };
+
+      database.createLocally = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Whether to create database and user on the local machine if they do not exist.
+          This includes enabling unix domain socket authentication for the configured user.
+        '';
+      };
+
+      logLevel = lib.mkOption {
+        type = lib.types.str;
+        default = "error";
+        description = ''
+          Log level to run with. This can be a simple log level like <literal>error</literal>
+          or <literal>trace</literal>, or a more complicated logging expression.
+        '';
+      };
+
+      secrets = lib.mkOption {
+        type = lib.types.path;
+        description = ''
+          A file containing the various secrets. Should be in the format expected by systemd's
+          <literal>EnvironmentFile</literal> directory. Two secrets are currently available:
+          <literal>SYNC_MASTER_SECRET</literal> and
+          <literal>SYNC_TOKENSERVER__FXA_METRICS_HASH_SECRET</literal>.
+        '';
+      };
+
+      singleNode = {
+        enable = lib.mkEnableOption "auto-configuration for a simple single-node setup";
+
+        enableTLS = lib.mkEnableOption "automatic TLS setup";
+
+        enableNginx = lib.mkEnableOption "nginx virtualhost definitions";
+
+        hostname = lib.mkOption {
+          type = lib.types.str;
+          description = ''
+            Host name to use for this service.
+          '';
+        };
+
+        capacity = lib.mkOption {
+          type = lib.types.ints.unsigned;
+          default = 10;
+          description = ''
+            How many sync accounts are allowed on this server. Setting this value
+            equal to or less than the number of currently active accounts will
+            effectively deny service to accounts not yet registered here.
+          '';
+        };
+
+        url = lib.mkOption {
+          type = lib.types.str;
+          default = "${if cfg.singleNode.enableTLS then "https" else "http"}://${cfg.singleNode.hostname}";
+          defaultText = lib.literalExpression ''
+            ''${if cfg.singleNode.enableTLS then "https" else "http"}://''${config.${opt.singleNode.hostname}}
+          '';
+          description = ''
+            URL of the host. If you are not using the automatic webserver proxy setup you will have
+            to change this setting or your sync server may not be functional.
+          '';
+        };
+      };
+
+      settings = lib.mkOption {
+        type = lib.types.submodule {
+          freeformType = format.type;
+
+          options = {
+            port = lib.mkOption {
+              type = lib.types.port;
+              default = 5000;
+              description = ''
+                Port to bind to.
+              '';
+            };
+
+            tokenserver.enabled = lib.mkOption {
+              type = lib.types.bool;
+              default = true;
+              description = ''
+                Whether to enable the token service as well.
+              '';
+            };
+          };
+        };
+        default = { };
+        description = ''
+          Settings for the sync server. These take priority over values computed
+          from NixOS options.
+
+          See the doc comments on the <literal>Settings</literal> structs in
+          <link xlink:href="https://github.com/mozilla-services/syncstorage-rs/blob/master/syncstorage/src/settings.rs" />
+          and
+          <link xlink:href="https://github.com/mozilla-services/syncstorage-rs/blob/master/syncstorage/src/tokenserver/settings.rs" />
+          for available options.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.mysql = lib.mkIf cfg.database.createLocally {
+      enable = true;
+      ensureDatabases = [ cfg.database.name ];
+      ensureUsers = [{
+        name = cfg.database.user;
+        ensurePermissions = {
+          "${cfg.database.name}.*" = "all privileges";
+        };
+      }];
+    };
+
+    systemd.services.firefox-syncserver = {
+      wantedBy = [ "multi-user.target" ];
+      requires = lib.mkIf dbIsLocal [ "mysql.service" ];
+      after = lib.mkIf dbIsLocal [ "mysql.service" ];
+      environment.RUST_LOG = cfg.logLevel;
+      serviceConfig = {
+        User = defaultUser;
+        Group = defaultUser;
+        ExecStart = "${cfg.package}/bin/syncstorage --config ${configFile}";
+        Stderr = "journal";
+        EnvironmentFile = lib.mkIf (cfg.secrets != null) "${cfg.secrets}";
+
+        # hardening
+        RemoveIPC = true;
+        CapabilityBoundingSet = [ "" ];
+        DynamicUser = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        ProtectClock = true;
+        ProtectKernelLogs = true;
+        ProtectControlGroups = true;
+        ProtectKernelModules = true;
+        SystemCallArchitectures = "native";
+        # syncstorage-rs uses python-cffi internally, and python-cffi does not
+        # work with MemoryDenyWriteExecute=true
+        MemoryDenyWriteExecute = false;
+        RestrictNamespaces = true;
+        RestrictSUIDSGID = true;
+        ProtectHostname = true;
+        LockPersonality = true;
+        ProtectKernelTunables = true;
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ];
+        RestrictRealtime = true;
+        ProtectSystem = "strict";
+        ProtectProc = "invisible";
+        ProcSubset = "pid";
+        ProtectHome = true;
+        PrivateUsers = true;
+        PrivateTmp = true;
+        SystemCallFilter = [ "@system-service" "~ @privileged @resources" ];
+        UMask = "0077";
+      };
+    };
+
+    systemd.services.firefox-syncserver-setup = lib.mkIf cfg.singleNode.enable {
+      wantedBy = [ "firefox-syncserver.service" ];
+      requires = [ "firefox-syncserver.service" ] ++ lib.optional dbIsLocal "mysql.service";
+      after = [ "firefox-syncserver.service" ] ++ lib.optional dbIsLocal "mysql.service";
+      path = [ config.services.mysql.package ];
+      script = ''
+        set -euo pipefail
+        shopt -s inherit_errexit
+
+        schema_configured() {
+          mysql ${cfg.database.name} -Ne 'SHOW TABLES' | grep -q services
+        }
+
+        services_configured() {
+          [ 1 != $(mysql ${cfg.database.name} -Ne 'SELECT COUNT(*) < 1 FROM `services`') ]
+        }
+
+        create_services() {
+          mysql ${cfg.database.name} <<"EOF"
+            BEGIN;
+
+            INSERT INTO `services` (`id`, `service`, `pattern`)
+              VALUES (1, 'sync-1.5', '{node}/1.5/{uid}');
+            INSERT INTO `nodes` (`id`, `service`, `node`, `available`, `current_load`,
+                                 `capacity`, `downed`, `backoff`)
+              VALUES (1, 1, '${cfg.singleNode.url}', ${toString cfg.singleNode.capacity},
+                      0, ${toString cfg.singleNode.capacity}, 0, 0);
+
+            COMMIT;
+        EOF
+        }
+
+        update_nodes() {
+          mysql ${cfg.database.name} <<"EOF"
+            UPDATE `nodes`
+              SET `capacity` = ${toString cfg.singleNode.capacity}
+              WHERE `id` = 1;
+        EOF
+        }
+
+        for (( try = 0; try < 60; try++ )); do
+          if ! schema_configured; then
+            sleep 2
+          elif services_configured; then
+            update_nodes
+            exit 0
+          else
+            create_services
+            exit 0
+          fi
+        done
+
+        echo "Single-node setup failed"
+        exit 1
+      '';
+    };
+
+    services.nginx.virtualHosts = lib.mkIf cfg.singleNode.enableNginx {
+      ${cfg.singleNode.hostname} = {
+        enableACME = cfg.singleNode.enableTLS;
+        forceSSL = cfg.singleNode.enableTLS;
+        locations."/" = {
+          proxyPass = "http://localhost:${toString cfg.settings.port}";
+          # source mentions that this header should be set
+          extraConfig = ''
+            add_header X-Content-Type-Options nosniff;
+          '';
+        };
+      };
+    };
+  };
+
+  meta = {
+    maintainers = with lib.maintainers; [ pennae ];
+    # Don't edit the docbook xml directly, edit the md and generate it:
+    # `pandoc firefox-syncserver.md -t docbook --top-level-division=chapter --extract-media=media -f markdown+smart > firefox-syncserver.xml`
+    doc = ./firefox-syncserver.xml;
+  };
+}

nixos/modules/services/networking/firefox-syncserver.xml

@@ -0,0 +1,77 @@
+<chapter xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xml:id="module-services-firefox-syncserver">
+  <title>Firefox Sync server</title>
+  <para>
+    A storage server for Firefox Sync that you can easily host yourself.
+  </para>
+  <section xml:id="module-services-firefox-syncserver-quickstart">
+    <title>Quickstart</title>
+    <para>
+      The absolute minimal configuration for the sync server looks like
+      this:
+    </para>
+    <programlisting language="nix">
+services.mysql.package = pkgs.mariadb;
+
+services.firefox-syncserver = {
+  enable = true;
+  secrets = builtins.toFile &quot;sync-secrets&quot; ''
+    SYNC_MASTER_SECRET=this-secret-is-actually-leaked-to-/nix/store
+  '';
+  singleNode = {
+    enable = true;
+    hostname = &quot;localhost&quot;;
+    url = &quot;http://localhost:5000&quot;;
+  };
+};
+</programlisting>
+    <para>
+      This will start a sync server that is only accessible locally.
+      Once the services is running you can navigate to
+      <literal>about:config</literal> in your Firefox profile and set
+      <literal>identity.sync.tokenserver.uri</literal> to
+      <literal>http://localhost:5000/1.0/sync/1.5</literal>. Your
+      browser will now use your local sync server for data storage.
+    </para>
+    <warning>
+      <para>
+        This configuration should never be used in production. It is not
+        encrypted and stores its secrets in a world-readable location.
+      </para>
+    </warning>
+  </section>
+  <section xml:id="module-services-firefox-syncserver-configuration">
+    <title>More detailed setup</title>
+    <para>
+      The <literal>firefox-syncserver</literal> service provides a
+      number of options to make setting up small deployment easier.
+      These are grouped under the <literal>singleNode</literal> element
+      of the option tree and allow simple configuration of the most
+      important parameters.
+    </para>
+    <para>
+      Single node setup is split into two kinds of options: those that
+      affect the sync server itself, and those that affect its
+      surroundings. Options that affect the sync server are
+      <literal>capacity</literal>, which configures how many accounts
+      may be active on this instance, and <literal>url</literal>, which
+      holds the URL under which the sync server can be accessed. The
+      <literal>url</literal> can be configured automatically when using
+      nginx.
+    </para>
+    <para>
+      Options that affect the surroundings of the sync server are
+      <literal>enableNginx</literal>, <literal>enableTLS</literal> and
+      <literal>hostnam</literal>. If <literal>enableNginx</literal> is
+      set the sync server module will automatically add an nginx virtual
+      host to the system using <literal>hostname</literal> as the domain
+      and set <literal>url</literal> accordingly. If
+      <literal>enableTLS</literal> is set the module will also enable
+      ACME certificates on the new virtual host and force all
+      connections to be made via TLS.
+    </para>
+    <para>
+      For actual deployment it is also recommended to store the
+      <literal>secrets</literal> file in a secure location.
+    </para>
+  </section>
+</chapter>

pkgs/servers/syncstorage-rs/default.nix

@@ -0,0 +1,63 @@
+{ fetchFromGitHub
+, rustPlatform
+, pkg-config
+, python3
+, openssl
+, cmake
+, libmysqlclient
+, makeBinaryWrapper
+, lib
+}:
+
+let
+  pyFxADeps = python3.withPackages (p: [
+    p.setuptools # imports pkg_resources
+    # remainder taken from requirements.txt
+    p.pyfxa
+    p.tokenlib
+    p.cryptography
+  ]);
+in
+
+rustPlatform.buildRustPackage rec {
+  pname = "syncstorage-rs";
+  version = "0.12.0";
+
+  src = fetchFromGitHub {
+    owner = "mozilla-services";
+    repo = pname;
+    rev = version;
+    hash = "sha256-VfIpjpBS7LXe32fxIFp7xmbm40VwxUdHIEm5PnMpd4s=";
+  };
+
+  nativeBuildInputs = [
+    cmake
+    makeBinaryWrapper
+    pkg-config
+    python3
+  ];
+
+  buildInputs = [
+    libmysqlclient
+    openssl
+  ];
+
+  preFixup = ''
+    wrapProgram $out/bin/syncstorage \
+      --prefix PATH : ${lib.makeBinPath [ pyFxADeps ]}
+  '';
+
+  cargoSha256 = "sha256-JXxArKA/2SIYJvjNA1yZHR9xDKt3N2U7HVMP/6M3BxE=";
+
+  buildFeatures = [ "grpcio/openssl" ];
+
+  # almost all tests need a DB to test against
+  doCheck = false;
+
+  meta = {
+    description = "Mozilla Sync Storage built with Rust";
+    homepage = "https://github.com/mozilla-services/syncstorage-rs";
+    license = lib.licenses.mpl20;
+    maintainers = with lib.maintainers; [ pennae ];
+  };
+}

pkgs/top-level/all-packages.nix

@@ -22805,6 +22805,8 @@ with pkgs;
 
   sympa = callPackage ../servers/mail/sympa { };
 
+  syncstorage-rs = callPackage ../servers/syncstorage-rs { };
+
   system-sendmail = lowPrio (callPackage ../servers/mail/system-sendmail { });
 
   # PulseAudio daemons