Merge branch 'nix'.

This is a squashed commit of the following:

commit 0dccab9f41
Author: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date:   Thu Jul 2 00:30:16 2015 +0200

    Typo

commit 2cd28517b1
Author: Ludovic Courtès <ludo@gnu.org>
Date:   Wed Jul 1 14:56:34 2015 +0200

    Preserve supplementary groups of build users

    The following patch is an attempt to address this bug (see
    <http://bugs.gnu.org/18994>) by preserving the supplementary groups of
    build users in the build environment.

    In practice, I would expect that supplementary groups would contain only
    one or two groups: the build users group, and possibly the “kvm” group.

    [Changed &at(0) to data() and removed tabs - Eelco]

commit 6e38685ef6
Author: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date:   Tue Jun 30 21:41:26 2015 +0200

    GC: Handle ENOSPC creating/moving to the trash directory

    Issue #564.

commit 5e0a9ae2e2
Author: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date:   Mon Jun 22 15:54:55 2015 +0200

    Use posix_fallocate to create /nix/var/nix/db/reserved

commit 4e5ab98d6d
Author: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date:   Mon Jun 22 15:47:40 2015 +0200

    Make /nix/var/nix/db/reserved bigger

    Issue #564.

commit 60bda60fc0
Author: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date:   Wed Jun 10 16:17:06 2015 +0200

    Export outputPaths function

    This is useful for the new hydra-queue-runner.

commit 5dfea34048
Author: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date:   Thu Jul 2 22:51:33 2015 +0200

    Use std::vector::data()

commit 2459458bc8
Author: Eelco Dolstra <eelco.dolstra@logicblox.com>
Date:   Thu Jun 4 16:04:41 2015 +0200

    Allow substitutes for builds that have preferLocalBuild set

    Not substituting builds with "preferLocalBuild = true" was a bad idea,
    because it didn't take the cost of dependencies into account. For
    instance, if we can't substitute a fetchgit call, then we have to
    download/build git and all its dependencies.

    Partially reverts 5558652709 and adds a
    new derivation attribute "allowSubstitutes" to specify whether a
    derivation may be substituted.

nix/libstore/build.cc

@@ -447,6 +447,7 @@ private:
     string user;
     uid_t uid;
     gid_t gid;
+    std::vector<gid_t> supplementaryGIDs;
 
 public:
     UserLock();
@@ -460,6 +461,7 @@ public:
     string getUser() { return user; }
     uid_t getUID() { return uid; }
     uid_t getGID() { return gid; }
+    std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
 
     bool enabled() { return uid != 0; }
 
@@ -539,6 +541,17 @@ void UserLock::acquire()
                 throw Error(format("the Nix user should not be a member of `%1%'")
                     % settings.buildUsersGroup);
 
+            /* Get the list of supplementary groups of this build user.  This
+               is usually either empty or contains a group such as "kvm".  */
+            supplementaryGIDs.resize(10);
+            int ngroups = supplementaryGIDs.size();
+            int err = getgrouplist(pw->pw_name, pw->pw_gid,
+                supplementaryGIDs.data(), &ngroups);
+            if (err == -1)
+                throw Error(format("failed to get list of supplementary groups for ‘%1%’") % pw->pw_name);
+
+            supplementaryGIDs.resize(ngroups);
+
             return;
         }
     }
@@ -1000,7 +1013,7 @@ void DerivationGoal::haveDerivation()
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
-    if (settings.useSubstitutes && !willBuildLocally(drv))
+    if (settings.useSubstitutes && substitutesAllowed(drv))
         foreach (PathSet::iterator, i, invalidOutputs)
             addWaitee(worker.makeSubstitutionGoal(*i, buildMode == bmRepair));
 
@@ -1188,35 +1201,32 @@ void DerivationGoal::inputsRealised()
 }
 
 
-PathSet outputPaths(const DerivationOutputs & outputs)
+static bool canBuildLocally(const string & platform)
 {
-    PathSet paths;
-    foreach (DerivationOutputs::const_iterator, i, outputs)
-        paths.insert(i->second.path);
-    return paths;
+    return platform == settings.thisSystem
+#if __linux__
+        || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux")
+#endif
+        ;
 }
 
 
-static string get(const StringPairs & map, const string & key)
+static string get(const StringPairs & map, const string & key, const string & def = "")
 {
     StringPairs::const_iterator i = map.find(key);
-    return i == map.end() ? (string) "" : i->second;
+    return i == map.end() ? def : i->second;
 }
 
 
-static bool canBuildLocally(const string & platform)
+bool willBuildLocally(const Derivation & drv)
 {
-    return platform == settings.thisSystem
-#if __linux__
-        || (platform == "i686-linux" && settings.thisSystem == "x86_64-linux")
-#endif
-        ;
+    return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform);
 }
 
 
-bool willBuildLocally(const Derivation & drv)
+bool substitutesAllowed(const Derivation & drv)
 {
-    return get(drv.env, "preferLocalBuild") == "1" && canBuildLocally(drv.platform);
+    return get(drv.env, "allowSubstitutes", "1") == "1";
 }
 
 
@@ -1242,7 +1252,7 @@ void DerivationGoal::tryToBuild()
        can't acquire the lock, then continue; hopefully some other
        goal can start a build, and if not, the main loop will sleep a
        few seconds and then retry this goal. */
-    if (!outputLocks.lockPaths(outputPaths(drv.outputs), "", false)) {
+    if (!outputLocks.lockPaths(outputPaths(drv), "", false)) {
         worker.waitForAWhile(shared_from_this());
         return;
     }
@@ -1263,7 +1273,7 @@ void DerivationGoal::tryToBuild()
         return;
     }
 
-    missingPaths = outputPaths(drv.outputs);
+    missingPaths = outputPaths(drv);
     if (buildMode != bmCheck)
         foreach (PathSet::iterator, i, validPaths) missingPaths.erase(*i);
 
@@ -2168,7 +2178,6 @@ void DerivationGoal::runChild()
         Strings envStrs;
         foreach (Environment::const_iterator, i, env)
             envStrs.push_back(rewriteHashes(i->first + "=" + i->second, rewritesToTmp));
-        auto envArr = stringsToCharPtrs(envStrs);
 
         /* If we are running in `build-users' mode, then switch to the
            user we allocated above.  Make sure that we drop all root
@@ -2177,10 +2186,11 @@ void DerivationGoal::runChild()
            setuid() when run as root sets the real, effective and
            saved UIDs. */
         if (buildUser.enabled()) {
-            printMsg(lvlChatty, format("switching to user `%1%'") % buildUser.getUser());
-
-            if (setgroups(0, 0) == -1)
-                throw SysError("cannot clear the set of supplementary groups");
+            /* Preserve supplementary groups of the build user, to allow
+               admins to specify groups such as "kvm".  */
+            if (setgroups(buildUser.getSupplementaryGIDs().size(),
+                          buildUser.getSupplementaryGIDs().data()) == -1)
+                throw SysError("cannot set supplementary groups of build user");
 
             if (setgid(buildUser.getGID()) == -1 ||
                 getgid() != buildUser.getGID() ||
@@ -2199,7 +2209,6 @@ void DerivationGoal::runChild()
         args.push_back(builderBasename);
         foreach (Strings::iterator, i, drv.args)
             args.push_back(rewriteHashes(*i, rewritesToTmp));
-        auto argArr = stringsToCharPtrs(args);
 
         restoreSIGPIPE();
 
@@ -2207,7 +2216,7 @@ void DerivationGoal::runChild()
         writeFull(STDERR_FILENO, "\n");
 
         /* Execute the program.  This should not return. */
-        execve(drv.builder.c_str(), (char * *) &argArr[0], (char * *) &envArr[0]);
+        execve(drv.builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
 
         throw SysError(format("executing `%1%'") % drv.builder);
 
@@ -2837,7 +2846,6 @@ void SubstitutionGoal::tryToRun()
     args.push_back("--substitute");
     args.push_back(storePath);
     args.push_back(destPath);
-    auto argArr = stringsToCharPtrs(args);
 
     /* Fork the substitute program. */
     pid = startProcess([&]() {
@@ -2847,7 +2855,7 @@ void SubstitutionGoal::tryToRun()
         if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
             throw SysError("cannot dup output pipe into stdout");
 
-        execv(sub.c_str(), (char * *) &argArr[0]);
+        execv(sub.c_str(), stringsToCharPtrs(args).data());
 
         throw SysError(format("executing `%1%'") % sub);
     });

nix/libstore/derivations.cc

@@ -285,4 +285,13 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
 }
 
 
+PathSet outputPaths(const Derivation & drv)
+{
+    PathSet paths;
+    for (auto & i : drv.outputs)
+        paths.insert(i.second.path);
+    return paths;
+}
+
+
 }

nix/libstore/derivations.hh

@@ -89,5 +89,6 @@ Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outpu
 
 bool wantOutput(const string & output, const std::set<string> & wanted);
 
+PathSet outputPaths(const Derivation & drv);
 
 }

nix/libstore/gc.cc

@@ -376,6 +376,7 @@ struct LocalStore::GCState
     bool gcKeepOutputs;
     bool gcKeepDerivations;
     unsigned long long bytesInvalidated;
+    bool moveToTrash = true;
     Path trashDir;
     bool shouldDelete;
     GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { }
@@ -428,16 +429,23 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
        not holding the global GC lock) we can delete the path without
        being afraid that the path has become alive again.  Otherwise
        delete it right away. */
-    if (S_ISDIR(st.st_mode)) {
+    if (state.moveToTrash && S_ISDIR(st.st_mode)) {
         // Estimate the amount freed using the narSize field.  FIXME:
         // if the path was not valid, need to determine the actual
         // size.
-        state.bytesInvalidated += size;
-        if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
-            throw SysError(format("making `%1%' writable") % path);
-        Path tmp = state.trashDir + "/" + baseNameOf(path);
-        if (rename(path.c_str(), tmp.c_str()))
-            throw SysError(format("unable to rename `%1%' to `%2%'") % path % tmp);
+        try {
+            if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
+                throw SysError(format("making `%1%' writable") % path);
+            Path tmp = state.trashDir + "/" + baseNameOf(path);
+            if (rename(path.c_str(), tmp.c_str()))
+                throw SysError(format("unable to rename `%1%' to `%2%'") % path % tmp);
+            state.bytesInvalidated += size;
+        } catch (SysError & e) {
+            if (e.errNo == ENOSPC) {
+                printMsg(lvlInfo, format("note: can't create move `%1%': %2%") % path % e.msg());
+                deleteGarbage(state, path);
+            }
+        }
     } else
         deleteGarbage(state, path);
 
@@ -636,7 +644,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     if (state.shouldDelete) {
         if (pathExists(state.trashDir)) deleteGarbage(state, state.trashDir);
-        createDirs(state.trashDir);
+        try {
+            createDirs(state.trashDir);
+        } catch (SysError & e) {
+            if (e.errNo == ENOSPC) {
+                printMsg(lvlInfo, format("note: can't create trash directory: %1%") % e.msg());
+                state.moveToTrash = false;
+            }
+        }
     }
 
     /* Now either delete all garbage paths, or just the specified

nix/libstore/globals.cc

@@ -36,7 +36,7 @@ Settings::Settings()
     buildTimeout = 0;
     useBuildHook = true;
     printBuildTrace = false;
-    reservedSize = 1024 * 1024;
+    reservedSize = 8 * 1024 * 1024;
     fsyncMetadata = true;
     useSQLiteWAL = true;
     syncBeforeRegistering = false;

nix/libstore/local-store.cc

@@ -288,7 +288,17 @@ LocalStore::LocalStore(bool reserveSpace)
             struct stat st;
             if (stat(reservedPath.c_str(), &st) == -1 ||
                 st.st_size != settings.reservedSize)
-                writeFile(reservedPath, string(settings.reservedSize, 'X'));
+            {
+                AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT, 0600);
+                int res = -1;
+#if HAVE_POSIX_FALLOCATE
+                res = posix_fallocate(fd, 0, settings.reservedSize);
+#endif
+                if (res == -1) {
+                    writeFull(fd, string(settings.reservedSize, 'X'));
+                    ftruncate(fd, settings.reservedSize);
+                }
+            }
         }
         else
             deletePath(reservedPath);
@@ -1166,10 +1176,8 @@ string LocalStore::getLineFromSubstituter(RunningSubstituter & run)
             if (n == 0) throw EndOfFile(format("substituter `%1%' died unexpectedly") % run.program);
             err.append(buf, n);
             string::size_type p;
-            while (((p = err.find('\n')) != string::npos)
-		   || ((p = err.find('\r')) != string::npos)) {
-	        string thing(err, 0, p + 1);
-		writeToStderr(run.program + ": " + thing);
+            while ((p = err.find('\n')) != string::npos) {
+                printMsg(lvlError, run.program + ": " + string(err, 0, p));
                 err = string(err, p + 1);
             }
         }

nix/libstore/misc.cc

@@ -120,7 +120,7 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
                 if (invalid.empty()) continue;
 
                 todoDrv.insert(*i);
-                if (settings.useSubstitutes && !willBuildLocally(drv))
+                if (settings.useSubstitutes && substitutesAllowed(drv))
                     query.insert(invalid.begin(), invalid.end());
             }
 
@@ -144,7 +144,7 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
 
             PathSet outputs;
             bool mustBuild = false;
-            if (settings.useSubstitutes && !willBuildLocally(drv)) {
+            if (settings.useSubstitutes && substitutesAllowed(drv)) {
                 foreach (DerivationOutputs::iterator, j, drv.outputs) {
                     if (!wantOutput(j->first, i2.second)) continue;
                     if (!store.isValidPath(j->second.path)) {

nix/libstore/misc.hh

@@ -34,5 +34,7 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
 
 bool willBuildLocally(const Derivation & drv);
 
+bool substitutesAllowed(const Derivation & drv);
+
 
 }

nix/libutil/util.cc

@@ -897,10 +897,10 @@ pid_t startProcess(std::function<void()> fun,
 }
 
 
-std::vector<const char *> stringsToCharPtrs(const Strings & ss)
+std::vector<char *> stringsToCharPtrs(const Strings & ss)
 {
-    std::vector<const char *> res;
-    for (auto & s : ss) res.push_back(s.c_str());
+    std::vector<char *> res;
+    for (auto & s : ss) res.push_back((char *) s.c_str());
     res.push_back(0);
     return res;
 }
@@ -921,12 +921,11 @@ string runProgram(Path program, bool searchPath, const Strings & args)
 
         Strings args_(args);
         args_.push_front(program);
-        auto cargs = stringsToCharPtrs(args_);
 
         if (searchPath)
-            execvp(program.c_str(), (char * *) &cargs[0]);
+            execvp(program.c_str(), stringsToCharPtrs(args_).data());
         else
-            execv(program.c_str(), (char * *) &cargs[0]);
+            execv(program.c_str(), stringsToCharPtrs(args_).data());
 
         throw SysError(format("executing `%1%'") % program);
     });

nix/libutil/util.hh

@@ -284,7 +284,7 @@ MakeError(ExecError, Error)
 /* Convert a list of strings to a null-terminated vector of char
    *'s. The result must not be accessed beyond the lifetime of the
    list of strings. */
-std::vector<const char *> stringsToCharPtrs(const Strings & ss);
+std::vector<char *> stringsToCharPtrs(const Strings & ss);
 
 /* Close all file descriptors except stdin, stdout, stderr, and those
    listed in the given set.  Good practice in child processes. */

nix/nix-daemon/nix-daemon.cc

@@ -440,10 +440,7 @@ static void performOp(bool trusted, unsigned int clientVersion,
     case wopImportPaths: {
         startWork();
         TunnelSource source(from);
-
-	/* Unlike Nix, always require a signature, even for "trusted"
-	   users.  */
-        Paths paths = store->importPaths(true, source);
+        Paths paths = store->importPaths(!trusted, source);
         stopWork();
         writeStrings(paths, to);
         break;