Run test suite with druntime, not Phobos

ci/run.sh

@@ -128,7 +128,7 @@ test_dmd() {
     fi
 
     $build_path/dmd -g -i -Icompiler/test -release compiler/test/run.d -ofgenerated/run
-    generated/run -j$N --environment MODEL=$MODEL HOST_DMD=$build_path/dmd "${args[@]}"
+    generated/run -j$N --environment MODEL=$MODEL HOST_DMD="$PWD/$build_path/dmd" "${args[@]}"
 }
 
 # build and run druntime unit tests

compiler/test/compilable/b19294.d

@@ -1,3 +1,4 @@
+// COMPILABLE_MATH_TEST
 alias MT = MyStruct!int;
 
 struct MyStruct(T)

compiler/test/compilable/compile1.d

@@ -1,10 +1,9 @@
-// COMPILABLE_MATH_TEST
 // PERMUTE_ARGS:
 // REQUIRED_ARGS: -verrors=simple
 // EXTRA_FILES: imports/a12506.d
 /* TEST_OUTPUT:
 ---
-compilable/compile1.d(231): Deprecation: use of complex type `cdouble` is deprecated, use `std.complex.Complex!(double)` instead
+compilable/compile1.d(230): Deprecation: use of complex type `cdouble` is deprecated, use `std.complex.Complex!(double)` instead
 ---
 */
 
@@ -872,8 +871,6 @@ alias TT14166(T...) = T;
 static assert(is(typeof((s14166.x += 1) = 2) == int));     // ok <- error
 static assert(is(typeof(s14166.a.length += 2) == size_t)); // ok <- error
 static assert(is(typeof(s14166++) == S14166));             // ok <- error
-static assert(is(typeof(s14166.x ^^ 2) == int));           // ok <- error
-static assert(is(typeof(s14166.y ^^= 2.5) == double));     // ok <- error
 static assert(is(typeof(makeAA14166()[0] = 1) == X14166)); // ok <- error
 static assert(is(typeof(tup14166.field = makeTup14166()) == TT14166!(int, int))); // ok <- error
 

compiler/test/compilable/ddoc24871.d

@@ -2,7 +2,7 @@
 // REQUIRED_ARGS: -D -Dd${RESULTS_DIR}/compilable -o-
 // POST_SCRIPT: compilable/extra-files/ddocAny-postscript.sh
 // EXTRA_SOURCES: extra-files/ddoc_minimal.ddoc
-import std.stdio;
+import core.stdc.stdio;
 
 /// Example
 /// ---
@@ -14,6 +14,6 @@ import std.stdio;
 /// ---
 void main() {
 
-    writeln("Hello, World!");
+    printf("Hello, World!\n");
 
 }

compiler/test/compilable/extra-files/json2.json

@@ -8,8 +8,7 @@
         "defFile": null,
         "importPaths": [
             "compilable",
-            "../../druntime/import",
-            "../../../phobos"
+            "../../druntime/import"
         ],
         "libName": "VALUE_REMOVED_FOR_TEST",
         "libraryFiles": [
@@ -89,4 +88,4 @@
             }
         ]
     }
-}
+}

compiler/test/compilable/paranoia_ctfe.d

@@ -1,3 +1,4 @@
+// COMPILABLE_MATH_TEST
 /*
 EXTRA_SOURCES: ../runnable/extra-files/paranoia.d
 REQUIRED_ARGS: -o- -version=CTFE

compiler/test/fail_compilation/b3841.d

@@ -70,13 +70,4 @@ void main()
         f!(op, int, float)();
         f!(op, long, double)();
     }
-
-    // OK
-    f!("^^=", int, int)();
-    f!("^^=", long, int)();
-    f!("^^=", long, short)();
-    f!("^^=", float, long)();
-    f!("^^=", double, float)();
-    // Should that really be OK ?
-    f!("^^=", float, double)();
 }

compiler/test/fail_compilation/diag10327.d

@@ -1,11 +1,10 @@
 /*
 TEST_OUTPUT:
 ---
-fail_compilation/diag10327.d(12): Error: unable to read module `test10327`
-fail_compilation/diag10327.d(12):        Expected 'imports/test10327.d' or 'imports/test10327/package.d' in one of the following import paths:
+fail_compilation/diag10327.d(11): Error: unable to read module `test10327`
+fail_compilation/diag10327.d(11):        Expected 'imports/test10327.d' or 'imports/test10327/package.d' in one of the following import paths:
 import path[0] = fail_compilation
 import path[1] = $p:druntime/import$
-import path[2] = $p:phobos$
 ---
 */
 

compiler/test/fail_compilation/fail18938.d

@@ -1,3 +1,4 @@
+// RUNNABLE_PHOBOS_TEST
 // REQUIRED_ARGS: -c -Ifail_compilation/imports/
 // EXTRA_SOURCES: imports/test18938a/cache.d imports/test18938a/file.d
 // EXTRA_FILES: imports/test18938b/file.d

compiler/test/fail_compilation/fail21045.d

@@ -1,11 +1,10 @@
 /*
 TEST_OUTPUT:
 ---
-fail_compilation/fail21045.d(12): Error: unable to read module `__stdin`
-fail_compilation/fail21045.d(12):        Expected '__stdin.d' or '__stdin/package.d' in one of the following import paths:
+fail_compilation/fail21045.d(11): Error: unable to read module `__stdin`
+fail_compilation/fail21045.d(11):        Expected '__stdin.d' or '__stdin/package.d' in one of the following import paths:
 import path[0] = fail_compilation
 import path[1] = $p:druntime/import$
-import path[2] = $p:phobos$
 ---
 */
 

compiler/test/fail_compilation/fail21091a.d

@@ -3,11 +3,10 @@
 /*
 TEST_OUTPUT:
 ----
-fail_compilation/fail21091a.d(16): Error: unable to read module `Ternary`
-fail_compilation/fail21091a.d(16):        Expected 'Ternary.d' or 'Ternary/package.d' in one of the following import paths:
+fail_compilation/fail21091a.d(15): Error: unable to read module `Ternary`
+fail_compilation/fail21091a.d(15):        Expected 'Ternary.d' or 'Ternary/package.d' in one of the following import paths:
 import path[0] = fail_compilation
 import path[1] = $p:druntime/import$
-import path[2] = $p:phobos$
 ----
 */
 

compiler/test/fail_compilation/fail21091b.d

@@ -3,11 +3,10 @@
 /*
 TEST_OUTPUT:
 ----
-fail_compilation/fail21091b.d(16): Error: unable to read module `Tid`
-fail_compilation/fail21091b.d(16):        Expected 'Tid.d' or 'Tid/package.d' in one of the following import paths:
+fail_compilation/fail21091b.d(15): Error: unable to read module `Tid`
+fail_compilation/fail21091b.d(15):        Expected 'Tid.d' or 'Tid/package.d' in one of the following import paths:
 import path[0] = fail_compilation
 import path[1] = $p:druntime/import$
-import path[2] = $p:phobos$
 ----
 */
 

compiler/test/fail_compilation/ice7782.d

@@ -2,11 +2,10 @@
 EXTRA_FILES: imports/ice7782algorithm.d imports/ice7782range.d
 TEST_OUTPUT:
 ----
-fail_compilation/ice7782.d(14): Error: unable to read module `ice7782math`
-fail_compilation/ice7782.d(14):        Expected 'imports/ice7782range/imports/ice7782math.d' or 'imports/ice7782range/imports/ice7782math/package.d' in one of the following import paths:
+fail_compilation/ice7782.d(13): Error: unable to read module `ice7782math`
+fail_compilation/ice7782.d(13):        Expected 'imports/ice7782range/imports/ice7782math.d' or 'imports/ice7782range/imports/ice7782math/package.d' in one of the following import paths:
 import path[0] = fail_compilation
 import path[1] = $p:druntime/import$
-import path[2] = $p:phobos$
 ----
 */
 

compiler/test/fail_compilation/issue11070.d

@@ -1,14 +1,13 @@
 /*
 TEST_OUTPUT:
 ---
-fail_compilation/issue11070.d(16): Error: undefined identifier `x`
+fail_compilation/issue11070.d(15): Error: undefined identifier `x`
 ---
 */
 
 int get() { return 1; }
 
 void test() {
-    import std.stdio : writeln;
     switch (auto x = get()) {
         default:
             auto z = x;

compiler/test/fail_compilation/test22361.d

@@ -1,11 +1,10 @@
 /*
 TEST_OUTPUT:
 ---
-fail_compilation/test22361.d(11): Error: unable to read module `this_module_does_not_exist`
-fail_compilation/test22361.d(11):        Expected 'this_module_does_not_exist.d' or 'this_module_does_not_exist/package.d' in one of the following import paths:
+fail_compilation/test22361.d(10): Error: unable to read module `this_module_does_not_exist`
+fail_compilation/test22361.d(10):        Expected 'this_module_does_not_exist.d' or 'this_module_does_not_exist/package.d' in one of the following import paths:
 import path[0] = fail_compilation
 import path[1] = $p:druntime/import$
-import path[2] = $p:phobos$
 ---
 */
 import this_module_does_not_exist;

compiler/test/run.d

@@ -128,6 +128,9 @@ Options:
     }
 
     // allow overwrites from the environment
+    // HOST_DMD must be either a command on PATH (e.g. `dmd`, `ldmd2`) or an
+    // absolute path; it is used to build test-harness code (dshell scripts,
+    // tools) which runs from various working directories.
     hostDMD = environment.get("HOST_DMD", "dmd");
     unitTestRunnerCommand = resultsDir.buildPath("unit_test_runner").exeName;
 
@@ -145,6 +148,7 @@ Options:
     }
 
     verifyCompilerExists(env);
+    ensureDruntime(env);
     prepareOutputDirectory(env);
 
     if (runUnitTests)
@@ -227,6 +231,48 @@ void verifyCompilerExists(const string[string] env)
     }
 }
 
+/**
+Maps the test runner's `BUILD` to a runtime library build flavor.
+
+`BUILD` may be a dmd build configuration that only describes the compiler
+binary (e.g. VisualD passes `Debug` or `RelWithAsserts`), whereas the
+druntime/phobos makefiles only accept `debug` or `release`. The runtime
+library flavor is independent from the compiler's configuration, so anything
+that isn't explicitly a debug build maps to `release`.
+*/
+string runtimeBuild(string build)
+{
+    return build.toLower == "debug" ? "debug" : "release";
+}
+
+/**
+Builds (or rebuilds) the druntime library that the tests link against.
+
+Prevent test failures due to forgetting to make druntime manually.
+*/
+void ensureDruntime(const string[string] env)
+{
+    const make = environment.get("MAKE", "make");
+    const druntimeDir = environment.get("DRUNTIME_PATH", testPath(buildPath("..", "..", "druntime")));
+    const buildCommand = [
+        make, "-C", druntimeDir,
+        "MODEL=" ~ env["MODEL"],
+        "BUILD=" ~ runtimeBuild(env["BUILD"]),
+        // Pass the existing compiler so the druntime Makefile doesn't try to
+        // (re)build dmd itself, which fails when the compiler lives in a
+        // different generated/ subdirectory (e.g. VisualD's RelWithAsserts).
+        "DMD=" ~ env["DMD"],
+    ];
+
+    writefln("Building druntime: %-(%s %)", buildCommand);
+    stdout.flush();
+    if (spawnProcess(buildCommand, env, Config.none, scriptDir).wait)
+    {
+        stderr.writeln("failed to build druntime");
+        quitSilently(1);
+    }
+}
+
 /// Creates the necessary directories and files for the test runner(s)
 void prepareOutputDirectory(const string[string] env)
 {
@@ -286,20 +332,18 @@ void ensureToolsExists(const string[string] env, const TestTool[] tools ...)
         auto lastModifiedBin = targetBin.timeLastModified.ifThrown(SysTime.init);
         if (lastModifiedBin >= sourceFile.timeLastModified)
         {
-            auto lastModifiedDmd = env["DMD"].timeLastModified.ifThrown(SysTime.init);
-            if (!tool.linksWithTests || lastModifiedBin >= lastModifiedDmd)
-            {
-                log("%s is already up-to-date", tool);
-                continue;
-            }
+            log("%s is already up-to-date", tool);
+            continue;
         }
 
         string[] buildCommand;
-        bool overrideEnv;
+        string[string] buildEnv = null;
         if (tool.linksWithTests)
         {
-            // This will compile the dshell library thus needs the actual
-            // DMD compiler under test
+            // The dshell library imports Phobos and is linked against the tests,
+            // so it must be built by the compiler under test (the host compiler
+            // may be too old to parse the current druntime/Phobos) using the
+            // Phobos DFLAGS rather than the druntime-only default.
             buildCommand = [
                 env["DMD"],
                 "-conf=",
@@ -308,7 +352,9 @@ void ensureToolsExists(const string[string] env, const TestTool[] tools ...)
                 "-c",
                 sourceFile
             ] ~ getPicFlags(env);
-            overrideEnv = true;
+            foreach (k, v; env)
+                buildEnv[k] = v;
+            buildEnv["DFLAGS"] = env.get("PHOBOS_DFLAGS", "");
         }
         else
         {
@@ -322,7 +368,7 @@ void ensureToolsExists(const string[string] env, const TestTool[] tools ...)
 
         writefln("Executing: %-(%s %)", buildCommand);
         stdout.flush();
-        if (spawnProcess(buildCommand, overrideEnv ? env : null).wait)
+        if (spawnProcess(buildCommand, buildEnv).wait)
         {
             stderr.writefln("failed to build '%s'", targetBin);
             atomicOp!"+="(failCount, 1);
@@ -551,29 +597,32 @@ string[string] getEnvironment()
     env["BUILD"] = build;
     env["EXE"] = exeExtension;
     env["DMD"] = dmdPath;
+    env["HOST_DMD"] = hostDMD;
     env.setDefault("DMD_TEST_COVERAGE", "0");
 
     const generatedSuffix = "generated/%s/%s/%s".format(os, build, model);
 
+    const druntimePath = environment.get("DRUNTIME_PATH", projectRootDir.buildPath("druntime"));
+    const druntimeLibDir = generatedDir.buildPath(os, runtimeBuild(build), model);
+
+    const phobosPath = environment.get("PHOBOS_PATH", projectRootDir.dirName.buildPath("phobos"));
+
     version(Windows)
     {
         env.setDefault("ARGS", "-inline -release -g -O");
         env["OBJ"] = ".obj";
         env["DSEP"] = `\\`;
         env["SEP"] = `\`;
-        auto druntimePath = environment.get("DRUNTIME_PATH", testPath(`..\..\druntime`));
-        auto phobosPath = environment.get("PHOBOS_PATH", testPath(`..\..\..\phobos`));
-        env["DFLAGS"] = `-I"%s\import" -I"%s"`.format(druntimePath, phobosPath);
-        env["LIB"] = phobosPath ~ ";" ~ environment.get("LIB");
+        env["DFLAGS"] = `-I"%s\import" -defaultlib=druntime -debuglib=druntime`.format(druntimePath);
+        env["LIB"] = druntimeLibDir ~ ";" ~ phobosPath ~ ";" ~ environment.get("LIB");
+        env["PHOBOS_DFLAGS"] = `-I"%s\import" -I"%s"`.format(druntimePath, phobosPath);
     }
     else
     {
         env.setDefault("ARGS", "-inline -release -g -O -fPIC");
         env["OBJ"] = ".o";
         env["DSEP"] = "/";
         env["SEP"] = "/";
-        auto druntimePath = environment.get("DRUNTIME_PATH", testPath(`../../druntime`));
-        auto phobosPath = environment.get("PHOBOS_PATH", testPath(`../../../phobos`));
 
         // default to PIC, use PIC=1/0 to en-/disable PIC.
         // Note that shared libraries and C files are always compiled with PIC.
@@ -582,11 +631,13 @@ string[string] getEnvironment()
             pic = false;
 
         env["PIC_FLAG"]  = pic ? "-fPIC" : "";
-        env["DFLAGS"] = "-I%s/import -I%s".format(druntimePath, phobosPath)
-            ~ " -L-L%s/%s".format(phobosPath, generatedSuffix);
+        env["DFLAGS"] = "-I%s/import -defaultlib=druntime -debuglib=druntime".format(druntimePath)
+            ~ " -L-L%s".format(druntimeLibDir);
+
+        env["PHOBOS_DFLAGS"] = "-I%s/import -I%s -L-L%s/%s".format(druntimePath, phobosPath, phobosPath, generatedSuffix);
         bool isShared = environment.get("SHARED") != "0" && os.among("linux", "freebsd", "hurd") > 0;
         if (isShared)
-            env["DFLAGS"] = env["DFLAGS"] ~ " -defaultlib=libphobos2.so -L-rpath=%s/%s".format(phobosPath, generatedSuffix);
+            env["PHOBOS_DFLAGS"] = env["PHOBOS_DFLAGS"] ~ " -defaultlib=libphobos2.so -L-rpath=%s/%s".format(phobosPath, generatedSuffix);
 
         if (pic)
             env["REQUIRED_ARGS"] = environment.get("REQUIRED_ARGS") ~ " " ~ env["PIC_FLAG"];

compiler/test/runnable/argufilem.d

@@ -15,12 +15,13 @@ why is 8 scared of 7? because789
 // main.d -------------------------------------------------------
 
 import argufile;
+import core.stdc.stdio : printf;
 
 int main(string[] args)
 {
         string message = arguments("bob is ", 7, " years old");
 
-        writefln(message);
+        printf("%.*s\n", cast(int) message.length, message.ptr);
 
         argufile.useargs(); // will crash here
 

compiler/test/runnable/arrayop.d

@@ -1,5 +1,4 @@
 // RUNNABLE_PHOBOS_TEST
-import std.math;
 
 extern(C) int printf(const char*, ...);