From dbb655a1e3a6e0b5525b8ef8109a0546d874f707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Tue, 13 Jun 2017 18:36:01 +0200 Subject: [PATCH 01/19] Change the for-loop desugar so the `break` does not affect type inference. Fixes #42618 --- src/libcore/iter/mod.rs | 6 ++++-- src/librustc/hir/lowering.rs | 37 ++++++++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/libcore/iter/mod.rs b/src/libcore/iter/mod.rs index ee81151348772..c91fd16391aaf 100644 --- a/src/libcore/iter/mod.rs +++ b/src/libcore/iter/mod.rs @@ -191,10 +191,12 @@ //! { //! let result = match IntoIterator::into_iter(values) { //! mut iter => loop { -//! let x = match iter.next() { -//! Some(val) => val, +//! let next; +//! match iter.next() { +//! Some(val) => next = val, //! None => break, //! }; +//! let x = next; //! let () = { println!("{}", x); }; //! }, //! }; diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index a6ab67e04693d..1283d136d3287 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -2170,11 +2170,13 @@ impl<'a> LoweringContext<'a> { // let result = match ::std::iter::IntoIterator::into_iter() { // mut iter => { // [opt_ident]: loop { - // let = match ::std::iter::Iterator::next(&mut iter) { - // ::std::option::Option::Some(val) => val, + // let next; + // match ::std::iter::Iterator::next(&mut iter) { + // ::std::option::Option::Some(val) => next = val, // ::std::option::Option::None => break // }; - // SemiExpr(); + // let = next; + // StmtExpr(); // } // } // }; @@ -2186,13 +2188,18 @@ impl<'a> LoweringContext<'a> { let iter = self.str_to_ident("iter"); - // `::std::option::Option::Some(val) => val` + let next_ident = self.str_to_ident("next"); + let next_pat = self.pat_ident(e.span, next_ident); + + // `::std::option::Option::Some(val) => next = val` let pat_arm = { let val_ident = self.str_to_ident("val"); let val_pat = self.pat_ident(e.span, val_ident); let val_expr = P(self.expr_ident(e.span, val_ident, val_pat.id)); + let next_expr = P(self.expr_ident(e.span, next_ident, next_pat.id)); + let assign = P(self.expr(e.span, hir::ExprAssign(next_expr, val_expr), ThinVec::new())); let some_pat = self.pat_some(e.span, val_pat); - self.arm(hir_vec![some_pat], val_expr) + self.arm(hir_vec![some_pat], assign) }; // `::std::option::Option::None => break` @@ -2222,10 +2229,20 @@ impl<'a> LoweringContext<'a> { hir::MatchSource::ForLoopDesugar), ThinVec::new())) }; + let match_stmt = respan(e.span, hir::StmtExpr(match_expr, self.next_id())); + + let next_expr = P(self.expr_ident(e.span, next_ident, next_pat.id)); + + // `let next` + let next_let = self.stmt_let_pat(e.span, + None, + next_pat, + hir::LocalSource::ForLoopDesugar); + // `let = next` let pat = self.lower_pat(pat); let pat_let = self.stmt_let_pat(e.span, - match_expr, + Some(next_expr), pat, hir::LocalSource::ForLoopDesugar); @@ -2234,7 +2251,7 @@ impl<'a> LoweringContext<'a> { let body_expr = P(self.expr_block(body_block, ThinVec::new())); let body_stmt = respan(e.span, hir::StmtExpr(body_expr, self.next_id())); - let loop_block = P(self.block_all(e.span, hir_vec![pat_let, body_stmt], None)); + let loop_block = P(self.block_all(e.span, hir_vec![next_let, match_stmt, pat_let, body_stmt], None)); // `[opt_ident]: loop { ... }` let loop_expr = hir::ExprLoop(loop_block, self.lower_opt_sp_ident(opt_ident), @@ -2601,14 +2618,14 @@ impl<'a> LoweringContext<'a> { fn stmt_let_pat(&mut self, sp: Span, - ex: P, + ex: Option>, pat: P, source: hir::LocalSource) -> hir::Stmt { let local = P(hir::Local { pat: pat, ty: None, - init: Some(ex), + init: ex, id: self.next_id(), span: sp, attrs: ThinVec::new(), @@ -2626,7 +2643,7 @@ impl<'a> LoweringContext<'a> { self.pat_ident(sp, ident) }; let pat_id = pat.id; - (self.stmt_let_pat(sp, ex, pat, hir::LocalSource::Normal), pat_id) + (self.stmt_let_pat(sp, Some(ex), pat, hir::LocalSource::Normal), pat_id) } fn block_expr(&mut self, expr: P) -> hir::Block { From 2d379b33936d6cd9a9b643cf5ddc36450176553c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Wed, 14 Jun 2017 13:36:30 +0200 Subject: [PATCH 02/19] Fix formatting and add a test for destruction order of unbound values --- src/librustc/hir/lowering.rs | 15 +++++++--- .../for-loop-lifetime-of-unbound-values.rs | 28 +++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 src/test/run-pass/for-loop-lifetime-of-unbound-values.rs diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index 1283d136d3287..f2a434979535d 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -2190,14 +2190,16 @@ impl<'a> LoweringContext<'a> { let next_ident = self.str_to_ident("next"); let next_pat = self.pat_ident(e.span, next_ident); - + // `::std::option::Option::Some(val) => next = val` let pat_arm = { let val_ident = self.str_to_ident("val"); let val_pat = self.pat_ident(e.span, val_ident); let val_expr = P(self.expr_ident(e.span, val_ident, val_pat.id)); let next_expr = P(self.expr_ident(e.span, next_ident, next_pat.id)); - let assign = P(self.expr(e.span, hir::ExprAssign(next_expr, val_expr), ThinVec::new())); + let assign = P(self.expr(e.span, + hir::ExprAssign(next_expr, val_expr), + ThinVec::new())); let some_pat = self.pat_some(e.span, val_pat); self.arm(hir_vec![some_pat], assign) }; @@ -2232,7 +2234,7 @@ impl<'a> LoweringContext<'a> { let match_stmt = respan(e.span, hir::StmtExpr(match_expr, self.next_id())); let next_expr = P(self.expr_ident(e.span, next_ident, next_pat.id)); - + // `let next` let next_let = self.stmt_let_pat(e.span, None, @@ -2251,7 +2253,12 @@ impl<'a> LoweringContext<'a> { let body_expr = P(self.expr_block(body_block, ThinVec::new())); let body_stmt = respan(e.span, hir::StmtExpr(body_expr, self.next_id())); - let loop_block = P(self.block_all(e.span, hir_vec![next_let, match_stmt, pat_let, body_stmt], None)); + let loop_block = P(self.block_all(e.span, + hir_vec![next_let, + match_stmt, + pat_let, + body_stmt], + None)); // `[opt_ident]: loop { ... }` let loop_expr = hir::ExprLoop(loop_block, self.lower_opt_sp_ident(opt_ident), diff --git a/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs b/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs new file mode 100644 index 0000000000000..a273fb579fa0b --- /dev/null +++ b/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs @@ -0,0 +1,28 @@ +use std::cell::Cell; + +struct Flag<'a>(&'a Cell); + +impl<'a> Drop for Flag<'a> { + fn drop(&mut self) { + self.0.set(false) + } +} + +fn main() { + let alive2 = Cell::new(true); + for _i in std::iter::once(Flag(&alive2)) { + // The Flag value should be alive in the for loop body + assert_eq!(alive2.get(), true); + } + // The Flag value should be dead outside of the loop + assert_eq!(alive2.get(), false); + + let alive = Cell::new(true); + for _ in std::iter::once(Flag(&alive)) { + // The Flag value should be alive in the for loop body even if it wasn't + // bound by the for loop + assert_eq!(alive.get(), true); + } + // The Flag value should be dead outside of the loop + assert_eq!(alive.get(), false); +} \ No newline at end of file From 8d65dd62b17892102231101a7e58de0a20b5dd21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Wed, 14 Jun 2017 19:26:42 +0200 Subject: [PATCH 03/19] Fix test formatting --- .../for-loop-lifetime-of-unbound-values.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs b/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs index a273fb579fa0b..a0562edfadd6c 100644 --- a/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs +++ b/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs @@ -1,11 +1,21 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + use std::cell::Cell; struct Flag<'a>(&'a Cell); impl<'a> Drop for Flag<'a> { - fn drop(&mut self) { - self.0.set(false) - } + fn drop(&mut self) { + self.0.set(false) + } } fn main() { @@ -16,7 +26,7 @@ fn main() { } // The Flag value should be dead outside of the loop assert_eq!(alive2.get(), false); - + let alive = Cell::new(true); for _ in std::iter::once(Flag(&alive)) { // The Flag value should be alive in the for loop body even if it wasn't From a80840f75196b69ea3f6c8b1fdaa032be609e9f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 15 Jun 2017 02:09:53 +0200 Subject: [PATCH 04/19] Added more tests --- .../for-loop-unconstrained-element-type.rs | 13 +++++++++++++ .../for-loop-lifetime-of-unbound-values.rs | 2 +- ...op-unconstrained-element-type-i32-fallback.rs | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 src/test/compile-fail/for-loop-unconstrained-element-type.rs create mode 100644 src/test/run-pass/for-loop-unconstrained-element-type-i32-fallback.rs diff --git a/src/test/compile-fail/for-loop-unconstrained-element-type.rs b/src/test/compile-fail/for-loop-unconstrained-element-type.rs new file mode 100644 index 0000000000000..f7dccc7f2ac19 --- /dev/null +++ b/src/test/compile-fail/for-loop-unconstrained-element-type.rs @@ -0,0 +1,13 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +fn main() { + for i in Vec::new() { } //~ ERROR type annotations needed +} diff --git a/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs b/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs index a0562edfadd6c..4653a493898ee 100644 --- a/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs +++ b/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs @@ -35,4 +35,4 @@ fn main() { } // The Flag value should be dead outside of the loop assert_eq!(alive.get(), false); -} \ No newline at end of file +} diff --git a/src/test/run-pass/for-loop-unconstrained-element-type-i32-fallback.rs b/src/test/run-pass/for-loop-unconstrained-element-type-i32-fallback.rs new file mode 100644 index 0000000000000..d9a876d9c9521 --- /dev/null +++ b/src/test/run-pass/for-loop-unconstrained-element-type-i32-fallback.rs @@ -0,0 +1,16 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +fn main() { + let mut sum = 0; + for i in Vec::new() { + sum += i; + } +} From d5fd8fef67a0a0b4d30dd8b23052c380632ae0e2 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Thu, 15 Jun 2017 12:27:15 -0400 Subject: [PATCH 05/19] explain purpose of test --- .../compile-fail/for-loop-unconstrained-element-type.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/compile-fail/for-loop-unconstrained-element-type.rs b/src/test/compile-fail/for-loop-unconstrained-element-type.rs index f7dccc7f2ac19..dd09e4a79ecdc 100644 --- a/src/test/compile-fail/for-loop-unconstrained-element-type.rs +++ b/src/test/compile-fail/for-loop-unconstrained-element-type.rs @@ -8,6 +8,12 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +// Test that `for` loops don't introduce artificial +// constraints on the type of the binding (`i`). +// Subtle changes in the desugaring can cause the +// type of elements in the vector to (incorrectly) +// fallback to `!` or `()`. + fn main() { for i in Vec::new() { } //~ ERROR type annotations needed } From e4baa26d2a24845bb960a9a51fa3c4ecf63bdd4a Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Thu, 15 Jun 2017 12:28:07 -0400 Subject: [PATCH 06/19] document purpose of test --- src/test/run-pass/for-loop-lifetime-of-unbound-values.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs b/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs index 4653a493898ee..7a088b5133472 100644 --- a/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs +++ b/src/test/run-pass/for-loop-lifetime-of-unbound-values.rs @@ -8,6 +8,11 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +// Test when destructors run in a for loop. The intention is +// that the value for each iteration is dropped *after* the loop +// body has executed. This is true even when the value is assigned +// to a `_` pattern (and hence ignored). + use std::cell::Cell; struct Flag<'a>(&'a Cell); From f11cf60944eb52eaebae8f4b510a0e805c679958 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Thu, 15 Jun 2017 12:31:45 -0400 Subject: [PATCH 07/19] Create for-loop-unconstrained-element-type-i32-fallback.rs --- .../for-loop-unconstrained-element-type-i32-fallback.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/run-pass/for-loop-unconstrained-element-type-i32-fallback.rs b/src/test/run-pass/for-loop-unconstrained-element-type-i32-fallback.rs index d9a876d9c9521..b36afcf87b3ee 100644 --- a/src/test/run-pass/for-loop-unconstrained-element-type-i32-fallback.rs +++ b/src/test/run-pass/for-loop-unconstrained-element-type-i32-fallback.rs @@ -8,6 +8,10 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. +// Test that the type of `sum` falls back to `i32` here, +// and that the for loop desugaring doesn't inferfere with +// that. + fn main() { let mut sum = 0; for i in Vec::new() { From 09bc09201ca02c9bdbcba1c72610209c3598bca1 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Thu, 15 Jun 2017 15:45:37 -0400 Subject: [PATCH 08/19] remove trailing whitespace --- src/test/compile-fail/for-loop-unconstrained-element-type.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/compile-fail/for-loop-unconstrained-element-type.rs b/src/test/compile-fail/for-loop-unconstrained-element-type.rs index dd09e4a79ecdc..fb5553166bafb 100644 --- a/src/test/compile-fail/for-loop-unconstrained-element-type.rs +++ b/src/test/compile-fail/for-loop-unconstrained-element-type.rs @@ -11,7 +11,7 @@ // Test that `for` loops don't introduce artificial // constraints on the type of the binding (`i`). // Subtle changes in the desugaring can cause the -// type of elements in the vector to (incorrectly) +// type of elements in the vector to (incorrectly) // fallback to `!` or `()`. fn main() { From bd7cc779b65e658ff5b6bbd9d2bdd8ed07ea38f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sat, 17 Jun 2017 01:51:55 +0200 Subject: [PATCH 09/19] Make the `next` variable mutable to allow for ref mut in for patterns. --- src/librustc/hir/lowering.rs | 14 +++++++------- src/test/run-pass/for-loop-mut-ref-element.rs | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 src/test/run-pass/for-loop-mut-ref-element.rs diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index f2a434979535d..4b8ee8d8aecfb 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -2170,12 +2170,12 @@ impl<'a> LoweringContext<'a> { // let result = match ::std::iter::IntoIterator::into_iter() { // mut iter => { // [opt_ident]: loop { - // let next; + // let mut _next; // match ::std::iter::Iterator::next(&mut iter) { - // ::std::option::Option::Some(val) => next = val, + // ::std::option::Option::Some(val) => _next = val, // ::std::option::Option::None => break // }; - // let = next; + // let = _next; // StmtExpr(); // } // } @@ -2188,8 +2188,8 @@ impl<'a> LoweringContext<'a> { let iter = self.str_to_ident("iter"); - let next_ident = self.str_to_ident("next"); - let next_pat = self.pat_ident(e.span, next_ident); + let next_ident = self.str_to_ident("_next"); + let next_pat = self.pat_ident_binding_mode(e.span, next_ident, hir::BindByValue(hir::MutMutable)); // `::std::option::Option::Some(val) => next = val` let pat_arm = { @@ -2235,13 +2235,13 @@ impl<'a> LoweringContext<'a> { let next_expr = P(self.expr_ident(e.span, next_ident, next_pat.id)); - // `let next` + // `let mut _next` let next_let = self.stmt_let_pat(e.span, None, next_pat, hir::LocalSource::ForLoopDesugar); - // `let = next` + // `let = _next` let pat = self.lower_pat(pat); let pat_let = self.stmt_let_pat(e.span, Some(next_expr), diff --git a/src/test/run-pass/for-loop-mut-ref-element.rs b/src/test/run-pass/for-loop-mut-ref-element.rs new file mode 100644 index 0000000000000..14ce23b07242c --- /dev/null +++ b/src/test/run-pass/for-loop-mut-ref-element.rs @@ -0,0 +1,15 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// Tests that for loops can bind elements as mutable references + +fn main() { + for ref mut _a in std::iter::once(true) {} +} \ No newline at end of file From 1409e707a2e6b152ab66a9d836e5e367f6f066cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sat, 17 Jun 2017 14:46:37 +0200 Subject: [PATCH 10/19] Fix formatting --- src/librustc/hir/lowering.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index 4b8ee8d8aecfb..9715db42474da 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -2189,7 +2189,9 @@ impl<'a> LoweringContext<'a> { let iter = self.str_to_ident("iter"); let next_ident = self.str_to_ident("_next"); - let next_pat = self.pat_ident_binding_mode(e.span, next_ident, hir::BindByValue(hir::MutMutable)); + let next_pat = self.pat_ident_binding_mode(e.span, + next_ident, + hir::BindByValue(hir::MutMutable)); // `::std::option::Option::Some(val) => next = val` let pat_arm = { From 0b937989599392d42de0a792a91fa7bc0bc53f92 Mon Sep 17 00:00:00 2001 From: Ariel Ben-Yehuda Date: Tue, 20 Jun 2017 15:07:47 +0300 Subject: [PATCH 11/19] mark calls in the unwind path as !noinline The unwind path is always cold, so that should not have bad performance implications. This avoids catastrophic exponential inlining, and also decreases the size of librustc.so by 1.5% (OTOH, the size of `libstd.so` increased by 0.5% for some reason). Fixes #41696. --- src/librustc_trans/mir/block.rs | 7 +++++++ src/test/run-pass/issue-41696.rs | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/librustc_trans/mir/block.rs b/src/librustc_trans/mir/block.rs index 4926485a12125..8863c7ffae6f4 100644 --- a/src/librustc_trans/mir/block.rs +++ b/src/librustc_trans/mir/block.rs @@ -146,6 +146,13 @@ impl<'a, 'tcx> MirContext<'a, 'tcx> { } else { let llret = bcx.call(fn_ptr, &llargs, cleanup_bundle); fn_ty.apply_attrs_callsite(llret); + if this.mir[bb].is_cleanup { + // Cleanup is always the cold path. Don't inline + // drop glue. Also, when there is a deeply-nested + // struct, there are "symmetry" issues that cause + // exponential inlining - see issue #41696. + llvm::Attribute::NoInline.apply_callsite(llvm::AttributePlace::Function, llret); + } if let Some((ret_dest, ret_ty, target)) = destination { let op = OperandRef { diff --git a/src/test/run-pass/issue-41696.rs b/src/test/run-pass/issue-41696.rs index 40fbf5ba75c8d..1888be58c57d0 100644 --- a/src/test/run-pass/issue-41696.rs +++ b/src/test/run-pass/issue-41696.rs @@ -9,8 +9,6 @@ // except according to those terms. // this used to cause exponential code-size blowup during LLVM passes. -// ignore-test FIXME #41696 -// min-llvm-version 3.9 #![feature(test)] From 4caa0b020f146e4504ab8ffdd52df29deaa49a09 Mon Sep 17 00:00:00 2001 From: Mark Simulacrum Date: Tue, 20 Jun 2017 18:04:36 -0600 Subject: [PATCH 12/19] Fixes bootstrapping with custom cargo/rustc. config.mk is now always read when parsing the configuration to prevent this from reoccurring in the future, hopefully. --- src/bootstrap/bin/main.rs | 8 +------- src/bootstrap/config.rs | 10 ++++++++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/bootstrap/bin/main.rs b/src/bootstrap/bin/main.rs index 5ca5ce1648f2f..5ef18b89841f0 100644 --- a/src/bootstrap/bin/main.rs +++ b/src/bootstrap/bin/main.rs @@ -26,12 +26,6 @@ use bootstrap::{Flags, Config, Build}; fn main() { let args = env::args().skip(1).collect::>(); let flags = Flags::parse(&args); - let mut config = Config::parse(&flags.build, flags.config.clone()); - - // compat with `./configure` while we're still using that - if std::fs::metadata("config.mk").is_ok() { - config.update_with_config_mk(); - } - + let config = Config::parse(&flags.build, flags.config.clone()); Build::new(flags, config).build(); } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index fd8aa320fb3d7..902cd0997a8ed 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::env; -use std::fs::File; +use std::fs::{self, File}; use std::io::prelude::*; use std::path::PathBuf; use std::process; @@ -410,6 +410,12 @@ impl Config { set(&mut config.rust_dist_src, t.src_tarball); } + + // compat with `./configure` while we're still using that + if fs::metadata("config.mk").is_ok() { + config.update_with_config_mk(); + } + return config } @@ -418,7 +424,7 @@ impl Config { /// While we still have `./configure` this implements the ability to decode /// that configuration into this. This isn't exactly a full-blown makefile /// parser, but hey it gets the job done! - pub fn update_with_config_mk(&mut self) { + fn update_with_config_mk(&mut self) { let mut config = String::new(); File::open("config.mk").unwrap().read_to_string(&mut config).unwrap(); for line in config.lines() { From 275f9a04af6191e3aee3852a5a1713130f635164 Mon Sep 17 00:00:00 2001 From: Stepan Koltsov Date: Wed, 21 Jun 2017 15:40:45 +0300 Subject: [PATCH 13/19] Better Debug for Args and ArgsOs Display actual args instead of two dots. --- src/libstd/env.rs | 18 ++++++++++++++++-- src/libstd/sys/redox/args.rs | 6 ++++++ src/libstd/sys/unix/args.rs | 6 ++++++ src/libstd/sys/windows/args.rs | 31 +++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/libstd/env.rs b/src/libstd/env.rs index 889ba81e77812..287c65e485379 100644 --- a/src/libstd/env.rs +++ b/src/libstd/env.rs @@ -712,7 +712,9 @@ impl DoubleEndedIterator for Args { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for Args { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.pad("Args { .. }") + f.debug_struct("Args") + .field("inner", &self.inner.inner.inner_debug()) + .finish() } } @@ -737,7 +739,9 @@ impl DoubleEndedIterator for ArgsOs { #[stable(feature = "std_debug", since = "1.16.0")] impl fmt::Debug for ArgsOs { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.pad("ArgsOs { .. }") + f.debug_struct("ArgsOs") + .field("inner", &self.inner.inner_debug()) + .finish() } } @@ -1085,4 +1089,14 @@ mod tests { r#""c:\te;st";c:\"#)); assert!(join_paths([r#"c:\te"st"#].iter().cloned()).is_err()); } + + #[test] + fn args_debug() { + assert_eq!( + format!("Args {{ inner: {:?} }}", args().collect::>()), + format!("{:?}", args())); + assert_eq!( + format!("ArgsOs {{ inner: {:?} }}", args_os().collect::>()), + format!("{:?}", args_os())); } +} diff --git a/src/libstd/sys/redox/args.rs b/src/libstd/sys/redox/args.rs index 212895d7b76fc..6e44ad705fe72 100644 --- a/src/libstd/sys/redox/args.rs +++ b/src/libstd/sys/redox/args.rs @@ -35,6 +35,12 @@ pub struct Args { _dont_send_or_sync_me: PhantomData<*mut ()>, } +impl Args { + pub fn inner_debug(&self) -> &[OsString] { + self.iter.as_slice() + } +} + impl Iterator for Args { type Item = OsString; fn next(&mut self) -> Option { self.iter.next() } diff --git a/src/libstd/sys/unix/args.rs b/src/libstd/sys/unix/args.rs index 6e35a472792c4..bbdcb5d36167e 100644 --- a/src/libstd/sys/unix/args.rs +++ b/src/libstd/sys/unix/args.rs @@ -35,6 +35,12 @@ pub struct Args { _dont_send_or_sync_me: PhantomData<*mut ()>, } +impl Args { + pub fn inner_debug(&self) -> &[OsString] { + self.iter.as_slice() + } +} + impl Iterator for Args { type Item = OsString; fn next(&mut self) -> Option { self.iter.next() } diff --git a/src/libstd/sys/windows/args.rs b/src/libstd/sys/windows/args.rs index aa61f9adb824d..4784633edc14d 100644 --- a/src/libstd/sys/windows/args.rs +++ b/src/libstd/sys/windows/args.rs @@ -16,6 +16,7 @@ use slice; use ops::Range; use ffi::OsString; use libc::{c_int, c_void}; +use fmt; pub unsafe fn init(_argc: isize, _argv: *const *const u8) { } @@ -39,6 +40,36 @@ pub struct Args { cur: *mut *mut u16, } +pub struct ArgsInnerDebug<'a> { + args: &'a Args, +} + +impl<'a> fmt::Debug for ArgsInnerDebug<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("[")?; + let mut first = true; + for i in self.args.range.clone() { + if !first { + f.write_str(", ")?; + } + first = false; + + // Here we do allocation which could be avoided. + fmt::Debug::fmt(&unsafe { os_string_from_ptr(*self.args.cur.offset(i)) }, f)?; + } + f.write_str("]")?; + Ok(()) + } +} + +impl Args { + pub fn inner_debug(&self) -> ArgsInnerDebug { + ArgsInnerDebug { + args: self + } + } +} + unsafe fn os_string_from_ptr(ptr: *mut u16) -> OsString { let mut len = 0; while *ptr.offset(len) != 0 { len += 1; } From 6e628bee95ab97f1dd9ab3cfb1e7b8a782564d8d Mon Sep 17 00:00:00 2001 From: Leonardo Yvens Date: Wed, 21 Jun 2017 10:47:29 -0300 Subject: [PATCH 14/19] Impl Clone for DefaultHasher --- src/libstd/collections/hash/map.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstd/collections/hash/map.rs b/src/libstd/collections/hash/map.rs index 8c4cbb66b454d..aef88e3d5636c 100644 --- a/src/libstd/collections/hash/map.rs +++ b/src/libstd/collections/hash/map.rs @@ -2384,7 +2384,7 @@ impl BuildHasher for RandomState { /// [`Hasher`]: ../../hash/trait.Hasher.html #[stable(feature = "hashmap_default_hasher", since = "1.13.0")] #[allow(deprecated)] -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct DefaultHasher(SipHasher13); impl DefaultHasher { From 201f06988fb77c9865b0ddc60a8d7b5701c6dbbe Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 15 Jun 2017 07:08:18 -0700 Subject: [PATCH 15/19] Integrate jobserver support to parallel codegen This commit integrates the `jobserver` crate into the compiler. The crate was previously integrated in to Cargo as part of rust-lang/cargo#4110. The purpose here is to two-fold: * Primarily the compiler can cooperate with Cargo on parallelism. When you run `cargo build -j4` then this'll make sure that the entire build process between Cargo/rustc won't use more than 4 cores, whereas today you'd get 4 rustc instances which may all try to spawn lots of threads. * Secondarily rustc/Cargo can now integrate with a foreign GNU `make` jobserver. This means that if you call cargo/rustc from `make` or another jobserver-compatible implementation it'll use foreign parallelism settings instead of creating new ones locally. As the number of parallel codegen instances in the compiler continues to grow over time with the advent of incremental compilation it's expected that this'll become more of a problem, so this is intended to nip concurrent concerns in the bud by having all the tools to cooperate! Note that while rustc has support for itself creating a jobserver it's far more likely that rustc will always use the jobserver configured by Cargo. Cargo today will now set a jobserver unconditionally for rustc to use. --- src/Cargo.lock | 6 + src/libcore/Cargo.toml | 3 + src/librustc/Cargo.toml | 1 + src/librustc/lib.rs | 1 + src/librustc/session/mod.rs | 28 +- src/librustc_trans/Cargo.toml | 2 + src/librustc_trans/back/link.rs | 29 +- src/librustc_trans/back/lto.rs | 97 +-- src/librustc_trans/back/write.rs | 576 +++++++++++------- src/librustc_trans/lib.rs | 2 + .../derive-no-std-not-supported.rs | 1 - .../compile-fail/asm-src-loc-codegen-units.rs | 5 +- .../compile-fail/auxiliary/issue-36881-aux.rs | 11 + .../auxiliary/lint_unused_extern_crate2.rs | 11 + .../auxiliary/lint_unused_extern_crate3.rs | 11 + .../auxiliary/lint_unused_extern_crate4.rs | 9 + src/test/compile-fail/issue-36881.rs | 6 +- .../compile-fail/lint-unused-extern-crate.rs | 22 +- .../run-pass/auxiliary/allocator-dummy.rs | 4 +- 19 files changed, 514 insertions(+), 311 deletions(-) create mode 100644 src/test/compile-fail/auxiliary/issue-36881-aux.rs create mode 100644 src/test/compile-fail/auxiliary/lint_unused_extern_crate2.rs create mode 100644 src/test/compile-fail/auxiliary/lint_unused_extern_crate3.rs create mode 100644 src/test/compile-fail/auxiliary/lint_unused_extern_crate4.rs diff --git a/src/Cargo.lock b/src/Cargo.lock index 6d7fcb71efa00..7e969ce9b8462 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -293,6 +293,9 @@ dependencies = [ [[package]] name = "core" version = "0.0.0" +dependencies = [ + "rand 0.0.0", +] [[package]] name = "crates-io" @@ -1099,6 +1102,7 @@ dependencies = [ "flate2 0.2.19 (registry+https://github.com/rust-lang/crates.io-index)", "fmt_macros 0.0.0", "graphviz 0.0.0", + "jobserver 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_back 0.0.0", @@ -1394,8 +1398,10 @@ dependencies = [ name = "rustc_trans" version = "0.0.0" dependencies = [ + "crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", "flate2 0.2.19 (registry+https://github.com/rust-lang/crates.io-index)", "gcc 0.3.51 (registry+https://github.com/rust-lang/crates.io-index)", + "jobserver 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "rustc 0.0.0", diff --git a/src/libcore/Cargo.toml b/src/libcore/Cargo.toml index 5af63aa970f2c..178df02ccdde3 100644 --- a/src/libcore/Cargo.toml +++ b/src/libcore/Cargo.toml @@ -9,6 +9,9 @@ path = "lib.rs" test = false bench = false +[dev-dependencies] +rand = { path = "../librand" } + [[test]] name = "coretests" path = "../libcore/tests/lib.rs" diff --git a/src/librustc/Cargo.toml b/src/librustc/Cargo.toml index 3d59a4eb882da..89169548bbb95 100644 --- a/src/librustc/Cargo.toml +++ b/src/librustc/Cargo.toml @@ -12,6 +12,7 @@ crate-type = ["dylib"] arena = { path = "../libarena" } fmt_macros = { path = "../libfmt_macros" } graphviz = { path = "../libgraphviz" } +jobserver = "0.1" log = "0.3" owning_ref = "0.3.3" rustc_back = { path = "../librustc_back" } diff --git a/src/librustc/lib.rs b/src/librustc/lib.rs index 77a43c5319c5c..b81c56e5ee8e4 100644 --- a/src/librustc/lib.rs +++ b/src/librustc/lib.rs @@ -61,6 +61,7 @@ extern crate rustc_errors as errors; #[macro_use] extern crate syntax; extern crate syntax_pos; #[macro_use] #[no_link] extern crate rustc_bitflags; +extern crate jobserver; extern crate serialize as rustc_serialize; // used by deriving diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs index 827fa72f03404..70c07982f83cb 100644 --- a/src/librustc/session/mod.rs +++ b/src/librustc/session/mod.rs @@ -38,14 +38,16 @@ use syntax_pos::{Span, MultiSpan}; use rustc_back::{LinkerFlavor, PanicStrategy}; use rustc_back::target::Target; use rustc_data_structures::flock; +use jobserver::Client; -use std::path::{Path, PathBuf}; use std::cell::{self, Cell, RefCell}; use std::collections::HashMap; use std::env; +use std::fmt; use std::io::Write; +use std::path::{Path, PathBuf}; use std::rc::Rc; -use std::fmt; +use std::sync::{Once, ONCE_INIT}; use std::time::Duration; mod code_stats; @@ -134,6 +136,10 @@ pub struct Session { pub print_fuel_crate: Option, /// Always set to zero and incremented so that we can print fuel expended by a crate. pub print_fuel: Cell, + + /// Loaded up early on in the initialization of this `Session` to avoid + /// false positives about a job server in our environment. + pub jobserver_from_env: Option, } pub struct PerfStats { @@ -697,6 +703,24 @@ pub fn build_session_(sopts: config::Options, print_fuel_crate: print_fuel_crate, print_fuel: print_fuel, out_of_fuel: Cell::new(false), + + // Note that this is unsafe because it may misinterpret file descriptors + // on Unix as jobserver file descriptors. We hopefully execute this near + // the beginning of the process though to ensure we don't get false + // positives, or in other words we try to execute this before we open + // any file descriptors ourselves. + // + // Also note that we stick this in a global because there could be + // multiple `Session` instances in this process, and the jobserver is + // per-process. + jobserver_from_env: unsafe { + static mut GLOBAL_JOBSERVER: *mut Option = 0 as *mut _; + static INIT: Once = ONCE_INIT; + INIT.call_once(|| { + GLOBAL_JOBSERVER = Box::into_raw(Box::new(Client::from_env())); + }); + (*GLOBAL_JOBSERVER).clone() + }, }; sess diff --git a/src/librustc_trans/Cargo.toml b/src/librustc_trans/Cargo.toml index a36c561371187..86590bff4ff7d 100644 --- a/src/librustc_trans/Cargo.toml +++ b/src/librustc_trans/Cargo.toml @@ -10,7 +10,9 @@ crate-type = ["dylib"] test = false [dependencies] +crossbeam = "0.2" flate2 = "0.2" +jobserver = "0.1.5" log = "0.3" owning_ref = "0.3.3" rustc = { path = "../librustc" } diff --git a/src/librustc_trans/back/link.rs b/src/librustc_trans/back/link.rs index 7cd1ef772981e..1f88f90dbbb28 100644 --- a/src/librustc_trans/back/link.rs +++ b/src/librustc_trans/back/link.rs @@ -329,34 +329,38 @@ pub fn filename_for_input(sess: &Session, } pub fn each_linked_rlib(sess: &Session, - f: &mut FnMut(CrateNum, &Path)) { + f: &mut FnMut(CrateNum, &Path)) -> Result<(), String> { let crates = sess.cstore.used_crates(LinkagePreference::RequireStatic).into_iter(); let fmts = sess.dependency_formats.borrow(); let fmts = fmts.get(&config::CrateTypeExecutable) .or_else(|| fmts.get(&config::CrateTypeStaticlib)) .or_else(|| fmts.get(&config::CrateTypeCdylib)) .or_else(|| fmts.get(&config::CrateTypeProcMacro)); - let fmts = fmts.unwrap_or_else(|| { - bug!("could not find formats for rlibs"); - }); + let fmts = match fmts { + Some(f) => f, + None => return Err(format!("could not find formats for rlibs")) + }; for (cnum, path) in crates { - match fmts[cnum.as_usize() - 1] { - Linkage::NotLinked | Linkage::IncludedFromDylib => continue, - _ => {} + match fmts.get(cnum.as_usize() - 1) { + Some(&Linkage::NotLinked) | + Some(&Linkage::IncludedFromDylib) => continue, + Some(_) => {} + None => return Err(format!("could not find formats for rlibs")) } let name = sess.cstore.crate_name(cnum).clone(); let path = match path { LibSource::Some(p) => p, LibSource::MetadataOnly => { - sess.fatal(&format!("could not find rlib for: `{}`, found rmeta (metadata) file", - name)); + return Err(format!("could not find rlib for: `{}`, found rmeta (metadata) file", + name)) } LibSource::None => { - sess.fatal(&format!("could not find rlib for: `{}`", name)); + return Err(format!("could not find rlib for: `{}`", name)) } }; f(cnum, &path); } + Ok(()) } fn out_filename(sess: &Session, @@ -669,7 +673,7 @@ fn link_staticlib(sess: &Session, objects: &[PathBuf], out_filename: &Path, let mut ab = link_rlib(sess, None, objects, out_filename, tempdir); let mut all_native_libs = vec![]; - each_linked_rlib(sess, &mut |cnum, path| { + let res = each_linked_rlib(sess, &mut |cnum, path| { let name = sess.cstore.crate_name(cnum); let native_libs = sess.cstore.native_libraries(cnum); @@ -694,6 +698,9 @@ fn link_staticlib(sess: &Session, objects: &[PathBuf], out_filename: &Path, all_native_libs.extend(sess.cstore.native_libraries(cnum)); }); + if let Err(e) = res { + sess.fatal(&e); + } ab.update_symbols(); ab.build(); diff --git a/src/librustc_trans/back/lto.rs b/src/librustc_trans/back/lto.rs index a5f9a41470de4..906815583bf09 100644 --- a/src/librustc_trans/back/lto.rs +++ b/src/librustc_trans/back/lto.rs @@ -10,15 +10,16 @@ use back::link; use back::write; -use back::symbol_export::{self, ExportedSymbols}; -use rustc::session::{self, config}; +use back::symbol_export; +use rustc::session::config; +use errors::FatalError; use llvm; use llvm::archive_ro::ArchiveRO; use llvm::{ModuleRef, TargetMachineRef, True, False}; use rustc::util::common::time; use rustc::util::common::path2cstr; use rustc::hir::def_id::LOCAL_CRATE; -use back::write::{ModuleConfig, with_llvm_pmb}; +use back::write::{ModuleConfig, with_llvm_pmb, CodegenContext}; use libc; use flate2::read::ZlibDecoder; @@ -39,30 +40,31 @@ pub fn crate_type_allows_lto(crate_type: config::CrateType) -> bool { } } -pub fn run(sess: &session::Session, +pub fn run(cgcx: &CodegenContext, llmod: ModuleRef, tm: TargetMachineRef, - exported_symbols: &ExportedSymbols, config: &ModuleConfig, - temp_no_opt_bc_filename: &Path) { - if sess.opts.cg.prefer_dynamic { - sess.struct_err("cannot prefer dynamic linking when performing LTO") + temp_no_opt_bc_filename: &Path) -> Result<(), FatalError> { + let handler = cgcx.handler; + if cgcx.opts.cg.prefer_dynamic { + handler.struct_err("cannot prefer dynamic linking when performing LTO") .note("only 'staticlib', 'bin', and 'cdylib' outputs are \ supported with LTO") .emit(); - sess.abort_if_errors(); + return Err(FatalError) } // Make sure we actually can run LTO - for crate_type in sess.crate_types.borrow().iter() { + for crate_type in cgcx.crate_types.iter() { if !crate_type_allows_lto(*crate_type) { - sess.fatal("lto can only be run for executables, cdylibs and \ - static library outputs"); + let e = handler.fatal("lto can only be run for executables, cdylibs and \ + static library outputs"); + return Err(e) } } let export_threshold = - symbol_export::crates_export_threshold(&sess.crate_types.borrow()); + symbol_export::crates_export_threshold(&cgcx.crate_types); let symbol_filter = &|&(ref name, level): &(String, _)| { if symbol_export::is_below_threshold(level, export_threshold) { @@ -74,7 +76,7 @@ pub fn run(sess: &session::Session, } }; - let mut symbol_white_list: Vec = exported_symbols + let mut symbol_white_list: Vec = cgcx.exported_symbols .exported_symbols(LOCAL_CRATE) .iter() .filter_map(symbol_filter) @@ -83,16 +85,11 @@ pub fn run(sess: &session::Session, // For each of our upstream dependencies, find the corresponding rlib and // load the bitcode from the archive. Then merge it into the current LLVM // module that we've got. - link::each_linked_rlib(sess, &mut |cnum, path| { - // `#![no_builtins]` crates don't participate in LTO. - if sess.cstore.is_no_builtins(cnum) { - return; - } - + for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() { symbol_white_list.extend( - exported_symbols.exported_symbols(cnum) - .iter() - .filter_map(symbol_filter)); + cgcx.exported_symbols.exported_symbols(cnum) + .iter() + .filter_map(symbol_filter)); let archive = ArchiveRO::open(&path).expect("wanted an rlib"); let bytecodes = archive.iter().filter_map(|child| { @@ -102,7 +99,7 @@ pub fn run(sess: &session::Session, let bc_encoded = data.data(); let bc_decoded = if is_versioned_bytecode_format(bc_encoded) { - time(sess.time_passes(), &format!("decode {}", name), || { + time(cgcx.time_passes, &format!("decode {}", name), || { // Read the version let version = extract_bytecode_format_version(bc_encoded); @@ -117,17 +114,19 @@ pub fn run(sess: &session::Session, let res = ZlibDecoder::new(compressed_data) .read_to_end(&mut inflated); if res.is_err() { - sess.fatal(&format!("failed to decompress bc of `{}`", - name)) + let msg = format!("failed to decompress bc of `{}`", + name); + Err(handler.fatal(&msg)) + } else { + Ok(inflated) } - inflated } else { - sess.fatal(&format!("Unsupported bytecode format version {}", - version)) + Err(handler.fatal(&format!("Unsupported bytecode format version {}", + version))) } - }) + })? } else { - time(sess.time_passes(), &format!("decode {}", name), || { + time(cgcx.time_passes, &format!("decode {}", name), || { // the object must be in the old, pre-versioning format, so // simply inflate everything and let LLVM decide if it can // make sense of it @@ -135,26 +134,29 @@ pub fn run(sess: &session::Session, let res = ZlibDecoder::new(bc_encoded) .read_to_end(&mut inflated); if res.is_err() { - sess.fatal(&format!("failed to decompress bc of `{}`", - name)) + let msg = format!("failed to decompress bc of `{}`", + name); + Err(handler.fatal(&msg)) + } else { + Ok(inflated) } - inflated - }) + })? }; let ptr = bc_decoded.as_ptr(); debug!("linking {}", name); - time(sess.time_passes(), &format!("ll link {}", name), || unsafe { - if !llvm::LLVMRustLinkInExternalBitcode(llmod, - ptr as *const libc::c_char, - bc_decoded.len() as libc::size_t) { - write::llvm_err(sess.diagnostic(), - format!("failed to load bc of `{}`", - name)); + time(cgcx.time_passes, &format!("ll link {}", name), || unsafe { + if llvm::LLVMRustLinkInExternalBitcode(llmod, + ptr as *const libc::c_char, + bc_decoded.len() as libc::size_t) { + Ok(()) + } else { + let msg = format!("failed to load bc of `{}`", name); + Err(write::llvm_err(handler, msg)) } - }); + })?; } - }); + } // Internalize everything but the exported symbols of the current module let arr: Vec<*const libc::c_char> = symbol_white_list.iter() @@ -167,13 +169,13 @@ pub fn run(sess: &session::Session, arr.len() as libc::size_t); } - if sess.no_landing_pads() { + if cgcx.no_landing_pads { unsafe { llvm::LLVMRustMarkAllFunctionsNounwind(llmod); } } - if sess.opts.cg.save_temps { + if cgcx.opts.cg.save_temps { let cstr = path2cstr(temp_no_opt_bc_filename); unsafe { llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr()); @@ -203,12 +205,13 @@ pub fn run(sess: &session::Session, assert!(!pass.is_null()); llvm::LLVMRustAddPass(pm, pass); - time(sess.time_passes(), "LTO passes", || + time(cgcx.time_passes, "LTO passes", || llvm::LLVMRunPassManager(pm, llmod)); llvm::LLVMDisposePassManager(pm); } debug!("lto done"); + Ok(()) } fn is_versioned_bytecode_format(bc: &[u8]) -> bool { diff --git a/src/librustc_trans/back/write.rs b/src/librustc_trans/back/write.rs index 4871d638d1294..549cb2567cfbb 100644 --- a/src/librustc_trans/back/write.rs +++ b/src/librustc_trans/back/write.rs @@ -9,7 +9,7 @@ // except according to those terms. use back::lto; -use back::link::{get_linker, remove}; +use back::link::{self, get_linker, remove}; use back::symbol_export::ExportedSymbols; use rustc_incremental::{save_trans_partition, in_incr_comp_dir}; use rustc::session::config::{self, OutputFilenames, OutputType, OutputTypes, Passes, SomePasses, @@ -19,21 +19,24 @@ use llvm; use llvm::{ModuleRef, TargetMachineRef, PassManagerRef, DiagnosticInfoRef, ContextRef}; use llvm::SMDiagnosticRef; use {CrateTranslation, ModuleLlvm, ModuleSource, ModuleTranslation}; +use rustc::hir::def_id::CrateNum; use rustc::util::common::{time, time_depth, set_time_depth, path2cstr}; use rustc::util::fs::link_or_copy; -use errors::{self, Handler, Level, DiagnosticBuilder}; +use errors::{self, Handler, Level, DiagnosticBuilder, FatalError}; use errors::emitter::Emitter; +use syntax::ext::hygiene::Mark; use syntax_pos::MultiSpan; use context::{is_pie_binary, get_reloc_model}; +use jobserver::{Client, Acquired}; +use crossbeam::{scope, Scope}; use std::cmp; use std::ffi::CString; use std::fs; +use std::io; use std::path::{Path, PathBuf}; use std::str; -use std::sync::{Arc, Mutex}; -use std::sync::mpsc::channel; -use std::thread; +use std::sync::mpsc::{channel, Sender}; use libc::{c_uint, c_void}; pub const RELOC_MODEL_ARGS : [(&'static str, llvm::RelocMode); 7] = [ @@ -54,10 +57,10 @@ pub const CODE_GEN_MODEL_ARGS : [(&'static str, llvm::CodeModel); 5] = [ ("large", llvm::CodeModel::Large), ]; -pub fn llvm_err(handler: &errors::Handler, msg: String) -> ! { +pub fn llvm_err(handler: &errors::Handler, msg: String) -> FatalError { match llvm::last_error() { - Some(err) => panic!(handler.fatal(&format!("{}: {}", msg, err))), - None => panic!(handler.fatal(&msg)), + Some(err) => handler.fatal(&format!("{}: {}", msg, err)), + None => handler.fatal(&msg), } } @@ -67,73 +70,16 @@ pub fn write_output_file( pm: llvm::PassManagerRef, m: ModuleRef, output: &Path, - file_type: llvm::FileType) { + file_type: llvm::FileType) -> Result<(), FatalError> { unsafe { let output_c = path2cstr(output); let result = llvm::LLVMRustWriteOutputFile( target, pm, m, output_c.as_ptr(), file_type); if result.into_result().is_err() { - llvm_err(handler, format!("could not write output to {}", output.display())); - } - } -} - - -struct Diagnostic { - msg: String, - code: Option, - lvl: Level, -} - -// We use an Arc instead of just returning a list of diagnostics from the -// child thread because we need to make sure that the messages are seen even -// if the child thread panics (for example, when `fatal` is called). -#[derive(Clone)] -struct SharedEmitter { - buffer: Arc>>, -} - -impl SharedEmitter { - fn new() -> SharedEmitter { - SharedEmitter { - buffer: Arc::new(Mutex::new(Vec::new())), - } - } - - fn dump(&mut self, handler: &Handler) { - let mut buffer = self.buffer.lock().unwrap(); - for diag in &*buffer { - match diag.code { - Some(ref code) => { - handler.emit_with_code(&MultiSpan::new(), - &diag.msg, - &code, - diag.lvl); - }, - None => { - handler.emit(&MultiSpan::new(), - &diag.msg, - diag.lvl); - }, - } - } - buffer.clear(); - } -} - -impl Emitter for SharedEmitter { - fn emit(&mut self, db: &DiagnosticBuilder) { - self.buffer.lock().unwrap().push(Diagnostic { - msg: db.message(), - code: db.code.clone(), - lvl: db.level, - }); - for child in &db.children { - self.buffer.lock().unwrap().push(Diagnostic { - msg: child.message(), - code: None, - lvl: child.level, - }); + let msg = format!("could not write output to {}", output.display()); + Err(llvm_err(handler, msg)) + } else { + Ok(()) } } } @@ -231,9 +177,9 @@ pub fn create_target_machine(sess: &Session) -> TargetMachineRef { }; if tm.is_null() { - llvm_err(sess.diagnostic(), - format!("Could not create LLVM TargetMachine for triple: {}", - triple).to_string()); + let msg = format!("Could not create LLVM TargetMachine for triple: {}", + triple); + panic!(llvm_err(sess.diagnostic(), msg)); } else { return tm; }; @@ -333,36 +279,28 @@ impl ModuleConfig { } /// Additional resources used by optimize_and_codegen (not module specific) -struct CodegenContext<'a> { - // Extra resources used for LTO: (sess, reachable). This will be `None` - // when running in a worker thread. - lto_ctxt: Option<(&'a Session, &'a ExportedSymbols)>, +pub struct CodegenContext<'a> { + // Resouces needed when running LTO + pub time_passes: bool, + pub lto: bool, + pub no_landing_pads: bool, + pub exported_symbols: &'a ExportedSymbols, + pub opts: &'a config::Options, + pub crate_types: Vec, + pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>, // Handler to use for diagnostics produced during codegen. - handler: &'a Handler, + pub handler: &'a Handler, // LLVM passes added by plugins. - plugin_passes: Vec, + pub plugin_passes: Vec, // LLVM optimizations for which we want to print remarks. - remark: Passes, + pub remark: Passes, // Worker thread number - worker: usize, + pub worker: usize, // The incremental compilation session directory, or None if we are not // compiling incrementally - incr_comp_session_dir: Option -} - -impl<'a> CodegenContext<'a> { - fn new_with_session(sess: &'a Session, - exported_symbols: &'a ExportedSymbols) - -> CodegenContext<'a> { - CodegenContext { - lto_ctxt: Some((sess, exported_symbols)), - handler: sess.diagnostic(), - plugin_passes: sess.plugin_llvm_passes.borrow().clone(), - remark: sess.opts.cg.remark.clone(), - worker: 0, - incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()) - } - } + pub incr_comp_session_dir: Option, + // Channel back to the main control thread to send messages to + pub tx: Sender, } struct HandlerFreeVars<'a> { @@ -373,22 +311,7 @@ struct HandlerFreeVars<'a> { unsafe extern "C" fn report_inline_asm<'a, 'b>(cgcx: &'a CodegenContext<'a>, msg: &'b str, cookie: c_uint) { - use syntax::ext::hygiene::Mark; - - match cgcx.lto_ctxt { - Some((sess, _)) => { - match Mark::from_u32(cookie).expn_info() { - Some(ei) => sess.span_err(ei.call_site, msg), - None => sess.err(msg), - }; - } - - None => { - cgcx.handler.struct_err(msg) - .note("build without -C codegen-units for more exact errors") - .emit(); - } - } + drop(cgcx.tx.send(Message::InlineAsmError(cookie as u32, msg.to_string()))); } unsafe extern "C" fn inline_asm_handler(diag: SMDiagnosticRef, @@ -437,7 +360,9 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, mtrans: ModuleTranslation, mllvm: ModuleLlvm, config: ModuleConfig, - output_names: OutputFilenames) { + output_names: OutputFilenames) + -> Result<(), FatalError> +{ let llmod = mllvm.llmod; let llcx = mllvm.llcx; let tm = config.tm; @@ -525,25 +450,21 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, llvm::LLVMDisposePassManager(fpm); llvm::LLVMDisposePassManager(mpm); - match cgcx.lto_ctxt { - Some((sess, exported_symbols)) if sess.lto() => { - time(sess.time_passes(), "all lto passes", || { - let temp_no_opt_bc_filename = - output_names.temp_path_ext("no-opt.lto.bc", module_name); - lto::run(sess, - llmod, - tm, - exported_symbols, - &config, - &temp_no_opt_bc_filename); - }); - if config.emit_lto_bc { - let out = output_names.temp_path_ext("lto.bc", module_name); - let out = path2cstr(&out); - llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()); - } - }, - _ => {}, + if cgcx.lto { + time(cgcx.time_passes, "all lto passes", || { + let temp_no_opt_bc_filename = + output_names.temp_path_ext("no-opt.lto.bc", module_name); + lto::run(cgcx, + llmod, + tm, + &config, + &temp_no_opt_bc_filename) + })?; + if config.emit_lto_bc { + let out = output_names.temp_path_ext("lto.bc", module_name); + let out = path2cstr(&out); + llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()); + } } } @@ -555,16 +476,16 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, // pass manager passed to the closure should be ensured to not // escape the closure itself, and the manager should only be // used once. - unsafe fn with_codegen(tm: TargetMachineRef, - llmod: ModuleRef, - no_builtins: bool, - f: F) where - F: FnOnce(PassManagerRef), + unsafe fn with_codegen(tm: TargetMachineRef, + llmod: ModuleRef, + no_builtins: bool, + f: F) -> R + where F: FnOnce(PassManagerRef) -> R, { let cpm = llvm::LLVMCreatePassManager(); llvm::LLVMRustAddAnalysisPasses(tm, cpm, llmod); llvm::LLVMRustAddLibraryInfo(cpm, llmod, no_builtins); - f(cpm); + f(cpm) } // Change what we write and cleanup based on whether obj files are @@ -584,7 +505,8 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, llvm::LLVMWriteBitcodeToFile(llmod, bc_out_c.as_ptr()); } - time(config.time_passes, &format!("codegen passes [{}]", cgcx.worker), || { + time(config.time_passes, &format!("codegen passes [{}]", cgcx.worker), + || -> Result<(), FatalError> { if config.emit_ir { let out = output_names.temp_path(OutputType::LlvmAssembly, module_name); let out = path2cstr(&out); @@ -607,8 +529,8 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, }; with_codegen(tm, llmod, config.no_builtins, |cpm| { write_output_file(cgcx.handler, tm, cpm, llmod, &path, - llvm::FileType::AssemblyFile); - }); + llvm::FileType::AssemblyFile) + })?; if config.emit_obj { llvm::LLVMDisposeModule(llmod); } @@ -617,10 +539,12 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, if write_obj { with_codegen(tm, llmod, config.no_builtins, |cpm| { write_output_file(cgcx.handler, tm, cpm, llmod, &obj_out, - llvm::FileType::ObjectFile); - }); + llvm::FileType::ObjectFile) + })?; } - }); + + Ok(()) + })?; if copy_bc_to_obj { debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out); @@ -637,6 +561,7 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, } llvm::LLVMRustDisposeTargetMachine(tm); + Ok(()) } @@ -781,19 +706,16 @@ pub fn run_passes(sess: &Session, dump_incremental_data(&trans); } - // Process the work items, optionally using worker threads. - // NOTE: We are hardcoding a limit of worker threads for now. With - // incremental compilation we can run into situations where we would - // open hundreds of threads otherwise -- which can make things slower - // if things don't fit into memory anymore, or can cause the compiler - // to crash because of too many open file handles. See #39280 for - // some discussion on how to improve this in the future. - let num_workers = cmp::min(work_items.len() - 1, 32); - if num_workers <= 1 { - run_work_singlethreaded(sess, &trans.exported_symbols, work_items); - } else { - run_work_multithreaded(sess, work_items, num_workers); - } + let client = sess.jobserver_from_env.clone().unwrap_or_else(|| { + // Pick a "reasonable maximum" if we don't otherwise have a jobserver in + // our environment, capping out at 32 so we don't take everything down + // by hogging the process run queue. + let num_workers = cmp::min(work_items.len() - 1, 32); + Client::new(num_workers).expect("failed to create jobserver") + }); + scope(|scope| { + execute_work(sess, work_items, client, &trans.exported_symbols, scope); + }); // If in incr. comp. mode, preserve the `.o` files for potential re-use for mtrans in trans.modules.iter() { @@ -995,8 +917,9 @@ fn build_work_item(sess: &Session, } } -fn execute_work_item(cgcx: &CodegenContext, - work_item: WorkItem) { +fn execute_work_item(cgcx: &CodegenContext, work_item: WorkItem) + -> Result<(), FatalError> +{ unsafe { match work_item.mtrans.source { ModuleSource::Translated(mllvm) => { @@ -1005,7 +928,7 @@ fn execute_work_item(cgcx: &CodegenContext, work_item.mtrans, mllvm, work_item.config, - work_item.output_names); + work_item.output_names)?; } ModuleSource::Preexisting(wp) => { let incr_comp_session_dir = cgcx.incr_comp_session_dir @@ -1033,94 +956,283 @@ fn execute_work_item(cgcx: &CodegenContext, } } } + + Ok(()) } -fn run_work_singlethreaded(sess: &Session, - exported_symbols: &ExportedSymbols, - work_items: Vec) { - let cgcx = CodegenContext::new_with_session(sess, exported_symbols); +pub enum Message { + Token(io::Result), + Diagnostic(Diagnostic), + Done { success: bool }, + InlineAsmError(u32, String), + AbortIfErrors, +} - // Since we're running single-threaded, we can pass the session to - // the proc, allowing `optimize_and_codegen` to perform LTO. - for work in work_items.into_iter().rev() { - execute_work_item(&cgcx, work); - } +pub struct Diagnostic { + msg: String, + code: Option, + lvl: Level, } -fn run_work_multithreaded(sess: &Session, - work_items: Vec, - num_workers: usize) { - assert!(num_workers > 0); - - // Run some workers to process the work items. - let work_items_arc = Arc::new(Mutex::new(work_items)); - let mut diag_emitter = SharedEmitter::new(); - let mut futures = Vec::with_capacity(num_workers); - - for i in 0..num_workers { - let work_items_arc = work_items_arc.clone(); - let diag_emitter = diag_emitter.clone(); - let plugin_passes = sess.plugin_llvm_passes.borrow().clone(); - let remark = sess.opts.cg.remark.clone(); - - let (tx, rx) = channel(); - let mut tx = Some(tx); - futures.push(rx); - - let incr_comp_session_dir = sess.incr_comp_session_dir_opt().map(|r| r.clone()); - - let depth = time_depth(); - thread::Builder::new().name(format!("codegen-{}", i)).spawn(move || { - set_time_depth(depth); - - let diag_handler = Handler::with_emitter(true, false, box diag_emitter); - - // Must construct cgcx inside the proc because it has non-Send - // fields. - let cgcx = CodegenContext { - lto_ctxt: None, - handler: &diag_handler, - plugin_passes: plugin_passes, - remark: remark, - worker: i, - incr_comp_session_dir: incr_comp_session_dir - }; +fn execute_work<'a>(sess: &'a Session, + mut work_items: Vec, + jobserver: Client, + exported_symbols: &'a ExportedSymbols, + scope: &Scope<'a>) { + let (tx, rx) = channel(); + let tx2 = tx.clone(); + + // First up, convert our jobserver into a helper thread so we can use normal + // mpsc channels to manage our messages and such. Once we've got the helper + // thread then request `n-1` tokens because all of our work items are ready + // to go. + // + // Note that the `n-1` is here because we ourselves have a token (our + // process) and we'll use that token to execute at least one unit of work. + // + // After we've requested all these tokens then we'll, when we can, get + // tokens on `rx` above which will get managed in the main loop below. + let helper = jobserver.into_helper_thread(move |token| { + drop(tx2.send(Message::Token(token))); + }).expect("failed to spawn helper thread"); + for _ in 0..work_items.len() - 1 { + helper.request_token(); + } + + // This is the "main loop" of parallel work happening for parallel codegen. + // It's here that we manage parallelism, schedule work, and work with + // messages coming from clients. + // + // Our channel `rx` created above is a channel of messages coming from our + // various worker threads. This includes the jobserver helper thread above + // as well as the work we'll spawn off here. Each turn of this loop starts + // off by trying to spawn as much work as possible. After we've done that we + // then wait for an event and dispatch accordingly once the event is + // received. We're only done once all our work items have been drained and + // nothing is running, at which point we return back up the stack. + // + // ## Parallelism management + // + // It's worth also touching on the management of parallelism here. We don't + // want to just spawn a thread per work item because while that's optimal + // parallelism it may overload a system with too many threads or violate our + // configuration for the maximum amount of cpu to use for this process. To + // manage this we use the `jobserver` crate. + // + // Job servers are an artifact of GNU make and are used to manage + // parallelism between processes. A jobserver is a glorified IPC semaphore + // basically. Whenever we want to run some work we acquire the semaphore, + // and whenever we're done with that work we release the semaphore. In this + // manner we can ensure that the maximum number of parallel workers is + // capped at any one point in time. + // + // The jobserver protocol is a little unique, however. We, as a running + // process, already have an ephemeral token assigned to us. We're not going + // to be doing any productive work in this thread though so we're going to + // give this token to a worker thread (there's no actual token to give, this + // is just conceptually). As a result you'll see a few `+1` and `-1` + // instances below, and it's about working with this ephemeral token. + // + // To acquire tokens we have our `helper` thread above which is just in a + // loop acquiring tokens and sending them to us. We then store all tokens + // locally in a `tokens` vector once they're acquired. Currently we don't + // literally send a token to a worker thread to assist with management of + // our "ephemeral token". + // + // As a result, our "spawn as much work as possible" basically means that we + // fill up the `running` counter up to the limit of the `tokens` list. + // Whenever we get a new token this'll mean a new unit of work is spawned, + // and then whenever a unit of work finishes we relinquish a token, if we + // had one, to maybe get re-acquired later. + // + // Note that there's a race which may mean that we acquire more tokens than + // we originally anticipated. For example let's say we have 2 units of work. + // First we request one token from the helper thread and then we + // immediately spawn one unit of work with our ephemeral token after. We may + // then finish the first piece of work before the token is acquired, but we + // can continue to spawn the second piece of work with our ephemeral token. + // Before that work finishes, however, we may acquire a token. In that case + // we actually wastefully acquired the token, so we relinquish it back to + // the jobserver. + let mut tokens = Vec::new(); + let mut running = 0; + while work_items.len() > 0 || running > 0 { + + // Spin up what work we can, only doing this while we've got available + // parallelism slots and work left to spawn. + while work_items.len() > 0 && running < tokens.len() + 1 { + let item = work_items.pop().unwrap(); + let index = work_items.len(); + spawn_work(sess, exported_symbols, scope, tx.clone(), item, index); + running += 1; + } + + // Relinquish accidentally acquired extra tokens + tokens.truncate(running.saturating_sub(1)); + + match rx.recv().unwrap() { + // Save the token locally and the next turn of the loop will use + // this to spawn a new unit of work, or it may get dropped + // immediately if we have no more work to spawn. + Message::Token(token) => { + tokens.push(token.expect("failed to acquire jobserver token")); + } - loop { - // Avoid holding the lock for the entire duration of the match. - let maybe_work = work_items_arc.lock().unwrap().pop(); - match maybe_work { - Some(work) => { - execute_work_item(&cgcx, work); + // If a thread exits successfully then we drop a token associated + // with that worker and update our `running` count. We may later + // re-acquire a token to continue running more work. We may also not + // actually drop a token here if the worker was running with an + // "ephemeral token" + // + // Note that if the thread failed that means it panicked, so we + // abort immediately. + Message::Done { success: true } => { + drop(tokens.pop()); + running -= 1; + } + Message::Done { success: false } => { + sess.fatal("aborting due to worker thread panic"); + } - // Make sure to fail the worker so the main thread can - // tell that there were errors. - cgcx.handler.abort_if_errors(); + // Our worker wants us to emit an error message, so get ahold of our + // `sess` and print it out + Message::Diagnostic(diag) => { + let handler = sess.diagnostic(); + match diag.code { + Some(ref code) => { + handler.emit_with_code(&MultiSpan::new(), + &diag.msg, + &code, + diag.lvl); + } + None => { + handler.emit(&MultiSpan::new(), + &diag.msg, + diag.lvl); } - None => break, + } + } + Message::InlineAsmError(cookie, msg) => { + match Mark::from_u32(cookie).expn_info() { + Some(ei) => sess.span_err(ei.call_site, &msg), + None => sess.err(&msg), } } - tx.take().unwrap().send(()).unwrap(); - }).unwrap(); + // Sent to us after a worker sends us a batch of error messages, and + // it's the point at which we check for errors. + Message::AbortIfErrors => sess.diagnostic().abort_if_errors(), + } } - let mut panicked = false; - for rx in futures { - match rx.recv() { - Ok(()) => {}, - Err(_) => { - panicked = true; - }, + // Just in case, check this on the way out. + sess.diagnostic().abort_if_errors(); +} + +struct SharedEmitter { + tx: Sender, +} + +impl Emitter for SharedEmitter { + fn emit(&mut self, db: &DiagnosticBuilder) { + drop(self.tx.send(Message::Diagnostic(Diagnostic { + msg: db.message(), + code: db.code.clone(), + lvl: db.level, + }))); + for child in &db.children { + drop(self.tx.send(Message::Diagnostic(Diagnostic { + msg: child.message(), + code: None, + lvl: child.level, + }))); } - // Display any new diagnostics. - diag_emitter.dump(sess.diagnostic()); - } - if panicked { - sess.fatal("aborting due to worker thread panic"); + drop(self.tx.send(Message::AbortIfErrors)); } } +fn spawn_work<'a>(sess: &'a Session, + exported_symbols: &'a ExportedSymbols, + scope: &Scope<'a>, + tx: Sender, + work: WorkItem, + idx: usize) { + let plugin_passes = sess.plugin_llvm_passes.borrow().clone(); + let remark = sess.opts.cg.remark.clone(); + let incr_comp_session_dir = sess.incr_comp_session_dir_opt().map(|r| r.clone()); + let depth = time_depth(); + let lto = sess.lto(); + let crate_types = sess.crate_types.borrow().clone(); + let mut each_linked_rlib_for_lto = Vec::new(); + drop(link::each_linked_rlib(sess, &mut |cnum, path| { + // `#![no_builtins]` crates don't participate in LTO. + if sess.cstore.is_no_builtins(cnum) { + return + } + each_linked_rlib_for_lto.push((cnum, path.to_path_buf())); + })); + let time_passes = sess.time_passes(); + let no_landing_pads = sess.no_landing_pads(); + let opts = &sess.opts; + + scope.spawn(move || { + set_time_depth(depth); + + // Set up a destructor which will fire off a message that we're done as + // we exit. + struct Bomb { + tx: Sender, + success: bool, + } + impl Drop for Bomb { + fn drop(&mut self) { + drop(self.tx.send(Message::Done { success: self.success })); + } + } + let mut bomb = Bomb { + tx: tx.clone(), + success: false, + }; + + // Set up our non-`Send` `CodegenContext` now that we're in a helper + // thread and have all our info available to us. + let emitter = SharedEmitter { tx: tx.clone() }; + let diag_handler = Handler::with_emitter(true, false, Box::new(emitter)); + + let cgcx = CodegenContext { + crate_types: crate_types, + each_linked_rlib_for_lto: each_linked_rlib_for_lto, + lto: lto, + no_landing_pads: no_landing_pads, + opts: opts, + time_passes: time_passes, + exported_symbols: exported_symbols, + handler: &diag_handler, + plugin_passes: plugin_passes, + remark: remark, + worker: idx, + incr_comp_session_dir: incr_comp_session_dir, + tx: tx.clone(), + }; + + // Execute the work itself, and if it finishes successfully then flag + // ourselves as a success as well. + // + // Note that we ignore the result coming out of `execute_work_item` + // which will tell us if the worker failed with a `FatalError`. If that + // has happened, however, then a diagnostic was sent off to the main + // thread, along with an `AbortIfErrors` message. In that case the main + // thread is already exiting anyway most likely. + // + // In any case, there's no need for us to take further action here, so + // we just ignore the result and then send off our message saying that + // we're done, which if `execute_work_item` failed is unlikely to be + // seen by the main thread, but hey we might as well try anyway. + drop(execute_work_item(&cgcx, work).is_err()); + bomb.success = true; + }); +} + pub fn run_assembler(sess: &Session, outputs: &OutputFilenames) { let (pname, mut cmd, _) = get_linker(sess); diff --git a/src/librustc_trans/lib.rs b/src/librustc_trans/lib.rs index 41faf1fa768df..859c6574787ac 100644 --- a/src/librustc_trans/lib.rs +++ b/src/librustc_trans/lib.rs @@ -40,6 +40,7 @@ use rustc::dep_graph::WorkProduct; use syntax_pos::symbol::Symbol; extern crate flate2; +extern crate crossbeam; extern crate libc; extern crate owning_ref; #[macro_use] extern crate rustc; @@ -52,6 +53,7 @@ extern crate rustc_const_math; #[macro_use] #[no_link] extern crate rustc_bitflags; +extern crate jobserver; #[macro_use] extern crate log; #[macro_use] extern crate syntax; diff --git a/src/test/compile-fail-fulldeps/derive-no-std-not-supported.rs b/src/test/compile-fail-fulldeps/derive-no-std-not-supported.rs index 6ae5544d68699..1e97cb07f8960 100644 --- a/src/test/compile-fail-fulldeps/derive-no-std-not-supported.rs +++ b/src/test/compile-fail-fulldeps/derive-no-std-not-supported.rs @@ -10,7 +10,6 @@ #![no_std] -extern crate rand; extern crate serialize as rustc_serialize; #[derive(RustcEncodable)] //~ ERROR this trait cannot be derived diff --git a/src/test/compile-fail/asm-src-loc-codegen-units.rs b/src/test/compile-fail/asm-src-loc-codegen-units.rs index df1a6d52f5739..6c5c5b00776d8 100644 --- a/src/test/compile-fail/asm-src-loc-codegen-units.rs +++ b/src/test/compile-fail/asm-src-loc-codegen-units.rs @@ -7,17 +7,16 @@ // , at your // option. This file may not be copied, modified, or distributed // except according to those terms. -// + // WONTFIX(#20184) Needs landing pads (not present in stage1) or the compiler hangs. // ignore-stage1 // compile-flags: -C codegen-units=2 -// error-pattern: build without -C codegen-units for more exact errors // ignore-emscripten #![feature(asm)] fn main() { unsafe { - asm!("nowayisthisavalidinstruction"); + asm!("nowayisthisavalidinstruction"); //~ ERROR instruction } } diff --git a/src/test/compile-fail/auxiliary/issue-36881-aux.rs b/src/test/compile-fail/auxiliary/issue-36881-aux.rs new file mode 100644 index 0000000000000..33ac11feb2db6 --- /dev/null +++ b/src/test/compile-fail/auxiliary/issue-36881-aux.rs @@ -0,0 +1,11 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +pub trait Foo {} diff --git a/src/test/compile-fail/auxiliary/lint_unused_extern_crate2.rs b/src/test/compile-fail/auxiliary/lint_unused_extern_crate2.rs new file mode 100644 index 0000000000000..b61667cfd882c --- /dev/null +++ b/src/test/compile-fail/auxiliary/lint_unused_extern_crate2.rs @@ -0,0 +1,11 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +pub fn foo() {} diff --git a/src/test/compile-fail/auxiliary/lint_unused_extern_crate3.rs b/src/test/compile-fail/auxiliary/lint_unused_extern_crate3.rs new file mode 100644 index 0000000000000..b61667cfd882c --- /dev/null +++ b/src/test/compile-fail/auxiliary/lint_unused_extern_crate3.rs @@ -0,0 +1,11 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +pub fn foo() {} diff --git a/src/test/compile-fail/auxiliary/lint_unused_extern_crate4.rs b/src/test/compile-fail/auxiliary/lint_unused_extern_crate4.rs new file mode 100644 index 0000000000000..fc4bca865c932 --- /dev/null +++ b/src/test/compile-fail/auxiliary/lint_unused_extern_crate4.rs @@ -0,0 +1,9 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. diff --git a/src/test/compile-fail/issue-36881.rs b/src/test/compile-fail/issue-36881.rs index d75ac0c7f2ef3..e05dc06619969 100644 --- a/src/test/compile-fail/issue-36881.rs +++ b/src/test/compile-fail/issue-36881.rs @@ -8,9 +8,9 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -#![feature(rand)] +// aux-build:issue-36881-aux.rs fn main() { - extern crate rand; - use rand::Rng; //~ ERROR unresolved import + extern crate issue_36881_aux; + use issue_36881_aux::Foo; //~ ERROR unresolved import } diff --git a/src/test/compile-fail/lint-unused-extern-crate.rs b/src/test/compile-fail/lint-unused-extern-crate.rs index 010c55afb2b83..b12ef6277bb4b 100644 --- a/src/test/compile-fail/lint-unused-extern-crate.rs +++ b/src/test/compile-fail/lint-unused-extern-crate.rs @@ -9,34 +9,34 @@ // except according to those terms. // aux-build:lint_unused_extern_crate.rs +// aux-build:lint_unused_extern_crate2.rs +// aux-build:lint_unused_extern_crate3.rs +// aux-build:lint_unused_extern_crate4.rs #![deny(unused_extern_crates)] #![allow(unused_variables)] #![allow(deprecated)] -#![feature(alloc)] -#![feature(libc)] -#![feature(rand)] -extern crate libc; //~ ERROR: unused extern crate +extern crate lint_unused_extern_crate4; //~ ERROR: unused extern crate -extern crate alloc as collecs; // no error, it is used +extern crate lint_unused_extern_crate3; // no error, it is used -extern crate rand; // no error, the use marks it as used - // even if imported objects aren't used +extern crate lint_unused_extern_crate2; // no error, the use marks it as used + // even if imported objects aren't used extern crate lint_unused_extern_crate as other; // no error, the use * marks it as used #[allow(unused_imports)] -use rand::isaac::IsaacRng; +use lint_unused_extern_crate2::foo as bar; use other::*; mod foo { - // Test that this is unused even though an earler `extern crate rand` is used. - extern crate rand; //~ ERROR unused extern crate + // Test that this is unused even though an earler `extern crate` is used. + extern crate lint_unused_extern_crate2; //~ ERROR unused extern crate } fn main() { - let x: collecs::vec::Vec = Vec::new(); + lint_unused_extern_crate3::foo(); let y = foo(); } diff --git a/src/test/run-pass/auxiliary/allocator-dummy.rs b/src/test/run-pass/auxiliary/allocator-dummy.rs index a54233535a466..f4a32a93dfb71 100644 --- a/src/test/run-pass/auxiliary/allocator-dummy.rs +++ b/src/test/run-pass/auxiliary/allocator-dummy.rs @@ -10,11 +10,13 @@ // no-prefer-dynamic -#![feature(allocator, core_intrinsics)] +#![feature(allocator, core_intrinsics, panic_unwind)] #![allocator] #![crate_type = "rlib"] #![no_std] +extern crate unwind; + pub static mut HITS: usize = 0; type size_t = usize; From f441e07feb0d8421beaedc5223478bb6e08a5462 Mon Sep 17 00:00:00 2001 From: Stepan Koltsov Date: Wed, 21 Jun 2017 19:01:24 +0300 Subject: [PATCH 16/19] Pass path to python from bootstrap.py to bootstrap.rs When bootstrap is executed with python not in `$PATH`, (e. g. `c:\Python27\python.exe x.py test`) bootstrap cannot find python and crashes. This commit passes path to python in `BOOTSTRAP_PYTHON` env var. --- src/bootstrap/bootstrap.py | 1 + src/bootstrap/sanity.rs | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 1d3b77916d601..8dc2875ec4247 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -668,6 +668,7 @@ def bootstrap(): env["BUILD"] = rb.build env["SRC"] = rb.rust_root env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) + env["BOOTSTRAP_PYTHON"] = sys.executable run(args, env=env, verbose=rb.verbose) diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs index df6378a970bd4..5ccd131b77ae4 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -23,6 +23,7 @@ use std::env; use std::ffi::{OsStr, OsString}; use std::fs; use std::process::Command; +use std::path::PathBuf; use build_helper::output; @@ -86,6 +87,12 @@ pub fn check(build: &mut Build) { } } + if build.config.python.is_none() { + // set by bootstrap.py + if let Some(v) = env::var_os("BOOTSTRAP_PYTHON") { + build.config.python = Some(PathBuf::from(v)); + } + } if build.config.python.is_none() { build.config.python = have_cmd("python2.7".as_ref()); } From 305f5263f933d5d730f2bba5f0d43d9f9006361d Mon Sep 17 00:00:00 2001 From: Mark Simulacrum Date: Wed, 21 Jun 2017 10:04:21 -0600 Subject: [PATCH 17/19] Make rustc errors colorful. Rustbuild passes --message-format=json to Cargo to learn about the dependencies for a given build, which then makes Cargo steal the stderr/stdout for the compiler process, leading to non colorful output. To avoid this, detection of stderr being a tty is added to rustbuild, and an environment variable is used to communicate with the rustc shim. --- src/bootstrap/bin/rustc.rs | 9 +++++++++ src/bootstrap/compile.rs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs index 12152fc439901..8c6eaee24f294 100644 --- a/src/bootstrap/bin/rustc.rs +++ b/src/bootstrap/bin/rustc.rs @@ -236,6 +236,15 @@ fn main() { } } + let color = match env::var("RUSTC_COLOR") { + Ok(s) => usize::from_str(&s).expect("RUSTC_COLOR should be an integer"), + Err(_) => 0, + }; + + if color != 0 { + cmd.arg("--color=always"); + } + if verbose > 1 { writeln!(&mut io::stderr(), "rustc command: {:?}", cmd).unwrap(); } diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 9a07e8a8b1091..f92a199fa3fea 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -477,11 +477,43 @@ pub fn tool(build: &Build, stage: u32, target: &str, tool: &str) { build.run(&mut cargo); } + +// Avoiding a dependency on winapi to keep compile times down +#[cfg(unix)] +fn stderr_isatty() -> bool { + use libc; + unsafe { libc::isatty(libc::STDERR_FILENO) != 0 } +} +#[cfg(windows)] +fn stderr_isatty() -> bool { + type DWORD = u32; + type BOOL = i32; + type HANDLE = *mut u8; + const STD_ERROR_HANDLE: DWORD = -12i32 as DWORD; + extern "system" { + fn GetStdHandle(which: DWORD) -> HANDLE; + fn GetConsoleMode(hConsoleHandle: HANDLE, lpMode: *mut DWORD) -> BOOL; + } + unsafe { + let handle = GetStdHandle(STD_ERROR_HANDLE); + let mut out = 0; + GetConsoleMode(handle, &mut out) != 0 + } +} + fn run_cargo(build: &Build, cargo: &mut Command, stamp: &Path) { // Instruct Cargo to give us json messages on stdout, critically leaving // stderr as piped so we can get those pretty colors. cargo.arg("--message-format").arg("json") .stdout(Stdio::piped()); + + if stderr_isatty() { + // since we pass message-format=json to cargo, we need to tell the rustc + // wrapper to give us colored output if necessary. This is because we + // only want Cargo's JSON output, not rustcs. + cargo.env("RUSTC_COLOR", "1"); + } + build.verbose(&format!("running: {:?}", cargo)); let mut child = match cargo.spawn() { Ok(child) => child, From ae1dc2a6f902cea5b6e833497b11d8860acabfd9 Mon Sep 17 00:00:00 2001 From: Oliver Middleton Date: Wed, 21 Jun 2017 17:59:10 +0100 Subject: [PATCH 18/19] rustbuild: Fix compiler docs yet again Add support for `-Z force-unstable-if-unmarked` to rustdoc. --- src/bootstrap/bin/rustdoc.rs | 10 +++++----- src/librustdoc/core.rs | 7 ++++++- src/librustdoc/lib.rs | 6 +++++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/bin/rustdoc.rs index 3a1a9c3e40d66..d7d72d5dd56c9 100644 --- a/src/bootstrap/bin/rustdoc.rs +++ b/src/bootstrap/bin/rustdoc.rs @@ -41,11 +41,11 @@ fn main() { .env(bootstrap::util::dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - // Pass the `rustbuild` feature flag to crates which rustbuild is - // building. See the comment in bootstrap/lib.rs where this env var is - // set for more details. - if env::var_os("RUSTBUILD_UNSTABLE").is_some() { - cmd.arg("--cfg").arg("rustbuild"); + // Force all crates compiled by this compiler to (a) be unstable and (b) + // allow the `rustc_private` feature to link to other unstable crates + // also in the sysroot. + if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { + cmd.arg("-Z").arg("force-unstable-if-unmarked"); } std::process::exit(match cmd.status() { diff --git a/src/librustdoc/core.rs b/src/librustdoc/core.rs index 9a689ed079ee2..62b91feb09d0b 100644 --- a/src/librustdoc/core.rs +++ b/src/librustdoc/core.rs @@ -106,7 +106,8 @@ pub fn run_core(search_paths: SearchPaths, input: Input, triple: Option, maybe_sysroot: Option, - allow_warnings: bool) -> (clean::Crate, RenderInfo) + allow_warnings: bool, + force_unstable_if_unmarked: bool) -> (clean::Crate, RenderInfo) { // Parse, resolve, and typecheck the given crate. @@ -128,6 +129,10 @@ pub fn run_core(search_paths: SearchPaths, // Ensure that rustdoc works even if rustc is feature-staged unstable_features: UnstableFeatures::Allow, actually_rustdoc: true, + debugging_opts: config::DebuggingOptions { + force_unstable_if_unmarked: force_unstable_if_unmarked, + ..config::basic_debugging_options() + }, ..config::basic_options().clone() }; diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index f0b16ccf97531..a8bcf720e3961 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -410,13 +410,17 @@ where R: 'static + Send, F: 'static + Send + FnOnce(Output) -> R { info!("starting to run rustc"); let display_warnings = matches.opt_present("display-warnings"); + let force_unstable_if_unmarked = matches.opt_strs("Z").iter().any(|x| { + *x == "force-unstable-if-unmarked" + }); + let (tx, rx) = channel(); rustc_driver::monitor(move || { use rustc::session::config::Input; let (mut krate, renderinfo) = core::run_core(paths, cfgs, externs, Input::File(cr), triple, maybe_sysroot, - display_warnings); + display_warnings, force_unstable_if_unmarked); info!("finished with rustc"); From 826141359132123d7498a9fccf3b50742ce74ff7 Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Wed, 21 Jun 2017 17:38:22 +0300 Subject: [PATCH 19/19] debuginfo: Work around crash-bug in MSDIA library --- src/librustc_trans/debuginfo/mod.rs | 30 ++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/librustc_trans/debuginfo/mod.rs b/src/librustc_trans/debuginfo/mod.rs index d4d098ee5e748..68432c22f810c 100644 --- a/src/librustc_trans/debuginfo/mod.rs +++ b/src/librustc_trans/debuginfo/mod.rs @@ -43,7 +43,7 @@ use std::ptr; use syntax_pos::{self, Span, Pos}; use syntax::ast; use syntax::symbol::Symbol; -use rustc::ty::layout; +use rustc::ty::layout::{self, LayoutTyper}; pub mod gdb; mod utils; @@ -320,8 +320,32 @@ pub fn create_function_debug_context<'a, 'tcx>(cx: &CrateContext<'a, 'tcx>, }; // Arguments types - for &argument_type in inputs { - signature.push(type_metadata(cx, argument_type, syntax_pos::DUMMY_SP)); + if cx.sess().target.target.options.is_like_msvc { + // FIXME(#42800): + // There is a bug in MSDIA that leads to a crash when it encounters + // a fixed-size array of `u8` or something zero-sized in a + // function-type (see #40477). + // As a workaround, we replace those fixed-size arrays with a + // pointer-type. So a function `fn foo(a: u8, b: [u8; 4])` would + // appear as `fn foo(a: u8, b: *const u8)` in debuginfo, + // and a function `fn bar(x: [(); 7])` as `fn bar(x: *const ())`. + // This transformed type is wrong, but these function types are + // already inaccurate due to ABI adjustments (see #42800). + signature.extend(inputs.iter().map(|&t| { + let t = match t.sty { + ty::TyArray(ct, _) + if (ct == cx.tcx().types.u8) || + (cx.layout_of(ct).size(cx).bytes() == 0) => { + cx.tcx().mk_imm_ptr(ct) + } + _ => t + }; + type_metadata(cx, t, syntax_pos::DUMMY_SP) + })); + } else { + signature.extend(inputs.iter().map(|t| { + type_metadata(cx, t, syntax_pos::DUMMY_SP) + })); } if sig.abi == Abi::RustCall && !sig.inputs().is_empty() {