1#![allow(clippy::manual_assert)] 2 3mod progress; 4 5use self::progress::Progress; 6use anyhow::Result; 7use flate2::read::GzDecoder; 8use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; 9use std::collections::BTreeSet; 10use std::ffi::OsStr; 11use std::fs; 12use std::path::{Path, PathBuf}; 13use tar::Archive; 14use walkdir::{DirEntry, WalkDir}; 15 16const REVISION: &str = "b10cfcd65fd7f7b1ab9beb34798b2108de003452"; 17 18#[rustfmt::skip] 19static EXCLUDE_FILES: &[&str] = &[ 20 // TODO: CStr literals: c"…", cr"…" 21 // https://github.com/dtolnay/syn/issues/1502 22 "src/tools/clippy/tests/ui/needless_raw_string.rs", 23 "src/tools/clippy/tests/ui/needless_raw_string_hashes.rs", 24 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rs", 25 26 // TODO: explicit tail calls: `become _g()` 27 // https://github.com/dtolnay/syn/issues/1501 28 "tests/ui/explicit-tail-calls/return-lifetime-sub.rs", 29 30 // TODO: non-lifetime binders: `where for<'a, T> &'a Struct<T>: Trait` 31 // https://github.com/dtolnay/syn/issues/1435 32 "src/tools/rustfmt/tests/source/issue_5721.rs", 33 "src/tools/rustfmt/tests/source/non-lifetime-binders.rs", 34 "src/tools/rustfmt/tests/target/issue_5721.rs", 35 "src/tools/rustfmt/tests/target/non-lifetime-binders.rs", 36 "tests/rustdoc-json/non_lifetime_binders.rs", 37 "tests/rustdoc/inline_cross/auxiliary/non_lifetime_binders.rs", 38 "tests/rustdoc/non_lifetime_binders.rs", 39 40 // TODO: return type notation: `where T: Trait<method(): Send>` 41 // https://github.com/dtolnay/syn/issues/1434 42 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0208_associated_return_type_bounds.rs", 43 "tests/ui/associated-type-bounds/return-type-notation/basic.rs", 44 "tests/ui/feature-gates/feature-gate-return_type_notation.rs", 45 46 // TODO: lazy type alias syntax with where-clause in trailing position 47 // https://github.com/dtolnay/syn/issues/1525 48 "tests/rustdoc/typedef-inner-variants-lazy_type_alias.rs", 49 50 // TODO: gen blocks and functions 51 // https://github.com/dtolnay/syn/issues/1526 52 "compiler/rustc_codegen_cranelift/example/gen_block_iterate.rs", 53 "tests/ui/coroutine/gen_block_is_iter.rs", 54 "tests/ui/coroutine/gen_block_iterate.rs", 55 56 // TODO: struct literal in match guard 57 // https://github.com/dtolnay/syn/issues/1527 58 "tests/ui/parser/struct-literal-in-match-guard.rs", 59 60 // Compile-fail expr parameter in const generic position: f::<1 + 2>() 61 "tests/ui/const-generics/early/closing-args-token.rs", 62 "tests/ui/const-generics/early/const-expression-parameter.rs", 63 64 // Compile-fail variadics in not the last position of a function parameter list 65 "tests/ui/parser/variadic-ffi-syntactic-pass.rs", 66 67 // Need at least one trait in impl Trait, no such type as impl 'static 68 "tests/ui/type-alias-impl-trait/generic_type_does_not_live_long_enough.rs", 69 70 // Negative polarity trait bound: `where T: !Copy` 71 "src/tools/rustfmt/tests/target/negative-bounds.rs", 72 73 // Lifetime bound inside for<>: `T: ~const ?for<'a: 'b> Trait<'a>` 74 "tests/ui/rfcs/rfc-2632-const-trait-impl/tilde-const-syntax.rs", 75 76 // Const impl that is not a trait impl: `impl ~const T {}` 77 "tests/ui/rfcs/rfc-2632-const-trait-impl/syntax.rs", 78 79 // Deprecated anonymous parameter syntax in traits 80 "src/tools/rustfmt/tests/source/trait.rs", 81 "src/tools/rustfmt/tests/target/trait.rs", 82 "tests/ui/issues/issue-13105.rs", 83 "tests/ui/issues/issue-13775.rs", 84 "tests/ui/issues/issue-34074.rs", 85 "tests/ui/proc-macro/trait-fn-args-2015.rs", 86 87 // Deprecated where-clause location 88 "src/tools/rustfmt/tests/source/issue_4257.rs", 89 "src/tools/rustfmt/tests/source/issue_4911.rs", 90 "src/tools/rustfmt/tests/target/issue_4257.rs", 91 "src/tools/rustfmt/tests/target/issue_4911.rs", 92 "tests/pretty/gat-bounds.rs", 93 "tests/rustdoc/generic-associated-types/gats.rs", 94 95 // Deprecated trait object syntax with parenthesized generic arguments and no dyn keyword 96 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0004_value_parameters_no_patterns.rs", 97 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0104_path_fn_trait_args.rs", 98 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0202_typepathfn_with_coloncolon.rs", 99 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0209_bare_dyn_types_with_paren_as_generic_args.rs", 100 "src/tools/rustfmt/tests/source/attrib.rs", 101 "src/tools/rustfmt/tests/source/closure.rs", 102 "src/tools/rustfmt/tests/source/existential_type.rs", 103 "src/tools/rustfmt/tests/source/fn-simple.rs", 104 "src/tools/rustfmt/tests/source/fn_args_layout-vertical.rs", 105 "src/tools/rustfmt/tests/source/issue-4689/one.rs", 106 "src/tools/rustfmt/tests/source/issue-4689/two.rs", 107 "src/tools/rustfmt/tests/source/paths.rs", 108 "src/tools/rustfmt/tests/source/structs.rs", 109 "src/tools/rustfmt/tests/target/attrib.rs", 110 "src/tools/rustfmt/tests/target/closure.rs", 111 "src/tools/rustfmt/tests/target/existential_type.rs", 112 "src/tools/rustfmt/tests/target/fn-simple.rs", 113 "src/tools/rustfmt/tests/target/fn.rs", 114 "src/tools/rustfmt/tests/target/fn_args_layout-vertical.rs", 115 "src/tools/rustfmt/tests/target/issue-4689/one.rs", 116 "src/tools/rustfmt/tests/target/issue-4689/two.rs", 117 "src/tools/rustfmt/tests/target/paths.rs", 118 "src/tools/rustfmt/tests/target/structs.rs", 119 "tests/codegen-units/item-collection/non-generic-closures.rs", 120 "tests/debuginfo/recursive-enum.rs", 121 "tests/pretty/closure-reform-pretty.rs", 122 "tests/run-make/reproducible-build-2/reproducible-build.rs", 123 "tests/run-make/reproducible-build/reproducible-build.rs", 124 "tests/ui/auxiliary/typeid-intrinsic-aux1.rs", 125 "tests/ui/auxiliary/typeid-intrinsic-aux2.rs", 126 "tests/ui/impl-trait/generic-with-implicit-hrtb-without-dyn.rs", 127 "tests/ui/lifetimes/auxiliary/lifetime_bound_will_change_warning_lib.rs", 128 "tests/ui/lifetimes/bare-trait-object-borrowck.rs", 129 "tests/ui/lifetimes/bare-trait-object.rs", 130 "tests/ui/parser/bounds-obj-parens.rs", 131 132 // Invalid unparenthesized range pattern inside slice pattern: `[1..]` 133 "tests/ui/consts/miri_unleashed/const_refers_to_static_cross_crate.rs", 134 135 // Various extensions to Rust syntax made up by rust-analyzer 136 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0012_type_item_where_clause.rs", 137 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0058_range_pat.rs", 138 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0123_param_list_vararg.rs", 139 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0131_existential_type.rs", 140 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0156_fn_def_param.rs", 141 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0179_use_tree_abs_star.rs", 142 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0188_const_param_default_path.rs", 143 "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0015_use_tree.rs", 144 "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0029_range_forms.rs", 145 "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0051_parameter_attrs.rs", 146 "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0055_dot_dot_dot.rs", 147 "src/tools/rust-analyzer/crates/parser/test_data/parser/ok/0068_item_modifiers.rs", 148 "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0031_block_inner_attrs.rs", 149 "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0038_endless_inclusive_range.rs", 150 "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0045_ambiguous_trait_object.rs", 151 "src/tools/rust-analyzer/crates/syntax/test_data/parser/validation/0046_mutable_const_item.rs", 152 153 // Placeholder syntax for "throw expressions" 154 "compiler/rustc_errors/src/translation.rs", 155 "src/tools/clippy/tests/ui/needless_return.rs", 156 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0204_yeet_expr.rs", 157 "tests/pretty/yeet-expr.rs", 158 "tests/ui/try-trait/yeet-for-option.rs", 159 "tests/ui/try-trait/yeet-for-result.rs", 160 161 // Edition 2015 code using identifiers that are now keywords 162 // TODO: some of these we should probably parse 163 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0159_try_macro_fallback.rs", 164 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/0160_try_macro_rules.rs", 165 "src/tools/rustfmt/tests/source/configs/indent_style/block_call.rs", 166 "src/tools/rustfmt/tests/source/configs/use_try_shorthand/false.rs", 167 "src/tools/rustfmt/tests/source/configs/use_try_shorthand/true.rs", 168 "src/tools/rustfmt/tests/source/issue_1306.rs", 169 "src/tools/rustfmt/tests/source/try-conversion.rs", 170 "src/tools/rustfmt/tests/target/configs/indent_style/block_call.rs", 171 "src/tools/rustfmt/tests/target/configs/use_try_shorthand/false.rs", 172 "src/tools/rustfmt/tests/target/issue-1681.rs", 173 "src/tools/rustfmt/tests/target/issue_1306.rs", 174 "tests/ui/dyn-keyword/dyn-2015-no-warnings-without-lints.rs", 175 "tests/ui/editions/edition-keywords-2015-2015.rs", 176 "tests/ui/editions/edition-keywords-2015-2018.rs", 177 "tests/ui/lint/lint_pre_expansion_extern_module_aux.rs", 178 "tests/ui/macros/macro-comma-support-rpass.rs", 179 "tests/ui/macros/try-macro.rs", 180 "tests/ui/parser/extern-crate-async.rs", 181 "tests/ui/try-block/try-is-identifier-edition2015.rs", 182 183 // Excessive nesting 184 "tests/ui/issues/issue-74564-if-expr-stack-overflow.rs", 185 186 // Testing tools on invalid syntax 187 "src/tools/rustfmt/tests/coverage/target/comments.rs", 188 "src/tools/rustfmt/tests/parser/issue-4126/invalid.rs", 189 "src/tools/rustfmt/tests/parser/issue_4418.rs", 190 "src/tools/rustfmt/tests/parser/unclosed-delims/issue_4466.rs", 191 "src/tools/rustfmt/tests/source/configs/disable_all_formatting/true.rs", 192 "src/tools/rustfmt/tests/source/configs/spaces_around_ranges/false.rs", 193 "src/tools/rustfmt/tests/source/configs/spaces_around_ranges/true.rs", 194 "src/tools/rustfmt/tests/source/type.rs", 195 "src/tools/rustfmt/tests/target/configs/spaces_around_ranges/false.rs", 196 "src/tools/rustfmt/tests/target/configs/spaces_around_ranges/true.rs", 197 "src/tools/rustfmt/tests/target/type.rs", 198 "tests/run-make/translation/test.rs", 199 "tests/ui/generics/issue-94432-garbage-ice.rs", 200 201 // Generated file containing a top-level expression, used with `include!` 202 "compiler/rustc_codegen_gcc/src/intrinsic/archs.rs", 203 204 // Clippy lint lists represented as expressions 205 "src/tools/clippy/clippy_lints/src/lib.deprecated.rs", 206 207 // Not actually test cases 208 "tests/ui/lint/expansion-time-include.rs", 209 "tests/ui/macros/auxiliary/macro-comma-support.rs", 210 "tests/ui/macros/auxiliary/macro-include-items-expr.rs", 211 "tests/ui/macros/include-single-expr-helper.rs", 212 "tests/ui/macros/include-single-expr-helper-1.rs", 213 "tests/ui/parser/issues/auxiliary/issue-21146-inc.rs", 214]; 215 216#[rustfmt::skip] 217static EXCLUDE_DIRS: &[&str] = &[ 218 // Inputs that intentionally do not parse 219 "src/tools/rust-analyzer/crates/parser/test_data/parser/err", 220 "src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err", 221 222 // Inputs that lex but do not necessarily parse 223 "src/tools/rust-analyzer/crates/parser/test_data/lexer", 224 225 // Inputs that used to crash rust-analyzer, but aren't necessarily supposed to parse 226 "src/tools/rust-analyzer/crates/syntax/test_data/parser/fuzz-failures", 227 "src/tools/rust-analyzer/crates/syntax/test_data/reparse/fuzz-failures", 228]; 229 230// Directories in which a .stderr implies the corresponding .rs is not expected 231// to work. 232static UI_TEST_DIRS: &[&str] = &["tests/ui", "tests/rustdoc-ui"]; 233 234pub fn for_each_rust_file(for_each: impl Fn(&Path) + Sync + Send) { 235 let mut rs_files = BTreeSet::new(); 236 237 let repo_dir = Path::new("tests/rust"); 238 for entry in WalkDir::new(repo_dir) 239 .into_iter() 240 .filter_entry(base_dir_filter) 241 { 242 let entry = entry.unwrap(); 243 if !entry.file_type().is_dir() { 244 rs_files.insert(entry.into_path()); 245 } 246 } 247 248 for ui_test_dir in UI_TEST_DIRS { 249 for entry in WalkDir::new(repo_dir.join(ui_test_dir)) { 250 let mut path = entry.unwrap().into_path(); 251 if path.extension() == Some(OsStr::new("stderr")) { 252 loop { 253 rs_files.remove(&path.with_extension("rs")); 254 path = path.with_extension(""); 255 if path.extension().is_none() { 256 break; 257 } 258 } 259 } 260 } 261 } 262 263 rs_files.par_iter().map(PathBuf::as_path).for_each(for_each); 264} 265 266pub fn base_dir_filter(entry: &DirEntry) -> bool { 267 let path = entry.path(); 268 269 let mut path_string = path.to_string_lossy(); 270 if cfg!(windows) { 271 path_string = path_string.replace('\\', "/").into(); 272 } 273 let path_string = if path_string == "tests/rust" { 274 return true; 275 } else if let Some(path) = path_string.strip_prefix("tests/rust/") { 276 path 277 } else { 278 panic!("unexpected path in Rust dist: {}", path_string); 279 }; 280 281 if path.is_dir() { 282 return !EXCLUDE_DIRS.contains(&path_string); 283 } 284 285 if path.extension() != Some(OsStr::new("rs")) { 286 return false; 287 } 288 289 !EXCLUDE_FILES.contains(&path_string) 290} 291 292#[allow(dead_code)] 293pub fn edition(path: &Path) -> &'static str { 294 if path.ends_with("dyn-2015-no-warnings-without-lints.rs") { 295 "2015" 296 } else { 297 "2018" 298 } 299} 300 301pub fn clone_rust() { 302 let needs_clone = match fs::read_to_string("tests/rust/COMMIT") { 303 Err(_) => true, 304 Ok(contents) => contents.trim() != REVISION, 305 }; 306 if needs_clone { 307 download_and_unpack().unwrap(); 308 } 309 310 let mut missing = String::new(); 311 let test_src = Path::new("tests/rust"); 312 313 let mut exclude_files_set = BTreeSet::new(); 314 for exclude in EXCLUDE_FILES { 315 if !exclude_files_set.insert(exclude) { 316 panic!("duplicate path in EXCLUDE_FILES: {}", exclude); 317 } 318 for dir in EXCLUDE_DIRS { 319 if Path::new(exclude).starts_with(dir) { 320 panic!("excluded file {} is inside an excluded dir", exclude); 321 } 322 } 323 if !test_src.join(exclude).is_file() { 324 missing += "\ntests/rust/"; 325 missing += exclude; 326 } 327 } 328 329 let mut exclude_dirs_set = BTreeSet::new(); 330 for exclude in EXCLUDE_DIRS { 331 if !exclude_dirs_set.insert(exclude) { 332 panic!("duplicate path in EXCLUDE_DIRS: {}", exclude); 333 } 334 if !test_src.join(exclude).is_dir() { 335 missing += "\ntests/rust/"; 336 missing += exclude; 337 missing += "/"; 338 } 339 } 340 341 if !missing.is_empty() { 342 panic!("excluded test file does not exist:{}\n", missing); 343 } 344} 345 346fn download_and_unpack() -> Result<()> { 347 let url = format!( 348 "https://github.com/rust-lang/rust/archive/{}.tar.gz", 349 REVISION 350 ); 351 let response = reqwest::blocking::get(url)?.error_for_status()?; 352 let progress = Progress::new(response); 353 let decoder = GzDecoder::new(progress); 354 let mut archive = Archive::new(decoder); 355 let prefix = format!("rust-{}", REVISION); 356 357 let tests_rust = Path::new("tests/rust"); 358 if tests_rust.exists() { 359 fs::remove_dir_all(tests_rust)?; 360 } 361 362 for entry in archive.entries()? { 363 let mut entry = entry?; 364 let path = entry.path()?; 365 if path == Path::new("pax_global_header") { 366 continue; 367 } 368 let relative = path.strip_prefix(&prefix)?; 369 let out = tests_rust.join(relative); 370 entry.unpack(&out)?; 371 } 372 373 fs::write("tests/rust/COMMIT", REVISION)?; 374 Ok(()) 375} 376