xref: /third_party/ninja/src/manifest_parser.cc (revision 695b41ee) 
1// Copyright 2011 Google Inc. All Rights Reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#include "manifest_parser.h"
16
17#include <assert.h>
18#include <stdio.h>
19#include <stdlib.h>
20
21#include <vector>
22
23#include "graph.h"
24#include "state.h"
25#include "util.h"
26#include "version.h"
27
28using namespace std;
29
30ManifestParser::ManifestParser(State* state, FileReader* file_reader,
31                               ManifestParserOptions options)
32    : Parser(state, file_reader),
33      options_(options), quiet_(false) {
34  env_ = &state->bindings_;
35}
36
37bool ManifestParser::Parse(const string& filename, const string& input,
38                           string* err) {
39  lexer_.Start(filename, input);
40
41  for (;;) {
42    Lexer::Token token = lexer_.ReadToken();
43    switch (token) {
44    case Lexer::POOL:
45      if (!ParsePool(err))
46        return false;
47      break;
48    case Lexer::BUILD:
49      if (!ParseEdge(err))
50        return false;
51      break;
52    case Lexer::RULE:
53      if (!ParseRule(err))
54        return false;
55      break;
56    case Lexer::DEFAULT:
57      if (!ParseDefault(err))
58        return false;
59      break;
60    case Lexer::IDENT: {
61      lexer_.UnreadToken();
62      string name;
63      EvalString let_value;
64      if (!ParseLet(&name, &let_value, err))
65        return false;
66      string value = let_value.Evaluate(env_);
67      // Check ninja_required_version immediately so we can exit
68      // before encountering any syntactic surprises.
69      if (name == "ninja_required_version")
70        CheckNinjaVersion(value);
71      env_->AddBinding(name, value);
72      break;
73    }
74    case Lexer::INCLUDE:
75      if (!ParseFileInclude(false, err))
76        return false;
77      break;
78    case Lexer::SUBNINJA:
79      if (!ParseFileInclude(true, err))
80        return false;
81      break;
82    case Lexer::ERROR: {
83      return lexer_.Error(lexer_.DescribeLastError(), err);
84    }
85    case Lexer::TEOF:
86      return true;
87    case Lexer::NEWLINE:
88      break;
89    default:
90      return lexer_.Error(string("unexpected ") + Lexer::TokenName(token),
91                          err);
92    }
93  }
94  return false;  // not reached
95}
96
97
98bool ManifestParser::ParsePool(string* err) {
99  string name;
100  if (!lexer_.ReadIdent(&name))
101    return lexer_.Error("expected pool name", err);
102
103  if (!ExpectToken(Lexer::NEWLINE, err))
104    return false;
105
106  if (state_->LookupPool(name) != NULL)
107    return lexer_.Error("duplicate pool '" + name + "'", err);
108
109  int depth = -1;
110
111  while (lexer_.PeekToken(Lexer::INDENT)) {
112    string key;
113    EvalString value;
114    if (!ParseLet(&key, &value, err))
115      return false;
116
117    if (key == "depth") {
118      string depth_string = value.Evaluate(env_);
119      depth = atol(depth_string.c_str());
120      if (depth < 0)
121        return lexer_.Error("invalid pool depth", err);
122    } else {
123      return lexer_.Error("unexpected variable '" + key + "'", err);
124    }
125  }
126
127  if (depth < 0)
128    return lexer_.Error("expected 'depth =' line", err);
129
130  state_->AddPool(new Pool(name, depth));
131  return true;
132}
133
134
135bool ManifestParser::ParseRule(string* err) {
136  string name;
137  if (!lexer_.ReadIdent(&name))
138    return lexer_.Error("expected rule name", err);
139
140  if (!ExpectToken(Lexer::NEWLINE, err))
141    return false;
142
143  if (env_->LookupRuleCurrentScope(name) != NULL)
144    return lexer_.Error("duplicate rule '" + name + "'", err);
145
146  Rule* rule = new Rule(name);  // XXX scoped_ptr
147
148  while (lexer_.PeekToken(Lexer::INDENT)) {
149    string key;
150    EvalString value;
151    if (!ParseLet(&key, &value, err))
152      return false;
153
154    if (Rule::IsReservedBinding(key)) {
155      rule->AddBinding(key, value);
156    } else {
157      // Die on other keyvals for now; revisit if we want to add a
158      // scope here.
159      return lexer_.Error("unexpected variable '" + key + "'", err);
160    }
161  }
162
163  if (rule->bindings_["rspfile"].empty() !=
164      rule->bindings_["rspfile_content"].empty()) {
165    return lexer_.Error("rspfile and rspfile_content need to be "
166                        "both specified", err);
167  }
168
169  if (rule->bindings_["command"].empty())
170    return lexer_.Error("expected 'command =' line", err);
171
172  env_->AddRule(rule);
173  return true;
174}
175
176bool ManifestParser::ParseLet(string* key, EvalString* value, string* err) {
177  if (!lexer_.ReadIdent(key))
178    return lexer_.Error("expected variable name", err);
179  if (!ExpectToken(Lexer::EQUALS, err))
180    return false;
181  if (!lexer_.ReadVarValue(value, err))
182    return false;
183  return true;
184}
185
186bool ManifestParser::ParseDefault(string* err) {
187  EvalString eval;
188  if (!lexer_.ReadPath(&eval, err))
189    return false;
190  if (eval.empty())
191    return lexer_.Error("expected target name", err);
192
193  do {
194    string path = eval.Evaluate(env_);
195    if (path.empty())
196      return lexer_.Error("empty path", err);
197    uint64_t slash_bits;  // Unused because this only does lookup.
198    CanonicalizePath(&path, &slash_bits);
199    std::string default_err;
200    if (!state_->AddDefault(path, &default_err))
201      return lexer_.Error(default_err, err);
202
203    eval.Clear();
204    if (!lexer_.ReadPath(&eval, err))
205      return false;
206  } while (!eval.empty());
207
208  return ExpectToken(Lexer::NEWLINE, err);
209}
210
211bool ManifestParser::ParseEdge(string* err) {
212  vector<EvalString> ins, outs, validations;
213
214  {
215    EvalString out;
216    if (!lexer_.ReadPath(&out, err))
217      return false;
218    while (!out.empty()) {
219      outs.push_back(out);
220
221      out.Clear();
222      if (!lexer_.ReadPath(&out, err))
223        return false;
224    }
225  }
226
227  // Add all implicit outs, counting how many as we go.
228  int implicit_outs = 0;
229  if (lexer_.PeekToken(Lexer::PIPE)) {
230    for (;;) {
231      EvalString out;
232      if (!lexer_.ReadPath(&out, err))
233        return false;
234      if (out.empty())
235        break;
236      outs.push_back(out);
237      ++implicit_outs;
238    }
239  }
240
241  if (outs.empty())
242    return lexer_.Error("expected path", err);
243
244  if (!ExpectToken(Lexer::COLON, err))
245    return false;
246
247  string rule_name;
248  if (!lexer_.ReadIdent(&rule_name))
249    return lexer_.Error("expected build command name", err);
250
251  const Rule* rule = env_->LookupRule(rule_name);
252  if (!rule)
253    return lexer_.Error("unknown build rule '" + rule_name + "'", err);
254
255  for (;;) {
256    // XXX should we require one path here?
257    EvalString in;
258    if (!lexer_.ReadPath(&in, err))
259      return false;
260    if (in.empty())
261      break;
262    ins.push_back(in);
263  }
264
265  // Add all implicit deps, counting how many as we go.
266  int implicit = 0;
267  if (lexer_.PeekToken(Lexer::PIPE)) {
268    for (;;) {
269      EvalString in;
270      if (!lexer_.ReadPath(&in, err))
271        return false;
272      if (in.empty())
273        break;
274      ins.push_back(in);
275      ++implicit;
276    }
277  }
278
279  // Add all order-only deps, counting how many as we go.
280  int order_only = 0;
281  if (lexer_.PeekToken(Lexer::PIPE2)) {
282    for (;;) {
283      EvalString in;
284      if (!lexer_.ReadPath(&in, err))
285        return false;
286      if (in.empty())
287        break;
288      ins.push_back(in);
289      ++order_only;
290    }
291  }
292
293  // Add all validations, counting how many as we go.
294  if (lexer_.PeekToken(Lexer::PIPEAT)) {
295    for (;;) {
296      EvalString validation;
297      if (!lexer_.ReadPath(&validation, err))
298        return false;
299      if (validation.empty())
300        break;
301      validations.push_back(validation);
302    }
303  }
304
305  if (!ExpectToken(Lexer::NEWLINE, err))
306    return false;
307
308  // Bindings on edges are rare, so allocate per-edge envs only when needed.
309  bool has_indent_token = lexer_.PeekToken(Lexer::INDENT);
310  BindingEnv* env = has_indent_token ? new BindingEnv(env_) : env_;
311  while (has_indent_token) {
312    string key;
313    EvalString val;
314    if (!ParseLet(&key, &val, err))
315      return false;
316
317    env->AddBinding(key, val.Evaluate(env_));
318    has_indent_token = lexer_.PeekToken(Lexer::INDENT);
319  }
320
321  Edge* edge = state_->AddEdge(rule);
322  edge->env_ = env;
323
324  string pool_name = edge->GetBinding("pool");
325  if (!pool_name.empty()) {
326    Pool* pool = state_->LookupPool(pool_name);
327    if (pool == NULL)
328      return lexer_.Error("unknown pool name '" + pool_name + "'", err);
329    edge->pool_ = pool;
330  }
331
332  edge->outputs_.reserve(outs.size());
333  for (size_t i = 0, e = outs.size(); i != e; ++i) {
334    string path = outs[i].Evaluate(env);
335    if (path.empty())
336      return lexer_.Error("empty path", err);
337    uint64_t slash_bits;
338    CanonicalizePath(&path, &slash_bits);
339    if (!state_->AddOut(edge, path, slash_bits)) {
340      if (options_.dupe_edge_action_ == kDupeEdgeActionError) {
341        lexer_.Error("multiple rules generate " + path, err);
342        return false;
343      } else {
344        if (!quiet_) {
345          Warning(
346              "multiple rules generate %s. builds involving this target will "
347              "not be correct; continuing anyway",
348              path.c_str());
349        }
350        if (e - i <= static_cast<size_t>(implicit_outs))
351          --implicit_outs;
352      }
353    }
354  }
355
356  if (edge->outputs_.empty()) {
357    // All outputs of the edge are already created by other edges. Don't add
358    // this edge.  Do this check before input nodes are connected to the edge.
359    state_->edges_.pop_back();
360    delete edge;
361    return true;
362  }
363  edge->implicit_outs_ = implicit_outs;
364
365  edge->inputs_.reserve(ins.size());
366  for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) {
367    string path = i->Evaluate(env);
368    if (path.empty())
369      return lexer_.Error("empty path", err);
370    uint64_t slash_bits;
371    CanonicalizePath(&path, &slash_bits);
372    state_->AddIn(edge, path, slash_bits);
373  }
374  edge->implicit_deps_ = implicit;
375  edge->order_only_deps_ = order_only;
376
377  edge->validations_.reserve(validations.size());
378  for (std::vector<EvalString>::iterator v = validations.begin();
379      v != validations.end(); ++v) {
380    string path = v->Evaluate(env);
381    if (path.empty())
382      return lexer_.Error("empty path", err);
383    uint64_t slash_bits;
384    CanonicalizePath(&path, &slash_bits);
385    state_->AddValidation(edge, path, slash_bits);
386  }
387
388  if (options_.phony_cycle_action_ == kPhonyCycleActionWarn &&
389      edge->maybe_phonycycle_diagnostic()) {
390    // CMake 2.8.12.x and 3.0.x incorrectly write phony build statements
391    // that reference themselves.  Ninja used to tolerate these in the
392    // build graph but that has since been fixed.  Filter them out to
393    // support users of those old CMake versions.
394    Node* out = edge->outputs_[0];
395    vector<Node*>::iterator new_end =
396        remove(edge->inputs_.begin(), edge->inputs_.end(), out);
397    if (new_end != edge->inputs_.end()) {
398      edge->inputs_.erase(new_end, edge->inputs_.end());
399      if (!quiet_) {
400        Warning("phony target '%s' names itself as an input; "
401                "ignoring [-w phonycycle=warn]",
402                out->path().c_str());
403      }
404    }
405  }
406
407  // Lookup, validate, and save any dyndep binding.  It will be used later
408  // to load generated dependency information dynamically, but it must
409  // be one of our manifest-specified inputs.
410  string dyndep = edge->GetUnescapedDyndep();
411  if (!dyndep.empty()) {
412    uint64_t slash_bits;
413    CanonicalizePath(&dyndep, &slash_bits);
414    edge->dyndep_ = state_->GetNode(dyndep, slash_bits);
415    edge->dyndep_->set_dyndep_pending(true);
416    vector<Node*>::iterator dgi =
417      std::find(edge->inputs_.begin(), edge->inputs_.end(), edge->dyndep_);
418    if (dgi == edge->inputs_.end()) {
419      return lexer_.Error("dyndep '" + dyndep + "' is not an input", err);
420    }
421    assert(!edge->dyndep_->generated_by_dep_loader());
422  }
423
424  return true;
425}
426
427bool ManifestParser::ParseFileInclude(bool new_scope, string* err) {
428  EvalString eval;
429  if (!lexer_.ReadPath(&eval, err))
430    return false;
431  string path = eval.Evaluate(env_);
432
433  ManifestParser subparser(state_, file_reader_, options_);
434  if (new_scope) {
435    subparser.env_ = new BindingEnv(env_);
436  } else {
437    subparser.env_ = env_;
438  }
439
440  if (!subparser.Load(path, err, &lexer_))
441    return false;
442
443  if (!ExpectToken(Lexer::NEWLINE, err))
444    return false;
445
446  return true;
447}
448