Skip to content

Add --depfile flag to inputs and multi-inputs tools. #2626

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add --depfile flag to inputs and multi-inputs tools. #2626

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion doc/manual.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,12 @@ executed in order, may be used to rebuild those targets, assuming that all
output files are out of date.

`inputs`:: given a list of targets, print a list of all inputs used to
rebuild those targets.
rebuild those targets. By default, this only includes static inputs declared
in the Ninja build plan itself.

Since Ninja 1.14, the `--depfile` flag can be used to include Ninja log / depfile
inputs in the result. Note that if the build plan was regenerated since the last
build, the log will contain stale entries, which may result in an incorrect output.
_Available since Ninja 1.11._

`multi-inputs`:: print one or more sets of inputs required to build targets.
Expand Down Expand Up @@ -341,6 +346,10 @@ target3 file3.c
+
_Note that a given input may appear for several targets if it is used by more
than one targets._

Like the `inputs` tool, this only reports static inputs from the build plan itself,
but since Ninja 1.14, the `--depfile` flag can be used to also includes entries
from the Ninja log or explicit depfiles.
_Available since Ninja 1.13._

`clean`:: remove built files. By default, it removes all built files
Expand Down
75 changes: 69 additions & 6 deletions misc/output_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def remove_non_visible_lines(raw_output: bytes) -> str:
class BuildDir:
def __init__(self, build_ninja: str):
self.build_ninja = dedent(build_ninja)
self.d = None
self.d : None | tempfile.TemporaryDirectory[str] = None

def __enter__(self):
self.d = tempfile.TemporaryDirectory()
Expand All @@ -72,6 +72,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):

@property
def path(self) -> str:
assert self.d
return os.path.realpath(self.d.name)


Expand Down Expand Up @@ -111,13 +112,13 @@ def run(
try:
if pipe:
output = subprocess.check_output(
[ninja_cmd], shell=True, cwd=self.d.name, env=env)
[ninja_cmd], shell=True, cwd=self.path, env=env)
elif platform.system() == 'Darwin':
output = subprocess.check_output(['script', '-q', '/dev/null', 'bash', '-c', ninja_cmd],
cwd=self.d.name, env=env)
cwd=self.path, env=env)
else:
output = subprocess.check_output(['script', '-qfec', ninja_cmd, '/dev/null'],
cwd=self.d.name, env=env)
cwd=self.path, env=env)
except subprocess.CalledProcessError as err:
if print_err_output:
sys.stdout.buffer.write(err.output)
Expand Down Expand Up @@ -366,8 +367,8 @@ def test_depfile_directory_creation(self) -> None:
self.assertEqual(b.run('', pipe=True), dedent('''\
[1/1] touch somewhere/out && echo "somewhere/out: extra" > somewhere_else/out.d
'''))
self.assertTrue(os.path.isfile(os.path.join(b.d.name, "somewhere", "out")))
self.assertTrue(os.path.isfile(os.path.join(b.d.name, "somewhere_else", "out.d")))
self.assertTrue(os.path.isfile(os.path.join(b.path, "somewhere", "out")))
self.assertTrue(os.path.isfile(os.path.join(b.path, "somewhere_else", "out.d")))

def test_status(self) -> None:
self.assertEqual(run(''), 'ninja: no work to do.\n')
Expand Down Expand Up @@ -468,6 +469,19 @@ def test_tool_inputs(self) -> None:
f'''in1\0out1\0out 2\0'''
)

def test_tool_inputs_with_depfile_entries(self) -> None:
build_ninja = '''
rule touch
command = touch $out && echo "$out: extra extra2" > $depfile
build out: touch
depfile = out.d
'''
with BuildDir(build_ninja) as b:
# Run the plan and create the log entry for 'out'
b.run('out')
self.assertEqual(b.run(flags='-t inputs out'), '')
self.assertEqual(b.run(flags='-t inputs --depfile out'), 'extra\nextra2\n')


def test_tool_compdb_targets(self) -> None:
plan = '''
Expand Down Expand Up @@ -556,6 +570,55 @@ def test_tool_multi_inputs(self) -> None:
)


def test_tool_multi_inputs_with_depfile_entries(self) -> None:
build_ninja = '''
rule touch1
command = touch $out && echo "$out: extra extra2" > $depfile
rule touch2
command = touch $out && echo "$out: extra extra3" > $depfile
build out1: touch1
depfile = out1.d
build out2: touch2
depfile = out2.d
'''
with BuildDir(build_ninja) as b:
# Run the plan and create the log entries for 'out1' and 'out2'
b.run('out1 out2')

self.assertEqual(b.run(flags='-t multi-inputs'), '')
self.assertEqual(b.run(flags='-t multi-inputs out1'), '')
self.assertEqual(b.run(flags='-t multi-inputs out2'), '')
self.assertEqual(b.run(flags='-t multi-inputs out1 out2'), '')
self.assertEqual(b.run(flags='-t multi-inputs out2 out1'), '')

self.assertEqual(b.run(flags='-t multi-inputs --depfile'), '''out1<TAB>extra
out1<TAB>extra2
out2<TAB>extra
out2<TAB>extra3
'''.replace('<TAB>', '\t'))

self.assertEqual(b.run(flags='-t multi-inputs --depfile out1'), '''out1<TAB>extra
out1<TAB>extra2
'''.replace('<TAB>', '\t'))

self.assertEqual(b.run(flags='-t multi-inputs --depfile out2'), '''out2<TAB>extra
out2<TAB>extra3
'''.replace('<TAB>', '\t'))

self.assertEqual(b.run(flags='-t multi-inputs --depfile out1 out2'), '''out1<TAB>extra
out1<TAB>extra2
out2<TAB>extra
out2<TAB>extra3
'''.replace('<TAB>', '\t'))

self.assertEqual(b.run(flags='-t multi-inputs --depfile out2 out1'), '''out2<TAB>extra
out2<TAB>extra3
out1<TAB>extra
out1<TAB>extra2
'''.replace('<TAB>', '\t'))



def test_explain_output(self):
b = BuildDir('''\
build .FORCE: phony
Expand Down
17 changes: 16 additions & 1 deletion src/graph.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -762,11 +762,26 @@ vector<Node*>::iterator ImplicitDepLoader::PreallocateSpace(Edge* edge,
}

void InputsCollector::VisitNode(const Node* node) {
const Edge* edge = node->in_edge();
Edge* edge = node->in_edge();

if (!edge) // A source file.
return;

if (implicit_dep_loader_ && !edge->deps_loaded_) {
// Record that the deps were loaded in |deps_loaded_| as
// multiple visits to the same edge can be performed by
// repeated InputsCollector uses, as for the multi-inputs tool.
edge->deps_loaded_ = true;

// Ignore errors when loading depfile entries.
std::string err;
if (!implicit_dep_loader_->LoadDeps(edge, &err)) {
// Print the error as a warning on stderr when an error occurred during
// the load.
Warning("%s", err.c_str());
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI I tried out this branch just now and got the following output on ninja itself:

(-zsh@hephaestus.local) ~/src/ninja (detached HEAD)
; ./ninja -t inputs --depfile
ninja: warning:
...
ninja: warning:
ninja: warning:
ninja: warning:
ninja: warning:
build/browse.o

Probably there is a bug here somewhere? I would at least expect err to be non-empty.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I also don't see it including any of the depfile inputs, which may or may not be related.

}
}

// Add inputs of the producing edge to the result,
// except if they are themselves produced by a phony
// edge.
Expand Down
7 changes: 7 additions & 0 deletions src/graph.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,12 @@ class EdgePriorityQueue:
/// vector.
///
struct InputsCollector {
/// Constructor allows passing a pointer to an optional ImplicitDepLoader
/// which will be used to load implicit dependencies from the Ninja log
/// or existing depfiles. Note that this will mutate the graph.
explicit InputsCollector(ImplicitDepLoader* implicit_dep_loader = nullptr)
: implicit_dep_loader_(implicit_dep_loader) {}

/// Visit a single @arg node during this collection.
void VisitNode(const Node* node);

Expand All @@ -459,6 +465,7 @@ struct InputsCollector {
}

private:
ImplicitDepLoader* implicit_dep_loader_ = nullptr;
std::vector<const Node*> inputs_;
std::set<const Node*> visited_nodes_;
};
Expand Down
33 changes: 29 additions & 4 deletions src/ninja.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -784,14 +784,20 @@ int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) {

optind = 1;
int opt;
bool include_depfile_inputs = false;
enum { OPT_DEPFILE = 1 };
const option kLongOptions[] = { { "help", no_argument, NULL, 'h' },
{ "no-shell-escape", no_argument, NULL, 'E' },
{ "print0", no_argument, NULL, '0' },
{ "depfile", no_argument, NULL, OPT_DEPFILE },
{ "dependency-order", no_argument, NULL,
'd' },
{ NULL, 0, NULL, 0 } };
while ((opt = getopt_long(argc, argv, "h0Ed", kLongOptions, NULL)) != -1) {
switch (opt) {
case OPT_DEPFILE:
include_depfile_inputs = true;
break;
case 'd':
dependency_order = true;
break;
Expand All @@ -815,6 +821,7 @@ int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) {
" -0, --print0 Use \\0, instead of \\n as a line terminator.\n"
" -E, --no-shell-escape Do not shell escape the result.\n"
" -d, --dependency-order Sort results by dependency order.\n"
" --depfile Include depfile inputs in the result.\n"
);
// clang-format on
return 1;
Expand All @@ -830,7 +837,13 @@ int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) {
return 1;
}

InputsCollector collector;
std::unique_ptr<ImplicitDepLoader> implicit_dep_loader;
if (include_depfile_inputs) {
implicit_dep_loader = std::make_unique<ImplicitDepLoader>(
&state_, &deps_log_, &disk_interface_, nullptr, nullptr);
}

InputsCollector collector(implicit_dep_loader.get());
for (const Node* node : nodes)
collector.VisitNode(node);

Expand Down Expand Up @@ -861,13 +874,18 @@ int NinjaMain::ToolMultiInputs(const Options* options, int argc, char* argv[]) {
int opt;
char terminator = '\n';
const char* delimiter = "\t";
bool include_depfile_inputs = false;
enum { OPT_DEPFILE = 1 };
const option kLongOptions[] = { { "help", no_argument, NULL, 'h' },
{ "delimiter", required_argument, NULL,
'd' },
{ "delimiter", required_argument, NULL, 'd' },
{ "depfile", no_argument, NULL, OPT_DEPFILE },
{ "print0", no_argument, NULL, '0' },
{ NULL, 0, NULL, 0 } };
while ((opt = getopt_long(argc, argv, "d:h0", kLongOptions, NULL)) != -1) {
switch (opt) {
case OPT_DEPFILE:
include_depfile_inputs = true;
break;
case 'd':
delimiter = optarg;
break;
Expand All @@ -888,6 +906,7 @@ int NinjaMain::ToolMultiInputs(const Options* options, int argc, char* argv[]) {
"Options:\n"
" -h, --help Print this message.\n"
" -d --delimiter=DELIM Use DELIM instead of TAB for field delimiter.\n"
" --depfile Include depfile inputs in the result.\n"
" -0, --print0 Use \\0, instead of \\n as a line terminator.\n"
);
// clang-format on
Expand All @@ -904,8 +923,14 @@ int NinjaMain::ToolMultiInputs(const Options* options, int argc, char* argv[]) {
return 1;
}

std::unique_ptr<ImplicitDepLoader> implicit_dep_loader;
if (include_depfile_inputs) {
implicit_dep_loader = std::make_unique<ImplicitDepLoader>(
&state_, &deps_log_, &disk_interface_, nullptr, nullptr);
}

for (const Node* node : nodes) {
InputsCollector collector;
InputsCollector collector(implicit_dep_loader.get());

collector.VisitNode(node);
std::vector<std::string> inputs = collector.GetInputsAsStrings();
Expand Down
Loading