diff --git a/Makefile b/Makefile index d1c2b1f7..011a3ead 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ LUA ?= lua5.1 LUA_VERSION = $(shell $(LUA) -e 'print(_VERSION:match("%d%.%d"))') LUAROCKS = luarocks-$(LUA_VERSION) -LUA_PATH_MAKE = $(shell $(LUAROCKS) path --lr-path);./?.lua;./?/init.lua -LUA_CPATH_MAKE = $(shell $(LUAROCKS) path --lr-cpath);./?.so +LUA_PATH_MAKE = $(shell $(LUAROCKS) path --lr-path);./?.lua;./?/init.lua;$(LUA_PATH) +LUA_CPATH_MAKE = $(shell $(LUAROCKS) path --lr-cpath);./?.so;$(LUA_CPATH) .PHONY: test local compile compile_system watch lint count show test: - busted + LUA_PATH='$(LUA_PATH_MAKE)' LUA_CPATH='$(LUA_CPATH_MAKE)' busted show: # LUA $(LUA) @@ -22,7 +22,7 @@ local: compile compile: LUA_PATH='$(LUA_PATH_MAKE)' LUA_CPATH='$(LUA_CPATH_MAKE)' $(LUA) bin/moonc moon/ moonscript/ echo "#!/usr/bin/env lua" > bin/moon - $(LUA) bin/moonc -p bin/moon.moon >> bin/moon + LUA_PATH='$(LUA_PATH_MAKE)' LUA_CPATH='$(LUA_CPATH_MAKE)' $(LUA) bin/moonc -p bin/moon.moon >> bin/moon echo "-- vim: set filetype=lua:" >> bin/moon compile_system: diff --git a/bin/moonc b/bin/moonc index 015979ad..82f376a3 100755 --- a/bin/moonc +++ b/bin/moonc @@ -1,232 +1,4 @@ #!/usr/bin/env lua -local argparse = require "argparse" -local lfs = require "lfs" - -local parser = argparse() - -parser:flag("-l --lint", "Perform a lint on the file instead of compiling") -parser:flag("-v --version", "Print version") -parser:flag("-w --watch", "Watch file/directory for updates") -parser:option("--transform", "Transform syntax tree with module") -parser:mutex( - parser:option("-t --output-to", "Specify where to place compiled files"), - parser:option("-o", "Write output to file"), - parser:flag("-p", "Write output to standard output"), - parser:flag("-T", "Write parse tree instead of code (to stdout)"), - parser:flag("-b", "Write parse and compile time instead of code(to stdout)"), - parser:flag("-X", "Write line rewrite map instead of code (to stdout)") -) -parser:flag("-", - "Read from standard in, print to standard out (Must be only argument)") - -local read_stdin = arg[1] == "--" -- luacheck: ignore 113 - -if not read_stdin then - parser:argument("file/directory"):args("+") -end - -local opts = parser:parse() - -if opts.version then - local v = require "moonscript.version" - v.print_version() - os.exit() -end - -function log_msg(...) - if not opts.p then - io.stderr:write(table.concat({...}, " ") .. "\n") - end -end - -local moonc = require("moonscript.cmd.moonc") -local util = require "moonscript.util" -local normalize_dir = moonc.normalize_dir -local compile_and_write = moonc.compile_and_write -local path_to_target = moonc.path_to_target - -local function scan_directory(root, collected) - root = normalize_dir(root) - collected = collected or {} - - for fname in lfs.dir(root) do - if not fname:match("^%.") then - local full_path = root..fname - - if lfs.attributes(full_path, "mode") == "directory" then - scan_directory(full_path, collected) - elseif fname:match("%.moon$") then - table.insert(collected, full_path) - end - end - end - - return collected -end - -local function remove_dups(tbl, key_fn) - local hash = {} - local final = {} - - for _, v in ipairs(tbl) do - local dup_key = key_fn and key_fn(v) or v - if not hash[dup_key] then - table.insert(final, v) - hash[dup_key] = true - end - end - - return final -end - --- creates tuples of input and target -local function get_files(fname, files) - files = files or {} - - if lfs.attributes(fname, "mode") == "directory" then - for _, sub_fname in ipairs(scan_directory(fname)) do - table.insert(files, { - sub_fname, - path_to_target(sub_fname, opts.output_to, fname) - }) - end - else - table.insert(files, { - fname, - path_to_target(fname, opts.output_to) - }) - end - - return files -end - -if read_stdin then - local parse = require "moonscript.parse" - local compile = require "moonscript.compile" - - local text = io.stdin:read("*a") - local tree, err = parse.string(text) - - if not tree then error(err) end - local code, err, pos = compile.tree(tree) - - if not code then - error(compile.format_error(err, pos, text)) - end - - print(code) - os.exit() -end - -local inputs = opts["file/directory"] - -local files = {} -for _, input in ipairs(inputs) do - get_files(input, files) -end - -files = remove_dups(files, function(f) - return f[2] -end) - --- returns an iterator that returns files that have been updated -local function create_watcher(files) - local watchers = require("moonscript.cmd.watchers") - - if watchers.InotifyWacher:available() then - return watchers.InotifyWacher(files):each_update() - end - - return watchers.SleepWatcher(files):each_update() -end - -if opts.watch then - -- build function to check for lint or compile in watch - local handle_file - if opts.lint then - local lint = require "moonscript.cmd.lint" - handle_file = lint.lint_file - else - handle_file = compile_and_write - end - - local watcher = create_watcher(files) - -- catches interrupt error for ctl-c - local protected = function() - local status, file = true, watcher() - if status then - return file - elseif file ~= "interrupted!" then - error(file) - end - end - - for fname in protected do - local target = path_to_target(fname, opts.t) - - if opts.o then - target = opts.o - end - - local success, err = handle_file(fname, target) - if opts.lint then - if success then - io.stderr:write(success .. "\n\n") - elseif err then - io.stderr:write(fname .. "\n" .. err .. "\n\n") - end - elseif not success then - io.stderr:write(table.concat({ - "", - "Error: " .. fname, - err, - "\n", - }, "\n")) - elseif success == "build" then - log_msg("Built", fname, "->", target) - end - end - - io.stderr:write("\nQuitting...\n") -elseif opts.lint then - local has_linted_with_error; - local lint = require "moonscript.cmd.lint" - for _, tuple in pairs(files) do - local fname = tuple[1] - local res, err = lint.lint_file(fname) - if res then - has_linted_with_error = true - io.stderr:write(res .. "\n\n") - elseif err then - has_linted_with_error = true - io.stderr:write(fname .. "\n" .. err.. "\n\n") - end - end - if has_linted_with_error then - os.exit(1) - end -else - for _, tuple in ipairs(files) do - local fname, target = util.unpack(tuple) - if opts.o then - target = opts.o - end - - local success, err = compile_and_write(fname, target, { - print = opts.p, - fname = fname, - benchmark = opts.b, - show_posmap = opts.X, - show_parse_tree = opts.T, - transform_module = opts.transform - }) - - if not success then - io.stderr:write(fname .. "\t" .. err .. "\n") - os.exit(1) - end - end -end - - +local moonc = require "moonscript.cmd.moonc" +moonc.main(arg) diff --git a/docs/command_line.md b/docs/command_line.md index f0e5d4b2..5cebd0c9 100644 --- a/docs/command_line.md +++ b/docs/command_line.md @@ -116,17 +116,38 @@ files in the same directories. $ moonc my_script1.moon my_script2.moon ... ``` -You can control where the compiled files are put using the `-t` flag, followed -by a directory. +You can control where the compiled files are put using the `-t`/`--output-to` +flag, followed by a directory. `moonc` can also take a directory as an argument, and it will recursively scan for all MoonScript files and compile them. +If you use `-t` and also specify a directory as argument, then similar to +`rsync`, handling of whether or not directory names are included in the output +file paths differs depending on whether or not you specify a trailing `/`. + +For example, if you have a folder called `src` containing a file `foo.moon`: + +```bash +$ moonc -t out src +``` + +Produces the file `out/src/foo.lua`. Alternatively: + +```bash +$ moonc -t out src/ +``` + +Produces the file `out/foo.lua`. It does not matter whether or not there is a +trailing `/` on the `--output-to` directory, only on each of the input +directories specified. + `moonc` can write to standard out by passing the `-p` flag. The `-w` flag can be used to enable watch mode. `moonc` will stay running, and watch for changes to the input files. If any of them change then they will be -compiled automatically. +compiled automatically. Watch mode also works in combination with the `-t` +flag. A full list of flags can be seen by passing the `-h` or `--help` flag. diff --git a/moonscript-dev-1.rockspec b/moonscript-dev-1.rockspec index 8970e306..bb8656f2 100644 --- a/moonscript-dev-1.rockspec +++ b/moonscript-dev-1.rockspec @@ -30,6 +30,7 @@ build = { ["moonscript.cmd.coverage"] = "moonscript/cmd/coverage.lua", ["moonscript.cmd.lint"] = "moonscript/cmd/lint.lua", ["moonscript.cmd.moonc"] = "moonscript/cmd/moonc.lua", + ["moonscript.cmd.path_handling"] = "moonscript/cmd/path_handling.lua", ["moonscript.cmd.watchers"] = "moonscript/cmd/watchers.lua", ["moonscript.compile"] = "moonscript/compile.lua", ["moonscript.compile.statement"] = "moonscript/compile/statement.lua", diff --git a/moonscript/cmd/args.lua b/moonscript/cmd/args.lua deleted file mode 100644 index 68b06fdc..00000000 --- a/moonscript/cmd/args.lua +++ /dev/null @@ -1,69 +0,0 @@ -local unpack -unpack = require("moonscript.util").unpack -local parse_spec -parse_spec = function(spec) - local flags, words - if type(spec) == "table" then - flags, words = unpack(spec), spec - else - flags, words = spec, { } - end - assert("no flags for arguments") - local out = { } - for part in flags:gmatch("%w:?") do - if part:match(":$") then - out[part:sub(1, 1)] = { - value = true - } - else - out[part] = { } - end - end - return out -end -local parse_arguments -parse_arguments = function(spec, args) - spec = parse_spec(spec) - local out = { } - local remaining = { } - local last_flag = nil - for _index_0 = 1, #args do - local _continue_0 = false - repeat - local arg = args[_index_0] - local group = { } - if last_flag then - out[last_flag] = arg - _continue_0 = true - break - end - do - local flag = arg:match("-(%w+)") - if flag then - do - local short_name = spec[flag] - if short_name then - out[short_name] = true - else - for char in flag:gmatch(".") do - out[char] = true - end - end - end - _continue_0 = true - break - end - end - table.insert(remaining, arg) - _continue_0 = true - until true - if not _continue_0 then - break - end - end - return out, remaining -end -return { - parse_arguments = parse_arguments, - parse_spec = parse_spec -} diff --git a/moonscript/cmd/args.moon b/moonscript/cmd/args.moon deleted file mode 100644 index ac65c0e3..00000000 --- a/moonscript/cmd/args.moon +++ /dev/null @@ -1,47 +0,0 @@ -import unpack from require "moonscript.util" -parse_spec = (spec) -> - flags, words = if type(spec) == "table" - unpack(spec), spec - else - spec, {} - - assert "no flags for arguments" - - out = {} - for part in flags\gmatch "%w:?" - if part\match ":$" - out[part\sub 1,1] = { value: true } - else - out[part] = {} - - out - -parse_arguments = (spec, args) -> - spec = parse_spec spec - - out = {} - - remaining = {} - last_flag = nil - - for arg in *args - group = {} - if last_flag - out[last_flag] = arg - continue - - if flag = arg\match "-(%w+)" - if short_name = spec[flag] - out[short_name] = true - else - for char in flag\gmatch "." - out[char] = true - continue - - table.insert remaining, arg - - out, remaining - - - -{ :parse_arguments, :parse_spec } diff --git a/moonscript/cmd/moonc.lua b/moonscript/cmd/moonc.lua index e7618ef8..de93a479 100644 --- a/moonscript/cmd/moonc.lua +++ b/moonscript/cmd/moonc.lua @@ -1,12 +1,253 @@ local lfs = require("lfs") local split split = require("moonscript.util").split -local dirsep, dirsep_chars, mkdir, normalize_dir, parse_dir, parse_file, convert_path, format_time, gettime, compile_file_text, write_file, compile_and_write, is_abs_path, path_to_target -dirsep = package.config:sub(1, 1) -if dirsep == "\\" then - dirsep_chars = "\\/" -else - dirsep_chars = dirsep +local dirsep, normalize_dir, normalize_path, parse_dir, parse_file, parse_subtree, convert_path +do + local _obj_0 = require("moonscript.cmd.path_handling") + dirsep, normalize_dir, normalize_path, parse_dir, parse_file, parse_subtree, convert_path = _obj_0.dirsep, _obj_0.normalize_dir, _obj_0.normalize_path, _obj_0.parse_dir, _obj_0.parse_file, _obj_0.parse_subtree, _obj_0.convert_path +end +local main, process_stdin_and_exit, parse_cli_paths, handle_watch_loop, create_watcher, scan_initial_files, lint_for, compile_for, mkdir, format_time, gettime, compile_file_text, write_file, compile_and_write, add_prefix_for_dir, output_for, process_filesystem_tree +main = function(cli_args) + local argparse = require("argparse") + local parser + do + local _with_0 = argparse("moonc", "The Moonscript compiler.") + _with_0:flag("-l --lint", "Perform a lint on the file instead of compiling.") + _with_0:flag("-v --version", "Print version.") + _with_0:flag("-w --watch", "Watch file/directory for updates.") + _with_0:option("--transform", "Transform syntax tree with module.") + _with_0:mutex(_with_0:option("-t --output-to", "Specify where to place compiled files."), _with_0:option("-o", "Write output to file."), _with_0:flag("-p", "Write output to standard output."), _with_0:flag("-T", "Write parse tree instead of code (to stdout)."), _with_0:flag("-b", "Write parse and compile time instead of code(to stdout)."), _with_0:flag("-X", "Write line rewrite map instead of code (to stdout).")) + _with_0:flag("-", "Read from standard in, print to standard out (Must be only argument).") + parser = _with_0 + end + local read_stdin = cli_args[1] == "--" + if not (read_stdin) then + parser:argument("file/directory"):args("+") + end + local opts = parser:parse(cli_args) + if opts.version then + local v = require("moonscript.version") + v.print_version() + os.exit() + end + if read_stdin then + process_stdin_and_exit() + end + local inputs = opts["file/directory"] + if inputs == nil then + error("No paths specified") + end + if opts.o then + if #inputs > 1 then + error("-o can only be used with a single input") + elseif (lfs.attributes(inputs[1], "mode")) == "directory" then + error("-o can only be used with a file input, not a directory") + end + end + local output_to, cli_paths, prefix_map = parse_cli_paths(inputs, opts.output_to) + if opts.watch then + return handle_watch_loop(opts, output_to, cli_paths, prefix_map) + else + local files = scan_initial_files(output_to, cli_paths, prefix_map) + if opts.lint then + return lint_for(files) + else + return compile_for(opts, files) + end + end +end +process_stdin_and_exit = function() + local parse = require("moonscript.parse") + local compile = require("moonscript.compile") + local text = io.stdin:read("*a") + local tree, err = parse.string(text) + if not (tree) then + error(err) + end + local code, pos + code, err, pos = compile.tree(tree) + if not (code) then + error(compile.format_error(err, pos, text)) + end + print(code) + return os.exit() +end +parse_cli_paths = function(input_paths, output_to) + if output_to == nil then + output_to = false + end + if not (input_paths) then + error("No paths specified") + end + if output_to then + output_to = normalize_dir(output_to) + end + local cli_paths = { } + local prefix_map = { } + for _index_0 = 1, #input_paths do + local path = input_paths[_index_0] + local mode, err_msg, err_code = lfs.attributes(path, 'mode') + local _exp_0 = mode + if 'file' == _exp_0 then + table.insert(cli_paths, { + (normalize_path(path)), + 'file' + }) + elseif 'directory' == _exp_0 then + add_prefix_for_dir(output_to, prefix_map, path) + table.insert(cli_paths, { + (normalize_dir(path)), + 'directory' + }) + elseif nil == _exp_0 then + error("Error code " .. tostring(err_code) .. " accessing given path `" .. tostring(path) .. "`, error message: " .. tostring(err_msg)) + else + error("Given path `" .. tostring(path) .. "` has unexpected filesystem mode `" .. tostring(mode) .. "`") + end + end + return output_to, cli_paths, prefix_map +end +handle_watch_loop = function(opts, output_to, input_paths, prefix_map) + local log_msg + log_msg = function(...) + if not (opts.p) then + return io.stderr:write(table.concat({ + ... + }, " ") .. "\n") + end + end + local remove_orphaned_output + remove_orphaned_output = function(target, path_type) + if path_type == "directory" then + local is_ok, err_string, err_no = lfs.rmdir(target) + if is_ok then + return log_msg("Removed output directory", target) + else + return log_msg("Error removing directory", target, "err_no", err_no, "msg", err_string) + end + elseif path_type == "file" then + local is_ok, err_string = os.remove(target) + if is_ok then + return log_msg("Removed output file", target) + else + return log_msg("Error removing file", target, err_string) + end + end + end + local watcher = create_watcher(output_to, input_paths, prefix_map) + for file_tuple in watcher do + local event_type, filename, target, path_type + event_type, filename, target, path_type = file_tuple[1], file_tuple[2], file_tuple[3], file_tuple[4] + if opts.o then + target = opts.o + end + if opts.lint then + if event_type == "changedfile" then + local lint = require("moonscript.cmd.lint") + local success, err = lint.lint_file(filename) + if success then + io.stderr:write(success .. "\n\n") + elseif err then + io.stderr:write(filename .. "\n" .. err .. "\n\n") + end + elseif event_type == "removed" then + remove_orphaned_output(target, path_type) + end + else + if event_type == "changedfile" then + local success, err = compile_and_write(filename, target) + if not success then + io.stderr:write(table.concat({ + "", + "Error: " .. filename, + err, + "\n" + }, "\n")) + elseif success == "build" then + log_msg("Built", filename, "->", target) + end + elseif event_type == "removed" then + remove_orphaned_output(target, path_type) + end + end + end + return io.stderr:write("\nQuitting...\n") +end +create_watcher = function(output_to, input_paths, prefix_map) + local watchers = require("moonscript.cmd.watchers") + local watcher + if watchers.InotifyWatcher:available() then + watcher = watchers.InotifyWatcher(output_to, input_paths, prefix_map) + else + watcher = watchers.SleepWatcher(output_to, input_paths, prefix_map) + end + return watcher:each_update() +end +scan_initial_files = function(output_to, input_paths, prefix_map) + local files = { } + for _index_0 = 1, #input_paths do + local path_tuple = input_paths[_index_0] + local path, path_type + path, path_type = path_tuple[1], path_tuple[2] + if path_type == "directory" then + process_filesystem_tree(path, nil, function(file_path) + if file_path:match("%.moon$") then + return table.insert(files, { + file_path, + output_for(output_to, prefix_map, file_path, 'file') + }) + end + end) + else + table.insert(files, { + path, + output_for(output_to, prefix_map, path, 'file') + }) + end + end + return files +end +lint_for = function(files) + local has_linted_with_error + local lint = require("moonscript.cmd.lint") + for _index_0 = 1, #files do + local tuple = files[_index_0] + local filename, _target + filename, _target = tuple[1], tuple[2] + local res, err = lint.lint_file(filename) + if res then + has_linted_with_error = true + io.stderr:write(res .. "\n\n") + elseif err then + has_linted_with_error = true + io.stderr:write(filename .. "\n" .. err .. "\n\n") + end + end + if has_linted_with_error then + return os.exit(1) + end +end +compile_for = function(opts, files) + for _index_0 = 1, #files do + local tuple = files[_index_0] + local filename, target + filename, target = tuple[1], tuple[2] + if opts.o then + target = opts.o + end + local success, err = compile_and_write(filename, target, { + print = opts.p, + filename = filename, + benchmark = opts.b, + show_posmap = opts.X, + show_parse_tree = opts.T, + transform_module = opts.transform + }) + if not (success) then + io.stderr:write(filename .. "\t" .. err .. "\n") + os.exit(1) + end + end end mkdir = function(path) local chunks = split(path, dirsep) @@ -18,22 +259,6 @@ mkdir = function(path) end return lfs.attributes(path, "mode") end -normalize_dir = function(path) - return path:match("^(.-)[" .. tostring(dirsep_chars) .. "]*$") .. dirsep -end -parse_dir = function(path) - return (path:match("^(.-)[^" .. tostring(dirsep_chars) .. "]*$")) -end -parse_file = function(path) - return (path:match("^.-([^" .. tostring(dirsep_chars) .. "]*)$")) -end -convert_path = function(path) - local new_path = path:gsub("%.moon$", ".lua") - if new_path == path then - new_path = path .. ".lua" - end - return new_path -end format_time = function(time) return ("%.3fms"):format(time * 1000) end @@ -105,7 +330,7 @@ compile_file_text = function(text, opts) end if opts.benchmark then print(table.concat({ - opts.fname or "stdin", + opts.filename or "stdin", "Parse time \t" .. format_time(parse_time), "Compile time\t" .. format_time(compile_time), "" @@ -114,9 +339,9 @@ compile_file_text = function(text, opts) end return code end -write_file = function(fname, code) - mkdir(parse_dir(fname)) - local f, err = io.open(fname, "w") +write_file = function(filename, code) + mkdir(parse_dir(filename)) + local f, err = io.open(filename, "w") if not (f) then return nil, err end @@ -148,52 +373,95 @@ compile_and_write = function(src, dest, opts) end return write_file(dest, code) end -is_abs_path = function(path) - local first = path:sub(1, 1) +add_prefix_for_dir = function(output_to, prefix_map, directory) + local last = directory:sub(#directory, #directory) + local is_exclusive if dirsep == "\\" then - return first == "/" or first == "\\" or path:sub(2, 1) == ":" + is_exclusive = last == "/" or last == "\\" else - return first == dirsep + is_exclusive = last == dirsep end -end -path_to_target = function(path, target_dir, base_dir) - if target_dir == nil then - target_dir = nil - end - if base_dir == nil then - base_dir = nil + directory = normalize_dir(directory) + do + local prefix_start = output_to + if prefix_start then + if is_exclusive then + prefix_map[directory] = prefix_start .. parse_subtree(directory) + else + prefix_map[directory] = prefix_start .. directory + end + else + prefix_map[directory] = directory + end end - local target = convert_path(path) - if target_dir then - target_dir = normalize_dir(target_dir) +end +output_for = function(output_to, prefix_map, path, path_type) + if not (path_type == "directory") then + path = convert_path(path) end - if base_dir and target_dir then - local head = base_dir:match("^(.-)[^" .. tostring(dirsep_chars) .. "]*[" .. tostring(dirsep_chars) .. "]?$") - if head then - local start, stop = target:find(head, 1, true) - if start == 1 then - target = target:sub(stop + 1) + for prefix_path, prefix_output in pairs(prefix_map) do + local _continue_0 = false + repeat + if #path < #prefix_path then + _continue_0 = true + break + end + if path:sub(1, #prefix_path) == prefix_path then + local output_path = prefix_output .. path:sub(#prefix_path + 1, #path) + return output_path end + _continue_0 = true + until true + if not _continue_0 then + break end end - if target_dir then - if is_abs_path(target) then - target = parse_file(target) + if output_to then + path = output_to .. path + end + return path +end +process_filesystem_tree = function(root, directory_callback, file_callback) + root = normalize_dir(root) + if directory_callback then + directory_callback(root) + end + local ok, _iter, dirobj = pcall(lfs.dir, root) + if not (ok) then + return + end + while true do + local filename = dirobj:next() + if not (filename) then + break + end + if not (filename:match("^%.")) then + local fpath = root .. filename + local mode = lfs.attributes(fpath, "mode") + local _exp_0 = mode + if "directory" == _exp_0 then + fpath = fpath .. '/' + process_filesystem_tree(fpath, directory_callback, file_callback) + elseif "file" == _exp_0 then + if file_callback then + file_callback(fpath) + end + elseif nil == _exp_0 then + local _ = nil + else + error("Unexpected filetype " .. tostring(mode)) + end end - target = target_dir .. target end - return target end return { - dirsep = dirsep, + main = main, mkdir = mkdir, - normalize_dir = normalize_dir, - parse_dir = parse_dir, - parse_file = parse_file, - convert_path = convert_path, gettime = gettime, format_time = format_time, - path_to_target = path_to_target, compile_file_text = compile_file_text, - compile_and_write = compile_and_write + compile_and_write = compile_and_write, + process_filesystem_tree = process_filesystem_tree, + parse_cli_paths = parse_cli_paths, + output_for = output_for } diff --git a/moonscript/cmd/moonc.moon b/moonscript/cmd/moonc.moon index 3e05bb07..2bcc0780 100644 --- a/moonscript/cmd/moonc.moon +++ b/moonscript/cmd/moonc.moon @@ -3,14 +3,232 @@ lfs = require "lfs" import split from require "moonscript.util" +import dirsep, normalize_dir, normalize_path, parse_dir, parse_file, parse_subtree, convert_path from require "moonscript.cmd.path_handling" local * -dirsep = package.config\sub 1,1 -dirsep_chars = if dirsep == "\\" - "\\/" -- windows -else - dirsep +main = (cli_args) -> + argparse = require "argparse" + parser = with argparse("moonc", "The Moonscript compiler.") + \flag("-l --lint", "Perform a lint on the file instead of compiling.") + \flag("-v --version", "Print version.") + \flag("-w --watch", "Watch file/directory for updates.") + \option("--transform", "Transform syntax tree with module.") + \mutex( + \option("-t --output-to", "Specify where to place compiled files."), + \option("-o", "Write output to file."), + \flag("-p", "Write output to standard output."), + \flag("-T", "Write parse tree instead of code (to stdout)."), + \flag("-b", "Write parse and compile time instead of code(to stdout)."), + \flag("-X", "Write line rewrite map instead of code (to stdout).") + ) + \flag("-", + "Read from standard in, print to standard out (Must be only argument).") + read_stdin = cli_args[1] == "--" -- luacheck: ignore 113 + unless read_stdin + parser\argument("file/directory")\args("+") + + opts = parser\parse cli_args + + if opts.version + v = require "moonscript.version" + v.print_version! + os.exit! + + if read_stdin + process_stdin_and_exit! + + inputs = opts["file/directory"] + if inputs == nil + error "No paths specified" + + if opts.o + if #inputs > 1 + error "-o can only be used with a single input" + elseif (lfs.attributes inputs[1], "mode") == "directory" + error "-o can only be used with a file input, not a directory" + + -- Determine if the CLI paths are valid, and handle --output-to and exclusive + -- vs. inclusive directories + output_to, cli_paths, prefix_map = parse_cli_paths inputs, opts.output_to + + -- Handle file-watching, linting, and compilation + if opts.watch + handle_watch_loop opts, output_to, cli_paths, prefix_map + else + -- Scan CLI argument files/directories to get full set of files to build from + files = scan_initial_files output_to, cli_paths, prefix_map + if opts.lint + lint_for files + else + compile_for opts, files + +process_stdin_and_exit = () -> + parse = require "moonscript.parse" + compile = require "moonscript.compile" + + text = io.stdin\read "*a" + tree, err = parse.string text + + unless tree + error err + code, err, pos = compile.tree tree + + unless code + error(compile.format_error err, pos, text) + + print code + os.exit! + +parse_cli_paths = (input_paths, output_to=false) -> + unless input_paths + error "No paths specified" + if output_to + output_to = normalize_dir output_to + cli_paths = {} + -- Contains a map of given directories to their corresponding output + -- directory. The two will be equal if @output_to is not set. + prefix_map = {} + for path in *input_paths + mode, err_msg, err_code = lfs.attributes(path, 'mode') + switch mode + when 'file' + table.insert cli_paths, {(normalize_path path), 'file'} + when 'directory' + add_prefix_for_dir output_to, prefix_map, path + table.insert cli_paths, {(normalize_dir path), 'directory'} + when nil + error "Error code #{err_code} accessing given path `#{path}`, error message: #{err_msg}" + else + error "Given path `#{path}` has unexpected filesystem mode `#{mode}`" + return output_to, cli_paths, prefix_map + +handle_watch_loop = (opts, output_to, input_paths, prefix_map) -> + log_msg = (...) -> + unless opts.p + io.stderr\write(table.concat({...}, " ") .. "\n") + + remove_orphaned_output = (target, path_type) -> + if path_type == "directory" + is_ok, err_string, err_no = lfs.rmdir target + -- rmdir() can legitimately fail if the dir being removed is non-empty + -- (e.g. if there is a vendored Lua file in here), TODO look up the Linux + -- and Windows error codes for 'directory not empty' and silently fail on + -- those, loudly fail on other error types? + if is_ok + log_msg "Removed output directory", target + else + log_msg "Error removing directory", target, "err_no", err_no, "msg", err_string + elseif path_type == "file" + is_ok, err_string = os.remove target + if is_ok + log_msg "Removed output file", target + else + log_msg "Error removing file", target, err_string + + watcher = create_watcher output_to, input_paths, prefix_map + + for file_tuple in watcher + {event_type, filename, target, path_type} = file_tuple + + if opts.o + target = opts.o + + if opts.lint + if event_type == "changedfile" + lint = require "moonscript.cmd.lint" + success, err = lint.lint_file filename + if success + io.stderr\write success .. "\n\n" + elseif err + io.stderr\write filename .. "\n" .. err .. "\n\n" + elseif event_type == "removed" + remove_orphaned_output target, path_type + else + if event_type == "changedfile" + success, err = compile_and_write filename, target + if not success + io.stderr\write table.concat({ + "", + "Error: " .. filename, + err, + "\n", + }, "\n") + elseif success == "build" + log_msg "Built", filename, "->", target + elseif event_type == "removed" + remove_orphaned_output target, path_type + + io.stderr\write "\nQuitting...\n" + +create_watcher = (output_to, input_paths, prefix_map) -> + watchers = require "moonscript.cmd.watchers" + + -- TODO cli argument to force sleep watcher (as it is potentially a little + -- more reliable) + watcher = if watchers.InotifyWatcher\available! + watchers.InotifyWatcher output_to, input_paths, prefix_map + else + watchers.SleepWatcher output_to, input_paths, prefix_map + + return watcher\each_update! + + +scan_initial_files = (output_to, input_paths, prefix_map) -> + files = {} + + for path_tuple in *input_paths + {path, path_type} = path_tuple + if path_type == "directory" + -- Recursively scan directories and add .moon files in them + process_filesystem_tree(path, nil, (file_path) -> + if file_path\match("%.moon$") + table.insert(files, { + file_path, output_for(output_to, prefix_map, file_path, 'file') + }) + ) + else + -- Add any file paths directly given on the CLI, even if they do not end + -- in .moon + table.insert(files, { + path, output_for(output_to, prefix_map, path, 'file') + }) + return files + +lint_for = (files) -> + local has_linted_with_error + lint = require "moonscript.cmd.lint" + + for tuple in *files + {filename, _target} = tuple + res, err = lint.lint_file filename + if res + has_linted_with_error = true + io.stderr\write res .. "\n\n" + elseif err + has_linted_with_error = true + io.stderr\write filename .. "\n" .. err.. "\n\n" + + if has_linted_with_error + os.exit 1 + +compile_for = (opts, files) -> + for tuple in *files do + {filename, target} = tuple + if opts.o + target = opts.o + + success, err = compile_and_write filename, target, + print: opts.p + filename: filename + benchmark: opts.b + show_posmap: opts.X + show_parse_tree: opts.T + transform_module: opts.transform + + unless success + io.stderr\write filename .. "\t" .. err .. "\n" + os.exit 1 -- similar to mkdir -p mkdir = (path) -> @@ -23,25 +241,6 @@ mkdir = (path) -> lfs.attributes path, "mode" --- strips excess / and ensures path ends with / -normalize_dir = (path) -> - path\match("^(.-)[#{dirsep_chars}]*$") .. dirsep - --- parse the directory out of a path -parse_dir = (path) -> - (path\match "^(.-)[^#{dirsep_chars}]*$") - --- parse the filename out of a path -parse_file = (path) -> - (path\match "^.-([^#{dirsep_chars}]*)$") - --- converts .moon to a .lua path for calcuating compile target -convert_path = (path) -> - new_path = path\gsub "%.moon$", ".lua" - if new_path == path - new_path = path .. ".lua" - new_path - format_time = (time) -> "%.3fms"\format time*1000 @@ -105,7 +304,7 @@ compile_file_text = (text, opts={}) -> if opts.benchmark print table.concat { - opts.fname or "stdin", + opts.filename or "stdin", "Parse time \t" .. format_time(parse_time), "Compile time\t" .. format_time(compile_time), "" @@ -114,9 +313,9 @@ compile_file_text = (text, opts={}) -> code -write_file = (fname, code) -> - mkdir parse_dir fname - f, err = io.open fname, "w" +write_file = (filename, code) -> + mkdir parse_dir filename + f, err = io.open filename, "w" unless f return nil, err @@ -147,51 +346,83 @@ compile_and_write = (src, dest, opts={}) -> write_file dest, code -is_abs_path = (path) -> - first = path\sub 1, 1 - if dirsep == "\\" - first == "/" or first == "\\" or path\sub(2,1) == ":" +-- - If output_to isn't given... then the output path is the same as the +-- input, except .moon -> .lua +-- - If output_to is given, and the path is relative and inclusive, like +-- "src", then the transform is like "src/foo.moon" -> +-- "output_to/src/foo.lua" +-- - If output_to is given, and the path is relative and exclusive, like +-- "src/", then the transform is "src/foo.moon" -> "output_to/foo.lua" +-- - If output_to is given, and the path is absolute: the output path is the +-- same as if it were relative, just relocated under output_to as if that +-- were / +add_prefix_for_dir = (output_to, prefix_map, directory) -> + last = directory\sub #directory, #directory + is_exclusive = if dirsep == "\\" + last == "/" or last == "\\" else - first == dirsep - - --- calcuate where a path should be compiled to --- target_dir: the directory to place the file (optional, from -t flag) --- base_dir: the directory where the file came from when globbing recursively -path_to_target = (path, target_dir=nil, base_dir=nil) -> - target = convert_path path - - if target_dir - target_dir = normalize_dir target_dir + last == dirsep - if base_dir and target_dir - -- one directory back - head = base_dir\match("^(.-)[^#{dirsep_chars}]*[#{dirsep_chars}]?$") - - if head - start, stop = target\find head, 1, true - if start == 1 - target = target\sub(stop + 1) - - if target_dir - if is_abs_path target - target = parse_file target - - target = target_dir .. target - - target + directory = normalize_dir directory + if prefix_start = output_to + if is_exclusive + prefix_map[directory] = prefix_start .. parse_subtree directory + else + prefix_map[directory] = prefix_start .. directory + else + prefix_map[directory] = directory + +-- Returns the corresponding .lua output for a given .moon path +output_for = (output_to, prefix_map, path, path_type) -> + unless path_type == "directory" + path = convert_path path + -- Handle inclusive and exclusive directories + for prefix_path, prefix_output in pairs prefix_map + if #path < #prefix_path + continue + + -- The given path is a child of one of the prefix directories + if path\sub(1, #prefix_path) == prefix_path + output_path = prefix_output .. path\sub(#prefix_path + 1, #path) + return output_path + -- Otherwise just apply output_to if set + if output_to + path = output_to .. path + return path + +process_filesystem_tree = (root, directory_callback, file_callback) -> + root = normalize_dir root + directory_callback root if directory_callback + ok, _iter, dirobj = pcall(lfs.dir, root) + return unless ok + while true + filename = dirobj\next! + break unless filename + unless filename\match("^%.") + fpath = root .. filename + mode = lfs.attributes(fpath, "mode") + switch mode + when "directory" + fpath = fpath .. '/' + process_filesystem_tree fpath, directory_callback, file_callback + when "file" + file_callback fpath if file_callback + when nil + nil -- TODO? log path ceasing to exist between dir() and attributes()? + else + error "Unexpected filetype #{mode}" { - :dirsep + :main + :mkdir - :normalize_dir - :parse_dir - :parse_file - :convert_path :gettime :format_time - :path_to_target :compile_file_text :compile_and_write + :process_filesystem_tree + + :parse_cli_paths + :output_for } diff --git a/moonscript/cmd/path_handling.lua b/moonscript/cmd/path_handling.lua new file mode 100644 index 00000000..e5d0bf1d --- /dev/null +++ b/moonscript/cmd/path_handling.lua @@ -0,0 +1,78 @@ +local dirsep, dirsep_chars, is_abs_path, normalize_dir, normalize_path, parse_dir, parse_file, parse_subtree, parse_root, convert_path, iterate_path +dirsep = package.config:sub(1, 1) +if dirsep == "\\" then + dirsep_chars = "\\/" +else + dirsep_chars = dirsep +end +is_abs_path = function(path) + local first = path:sub(1, 1) + if dirsep == "\\" then + return first == "/" or first == "\\" or path:sub(2, 1) == ":" + else + return first == dirsep + end +end +normalize_dir = function(path) + local normalized_dir + if is_abs_path(path) then + normalized_dir = dirsep + else + normalized_dir = "" + end + for path_element in iterate_path(path) do + normalized_dir = normalized_dir .. (path_element .. dirsep) + end + return normalized_dir +end +normalize_path = function(path) + local path_elements = { } + for path_element in iterate_path(path) do + table.insert(path_elements, path_element) + end + local normalized_path + if is_abs_path(path) then + normalized_path = dirsep + else + normalized_path = "" + end + for i = 1, #path_elements - 1 do + local path_element = path_elements[i] + normalized_path = normalized_path .. (path_element .. dirsep) + end + return normalized_path .. path_elements[#path_elements] +end +parse_dir = function(path) + return (path:match("^(.-)[^" .. tostring(dirsep_chars) .. "]*$")) +end +parse_file = function(path) + return (path:match("^.-([^" .. tostring(dirsep_chars) .. "]*)$")) +end +parse_subtree = function(path) + return (path:match("^.-[" .. tostring(dirsep_chars) .. "]+(.*)$")) +end +parse_root = function(path) + return (path:match("^(.-[" .. tostring(dirsep_chars) .. "]+).*$")) +end +convert_path = function(path) + local new_path = path:gsub("%.moon$", ".lua") + if new_path == path then + new_path = path .. ".lua" + end + return new_path +end +iterate_path = function(path) + return path:gmatch("([^" .. tostring(dirsep_chars) .. "]+)") +end +return { + dirsep = dirsep, + is_abs_path = is_abs_path, + normalize_dir = normalize_dir, + normalize_path = normalize_path, + parse_dir = parse_dir, + parse_file = parse_file, + parse_subtree = parse_subtree, + parse_root = parse_root, + convert_path = convert_path, + iterate_path = iterate_path +} diff --git a/moonscript/cmd/path_handling.moon b/moonscript/cmd/path_handling.moon new file mode 100644 index 00000000..cb35fab1 --- /dev/null +++ b/moonscript/cmd/path_handling.moon @@ -0,0 +1,85 @@ +-- Path-handling functions; these are on their own to allow test filesystem +-- stub functions to make use of them independently of the modules they are +-- testing +local * + +dirsep = package.config\sub 1,1 +dirsep_chars = if dirsep == "\\" + "\\/" -- windows +else + dirsep + +is_abs_path = (path) -> + first = path\sub 1, 1 + if dirsep == "\\" + first == "/" or first == "\\" or path\sub(2,1) == ":" + else + first == dirsep + +-- Strips excess / and ensures path ends with / +normalize_dir = (path) -> + normalized_dir = if is_abs_path(path) + dirsep + else + "" + for path_element in iterate_path(path) + normalized_dir ..= path_element .. dirsep + return normalized_dir + +-- Strips excess and trailing / +normalize_path = (path) -> + path_elements = {} + for path_element in iterate_path(path) + table.insert path_elements, path_element + + normalized_path = if is_abs_path(path) + dirsep + else + "" + + for i = 1, #path_elements - 1 + path_element = path_elements[i] + normalized_path ..= path_element .. dirsep + return normalized_path .. path_elements[#path_elements] + +-- parse the directory out of a path +parse_dir = (path) -> + (path\match "^(.-)[^#{dirsep_chars}]*$") + +-- parse the filename out of a path +parse_file = (path) -> + (path\match "^.-([^#{dirsep_chars}]*)$") + +-- parse the subtree (all but the top directory) out of a path +-- Invariants: +-- If input is already normalized, the output will also be in normalized form +parse_subtree = (path) -> + (path\match "^.-[#{dirsep_chars}]+(.*)$") + +-- parse the very first directory out of a path +parse_root = (path) -> + (path\match "^(.-[#{dirsep_chars}]+).*$") + +-- converts .moon to a .lua path for calcuating compile target +convert_path = (path) -> + new_path = path\gsub "%.moon$", ".lua" + if new_path == path + new_path = path .. ".lua" + new_path + +-- Iterates over the directories (and file) in a path +iterate_path = (path) -> + path\gmatch "([^#{dirsep_chars}]+)" + +{ + :dirsep + :is_abs_path + :normalize_dir + :normalize_path + :parse_dir + :parse_file + :parse_subtree + :parse_root + :convert_path + :iterate_path +} diff --git a/moonscript/cmd/watchers.lua b/moonscript/cmd/watchers.lua index 7e266b8d..cc41a528 100644 --- a/moonscript/cmd/watchers.lua +++ b/moonscript/cmd/watchers.lua @@ -1,3 +1,14 @@ +local lfs = require("lfs") +local moonc = require("moonscript.cmd.moonc") +local unpack +unpack = require("moonscript.util").unpack +local process_filesystem_tree +process_filesystem_tree = moonc.process_filesystem_tree +local iterate_path, parse_dir, parse_subtree, parse_root, dirsep, normalize_dir, normalize_path +do + local _obj_0 = require("moonscript.cmd.path_handling") + iterate_path, parse_dir, parse_subtree, parse_root, dirsep, normalize_dir, normalize_path = _obj_0.iterate_path, _obj_0.parse_dir, _obj_0.parse_subtree, _obj_0.parse_root, _obj_0.dirsep, _obj_0.normalize_dir, _obj_0.normalize_path +end local remove_dupes remove_dupes = function(list, key_fn) local seen = { } @@ -40,14 +51,31 @@ do local _class_0 local _base_0 = { start_msg = "Starting watch loop (Ctrl-C to exit)", + output_for = function(self, path, path_type) + return moonc.output_for(self.output_to, self.prefix_map, path, path_type) + end, print_start = function(self, mode, misc) return io.stderr:write(tostring(self.start_msg) .. " with " .. tostring(mode) .. " [" .. tostring(misc) .. "]\n") + end, + valid_moon_file = function(self, file) + file = normalize_path(file) + return file:match("%.moon$") ~= nil or self.initial_files[file] ~= nil end } _base_0.__index = _base_0 _class_0 = setmetatable({ - __init = function(self, file_list) - self.file_list = file_list + __init = function(self, output_to, initial_paths, prefix_map) + self.output_to, self.initial_paths, self.prefix_map = output_to, initial_paths, prefix_map + self.initial_files = { } + local _list_0 = self.initial_paths + for _index_0 = 1, #_list_0 do + local path_tuple = _list_0[_index_0] + local path, path_type + path, path_type = path_tuple[1], path_tuple[2] + if path_type == "file" then + self.initial_files[normalize_path(path)] = true + end + end end, __base = _base_0, __name = "Watcher" @@ -62,84 +90,169 @@ do _base_0.__class = _class_0 Watcher = _class_0 end -local InotifyWacher +local InotifyWatcher do local _class_0 local _parent_0 = Watcher local _base_0 = { - get_dirs = function(self) - local parse_dir - parse_dir = require("moonscript.cmd.moonc").parse_dir - local dirs - do - local _accum_0 = { } - local _len_0 = 1 - local _list_0 = self.file_list - for _index_0 = 1, #_list_0 do - local _des_0 = _list_0[_index_0] - local file_path - file_path = _des_0[1] - local dir = parse_dir(file_path) - if dir == "" then - dir = "./" - end - local _value_0 = dir - _accum_0[_len_0] = _value_0 - _len_0 = _len_0 + 1 - end - dirs = _accum_0 - end - return remove_dupes(dirs) - end, each_update = function(self) + self:print_start("inotify", (plural(#self.initial_paths, "path"))) return coroutine.wrap(function() - local dirs = self:get_dirs() - self:print_start("inotify", plural(#dirs, "dir")) - local wd_table = { } - local inotify = require("inotify") - local handle = inotify.init() - for _index_0 = 1, #dirs do - local dir = dirs[_index_0] - local wd = handle:addwatch(dir, inotify.IN_CLOSE_WRITE, inotify.IN_MOVED_TO) - wd_table[wd] = dir - end + self:register_initial_watchers() while true do - local events = handle:read() - if not (events) then - break + local events, err_msg, err_no = self.handle:read() + if events == nil then + error("Error reading events from inotify handle, errno " .. tostring(err_no) .. ", message: " .. tostring(err_msg)) end for _index_0 = 1, #events do - local _continue_0 = false - repeat - local ev = events[_index_0] - local fname = ev.name - if not (fname:match("%.moon$")) then - _continue_0 = true - break - end - local dir = wd_table[ev.wd] - if dir ~= "./" then - fname = dir .. fname - end - coroutine.yield(fname) - _continue_0 = true - until true - if not _continue_0 then - break - end + local ev = events[_index_0] + self:handle_event(ev) end end end) + end, + register_initial_watchers = function(self) + local _list_0 = self.initial_paths + for _index_0 = 1, #_list_0 do + local path_tuple = _list_0[_index_0] + local path, path_type + path, path_type = path_tuple[1], path_tuple[2] + if path_type == "file" then + self:register_watcher(path, path_type, self.file_event_types) + else + self:register_recursive_watchers(path) + end + end + end, + register_watcher = function(self, path, path_type, events) + if not (self.path_map[path]) then + local wd = self.handle:addwatch(path, unpack(events)) + self.wd_map[wd] = { + path, + path_type + } + self.path_map[path] = wd + self:seen_path_type(path, path_type) + if path_type == "file" then + return coroutine.yield({ + "changedfile", + path, + (self:output_for(path, path_type)) + }) + end + end + end, + register_recursive_watchers = function(self, path) + local directory_cb + directory_cb = function(directory_path) + return self:register_watcher(directory_path, "directory", self.dir_event_types) + end + local file_cb + file_cb = function(file_path) + if self:valid_moon_file(file_path) then + self:seen_path_type(file_path, "file") + return coroutine.yield({ + "changedfile", + file_path, + (self:output_for(file_path, "file")) + }) + end + end + return process_filesystem_tree(path, directory_cb, file_cb) + end, + handle_event = function(self, ev) + local path_tuple = self.wd_map[ev.wd] + if not (path_tuple) then + return + end + local path, path_type + path, path_type = path_tuple[1], path_tuple[2] + local is_dir = path_type == 'directory' + local _exp_0 = ev.mask + if self.inotify.IN_CLOSE_WRITE == _exp_0 then + if is_dir then + local subpath = path .. ev.name + local subpath_type = lfs.attributes(subpath, "mode") + if subpath_type == "file" and self:valid_moon_file(subpath) then + return coroutine.yield({ + "changedfile", + subpath, + (self:output_for(subpath, subpath_type)) + }) + end + else + return coroutine.yield({ + "changedfile", + path, + (self:output_for(path, path_type)) + }) + end + elseif self.inotify.IN_DELETE_SELF == _exp_0 or self.inotify.IN_MOVE_SELF == _exp_0 then + self.handle:rmwatch(ev.wd) + self.wd_map[ev.wd] = nil + self.path_map[path] = nil + self.path_type_map[path] = nil + return coroutine.yield({ + "removed", + path, + (self:output_for(path, path_type)), + path_type + }) + elseif self.inotify.IN_DELETE == _exp_0 or self.inotify.IN_MOVED_TO == _exp_0 then + local subpath = path .. ev.name + local subpath_type = self.path_type_map[subpath] + self.path_type_map[subpath] = nil + return coroutine.yield({ + "removed", + subpath, + (self:output_for(subpath, subpath_type)), + subpath_type + }) + elseif self.inotify.IN_CREATE == _exp_0 then + local subpath = path .. ev.name + local subpath_type = lfs.attributes(subpath, "mode") + if subpath_type == "directory" then + return self:register_recursive_watchers(normalize_dir(subpath)) + else + if self:valid_moon_file(subpath) then + return self:seen_path_type(subpath, subpath_type) + end + end + end + end, + seen_path_type = function(self, path, path_type) + path = normalize_path(path) + if not (self.path_type_map[path]) then + self.path_type_map[path] = path_type + end end } _base_0.__index = _base_0 setmetatable(_base_0, _parent_0.__base) _class_0 = setmetatable({ - __init = function(self, ...) - return _class_0.__parent.__init(self, ...) + __init = function(self, input_paths, output_to, prefix_map) + _class_0.__parent.__init(self, input_paths, output_to, prefix_map) + self.wd_map = { } + self.path_map = { } + self.path_type_map = { } + self.inotify = require("inotify") + self.file_event_types = { + self.inotify.IN_CLOSE_WRITE, + self.inotify.IN_MOVE_SELF, + self.inotify.IN_DELETE_SELF + } + self.dir_event_types = { + self.inotify.IN_CLOSE_WRITE, + self.inotify.IN_MOVE_SELF, + self.inotify.IN_DELETE_SELF, + self.inotify.IN_CREATE, + self.inotify.IN_MOVED_TO, + self.inotify.IN_DELETE + } + self.handle = self.inotify.init() end, __base = _base_0, - __name = "InotifyWacher", + __name = "InotifyWatcher", __parent = _parent_0 }, { __index = function(cls, name) @@ -169,7 +282,7 @@ do if _parent_0.__inherited then _parent_0.__inherited(_parent_0, _class_0) end - InotifyWacher = _class_0 + InotifyWatcher = _class_0 end local SleepWatcher do @@ -189,50 +302,113 @@ do return sleep end, each_update = function(self) + self:print_start("polling", (plural(#self.initial_paths, "path"))) return coroutine.wrap(function() - local lfs = require("lfs") - local sleep = self:get_sleep_func() - self:print_start("polling", plural(#self.file_list, "files")) - local mod_time = { } while true do - local _list_0 = self.file_list - for _index_0 = 1, #_list_0 do - local _continue_0 = false - repeat - local _des_0 = _list_0[_index_0] - local file - file = _des_0[1] - local time = lfs.attributes(file, "modification") - if not (time) then - mod_time[file] = nil - _continue_0 = true - break - end - if not (mod_time[file]) then - mod_time[file] = time - _continue_0 = true - break - end - if time > mod_time[file] then - mod_time[file] = time - coroutine.yield(file) - end - _continue_0 = true - until true - if not _continue_0 then - break - end - end - sleep(self.polling_rate) + self:scan_path_times() + self:remove_missing_paths() + self.sleep(self.polling_rate) end end) + end, + scan_path_times = function(self) + local _list_0 = self.initial_paths + for _index_0 = 1, #_list_0 do + local path_tuple = _list_0[_index_0] + local path, path_type + path, path_type = path_tuple[1], path_tuple[2] + local is_dir = path_type == 'directory' + if not (is_dir) then + self:process_file_time(path) + else + local directory_cb + directory_cb = function(directory_path) + return self:process_directory_time(directory_path) + end + local file_cb + file_cb = function(file_path) + return self:process_file_time(file_path) + end + process_filesystem_tree(path, directory_cb, file_cb) + end + end + end, + process_file_time = function(self, file) + local time = lfs.attributes(file, "modification") + if not (time) then + return + end + if not (self:valid_moon_file(file)) then + return + end + local output = self:output_for(file, 'file') + if not (self.mod_time[file]) then + self.mod_time[file] = time + self.path_type_map[file] = 'file' + return coroutine.yield({ + "changedfile", + file, + output + }) + elseif time ~= self.mod_time[file] then + self.mod_time[file] = time + return coroutine.yield({ + "changedfile", + file, + output + }) + end + end, + process_directory_time = function(self, directory) + local time = lfs.attributes(directory, "modification") + if not (time) then + return + end + directory = normalize_dir(directory) + if not (self.mod_time[directory]) then + self.path_type_map[directory] = 'directory' + end + end, + remove_missing_paths = function(self) + for path, path_type in pairs(self.path_type_map) do + local _continue_0 = false + repeat + local time = lfs.attributes(path, "modification") + if time then + _continue_0 = true + break + end + self.path_type_map[path] = nil + self.mod_time[path] = nil + coroutine.yield({ + "removed", + path, + (self:output_for(path, path_type)), + path_type + }) + _continue_0 = true + until true + if not _continue_0 then + break + end + end end } _base_0.__index = _base_0 setmetatable(_base_0, _parent_0.__base) _class_0 = setmetatable({ - __init = function(self, ...) - return _class_0.__parent.__init(self, ...) + __init = function(self, input_paths, output_to, prefix_map) + _class_0.__parent.__init(self, input_paths, output_to, prefix_map) + self.sleep = self:get_sleep_func() + self.mod_time = { } + self.path_type_map = { } + local _list_0 = self.initial_paths + for _index_0 = 1, #_list_0 do + local path_tuple = _list_0[_index_0] + local path, path_type + path, path_type = path_tuple[1], path_tuple[2] + self.path_type_map[path] = path_type + end end, __base = _base_0, __name = "SleepWatcher", @@ -264,5 +440,5 @@ end return { Watcher = Watcher, SleepWatcher = SleepWatcher, - InotifyWacher = InotifyWacher + InotifyWatcher = InotifyWatcher } diff --git a/moonscript/cmd/watchers.moon b/moonscript/cmd/watchers.moon index c147fc5e..721717a9 100644 --- a/moonscript/cmd/watchers.moon +++ b/moonscript/cmd/watchers.moon @@ -1,3 +1,9 @@ +lfs = require "lfs" +moonc = require "moonscript.cmd.moonc" +import unpack from require "moonscript.util" +import process_filesystem_tree from moonc +import iterate_path, parse_dir, parse_subtree, parse_root, dirsep, normalize_dir, normalize_path from require "moonscript.cmd.path_handling" + remove_dupes = (list, key_fn) -> seen = {} return for item in *list @@ -9,56 +15,187 @@ remove_dupes = (list, key_fn) -> plural = (count, word) -> "#{count} #{word}#{count == 1 and "" or "s"}" --- files is a list of tuples, {source, target} +-- input_paths is the raw list of files and directories passed to moonc +-- output_to is the folder specified by --output-to, or nil if not set class Watcher start_msg: "Starting watch loop (Ctrl-C to exit)" - new: (@file_list) => + new: (@output_to, @initial_paths, @prefix_map) => + -- Track files given in initial_paths, as unlike other files they will be + -- compiled even if they do not have the .moon extension + @initial_files = {} + for path_tuple in *@initial_paths + {path, path_type} = path_tuple + if path_type == "file" + @initial_files[normalize_path path] = true + + output_for: (path, path_type) => + moonc.output_for @output_to, @prefix_map, path, path_type print_start: (mode, misc) => io.stderr\write "#{@start_msg} with #{mode} [#{misc}]\n" -class InotifyWacher extends Watcher + -- We only compile .moon files, and non-.moon files given directly on the CLI + valid_moon_file: (file) => + file = normalize_path file + return file\match("%.moon$") != nil or @initial_files[file] != nil + +class InotifyWatcher extends Watcher @available: => pcall -> require "inotify" - get_dirs: => - import parse_dir from require "moonscript.cmd.moonc" - dirs = for {file_path} in *@file_list - dir = parse_dir file_path - dir = "./" if dir == "" - dir + new: (input_paths, output_to, prefix_map) => + super input_paths, output_to, prefix_map + + -- Maps wd handles to path tuples of watched paths + @wd_map = {} + -- Maps watched paths (not path tuples) to wd handles + @path_map = {} + -- Maps all 'seen' (not only those directly watched) paths to path type; + -- needed in order to sanely delete 'orphaned' output paths + @path_type_map = {} - remove_dupes dirs + @inotify = require "inotify" + @file_event_types = { + @inotify.IN_CLOSE_WRITE, + @inotify.IN_MOVE_SELF, + @inotify.IN_DELETE_SELF, + } + @dir_event_types = { + @inotify.IN_CLOSE_WRITE, + @inotify.IN_MOVE_SELF, + @inotify.IN_DELETE_SELF, + @inotify.IN_CREATE, + @inotify.IN_MOVED_TO, + @inotify.IN_DELETE, + } - -- creates an iterator that yields a file every time it's updated - -- TODO: detect when new files are added to directories + @handle = @inotify.init! + + -- Creates an iterator that yields an 'event tuple', with the specific + -- structure depending on the type of event. + -- Event types are "changedfile", for new/modified Moonscript files, and + -- "removed", for a previously-seen Moonscript file being deleted. + -- {"changedfile", source_file, target_file} + -- {"removed", source_path, target_path, path_type} each_update: => + @print_start "inotify", (plural #@initial_paths, "path") coroutine.wrap -> - dirs = @get_dirs! + @register_initial_watchers! + -- Wait for & process events + while true + events, err_msg, err_no = @handle\read! + if events == nil + error "Error reading events from inotify handle, errno #{err_no}, message: #{err_msg}" + + for ev in *events + @handle_event ev - @print_start "inotify", plural #dirs, "dir" + register_initial_watchers: () => + -- Register watchers for initial set of files and directories. Newly + -- created subdirectories will get watchers added dynamically. + for path_tuple in *@initial_paths + {path, path_type} = path_tuple + if path_type == "file" + @register_watcher path, path_type, @file_event_types + else + @register_recursive_watchers path - wd_table = {} + register_watcher: (path, path_type, events) => + -- We must guard against duplicate registrations due to a race condition: + -- 1. New subdirectory is created or moved into place, event A is fired on + -- its parent + -- 2. We process event A and add a watcher + -- 3. Before we finish recursively scanning the new subdirectory for + -- further directories to register, a new sub-subdirectory X is created, + -- and an event B is thus triggered on the subdirectory + -- 4. We finish adding new subchildren, including the one created mid-scan, + -- X + -- 4. When event B gets processed, it will trigger a duplicate registration + -- for sub-subdirectory X + unless @path_map[path] + wd = @handle\addwatch path, unpack events + @wd_map[wd] = {path, path_type} + @path_map[path] = wd + @seen_path_type path, path_type + if path_type == "file" + -- Initial compile + coroutine.yield {"changedfile", path, (@output_for path, path_type)} - inotify = require "inotify" - handle = inotify.init! + register_recursive_watchers: (path) => + directory_cb = (directory_path) -> + @register_watcher directory_path, "directory", @dir_event_types + file_cb = (file_path) -> + if @valid_moon_file file_path + @seen_path_type file_path, "file" + coroutine.yield {"changedfile", file_path, (@output_for file_path, "file")} + -- Handles recursively traversing the directory tree starting at path, + -- calling the provided callbacks for children along the way (does + -- pre-order traversal based on whatever ordering lfs.dir() uses) + process_filesystem_tree path, directory_cb, file_cb - for dir in *dirs - wd = handle\addwatch dir, inotify.IN_CLOSE_WRITE, inotify.IN_MOVED_TO - wd_table[wd] = dir + handle_event: (ev) => + path_tuple = @wd_map[ev.wd] + unless path_tuple + -- It's possible to get events for paths after they have been deleted and + -- unregistered from the wd_map; we just skip to the next event in this + -- case. + return + {path, path_type} = path_tuple + is_dir = path_type == 'directory' - while true - events = handle\read! - break unless events -- error? + switch ev.mask + when @inotify.IN_CLOSE_WRITE -- On both files and dirs + -- A file has been created or modified, spit out a {target, output} + -- tuple. - for ev in *events - fname = ev.name - continue unless fname\match "%.moon$" - dir = wd_table[ev.wd] - fname = dir .. fname if dir != "./" + if is_dir + subpath = path .. ev.name + subpath_type = lfs.attributes subpath, "mode" + if subpath_type == "file" and @valid_moon_file subpath + coroutine.yield {"changedfile", subpath, (@output_for subpath, subpath_type)} + else + coroutine.yield {"changedfile", path, (@output_for path, path_type)} + + + when @inotify.IN_DELETE_SELF, @inotify.IN_MOVE_SELF -- On both files and dirs + -- Remove the watch - TODO handle errors from rmwatch? + @handle\rmwatch ev.wd + @wd_map[ev.wd] = nil + @path_map[path] = nil + @path_type_map[path] = nil + + coroutine.yield {"removed", path, (@output_for path, path_type), path_type} + + when @inotify.IN_DELETE, @inotify.IN_MOVED_TO -- On dirs only + subpath = path .. ev.name + -- The path type can't be looked up directly, because the path no + -- longer exists, so we check our list of 'seen' paths in @path_map + subpath_type = @path_type_map[subpath] + + @path_type_map[subpath] = nil + coroutine.yield {"removed", subpath, (@output_for subpath, subpath_type), subpath_type} + + when @inotify.IN_CREATE -- On dirs only + subpath = path .. ev.name + subpath_type = lfs.attributes subpath, "mode" + if subpath_type == "directory" + -- Scan the new subdirectory for any subdirectories of its own, + -- register those, and compile any valid new paths + @register_recursive_watchers normalize_dir subpath + else + -- We don't need to do much for newly-created files, because they + -- also generate an IN_CLOSE_WRITE event which will potentially + -- yielding for them. We could also handle new directories off of + -- IN_CLOSE_WRITE, but it makes a bit more sense to do it on + -- IN_CREATE. + if @valid_moon_file subpath + @seen_path_type subpath, subpath_type + + seen_path_type: (path, path_type) => + path = normalize_path path + unless @path_type_map[path] + @path_type_map[path] = path_type - -- TODO: check to make sure the file was in the original set - coroutine.yield fname class SleepWatcher extends Watcher polling_rate: 1.0 @@ -75,29 +212,86 @@ class SleepWatcher extends Watcher error "Missing sleep function; install LuaSocket" unless sleep sleep + new: (input_paths, output_to, prefix_map) => + super input_paths, output_to, prefix_map + + @sleep = @get_sleep_func! + + -- Maps seen paths to their last-seen modification time + @mod_time = {} + -- Maps seen paths to path type; needed in order to sanely delete + -- 'orphaned' output paths + @path_type_map = {} + + for path_tuple in *@initial_paths + {path, path_type} = path_tuple + @path_type_map[path] = path_type + each_update: => + @print_start "polling", (plural #@initial_paths, "path") + coroutine.wrap -> - lfs = require "lfs" - sleep = @get_sleep_func! + while true + @scan_path_times! + @remove_missing_paths! + @.sleep @polling_rate - @print_start "polling", plural #@file_list, "files" - mod_time = {} + -- Scan all the given files and directories, recursively. + -- The callbacks given to process_filesystem_tree may call coroutine.yeidl(), + -- so keep in mind that calls to this should be done within a coroutine. + scan_path_times: () => + for path_tuple in *@initial_paths + {path, path_type} = path_tuple + is_dir = path_type == 'directory' - while true - for {file} in *@file_list - time = lfs.attributes file, "modification" - unless time -- file no longer exists - mod_time[file] = nil - continue + unless is_dir + @process_file_time path + else + -- Run check for each subfile + directory_cb = (directory_path) -> + @process_directory_time directory_path + file_cb = (file_path) -> + @process_file_time file_path + process_filesystem_tree path, directory_cb, file_cb + + process_file_time: (file) => + time = lfs.attributes file, "modification" + return unless time -- file no longer exists + return unless @valid_moon_file file + + output = @output_for file, 'file' + unless @mod_time[file] -- new file, add timestamp, do initial build + @mod_time[file] = time + @path_type_map[file] = 'file' + coroutine.yield {"changedfile", file, output} + elseif time != @mod_time[file] -- update timestamp and trigger build + -- != instead of > because the user could have e.g. replaced the file + -- with a backed-up copy using a utility that sets modification time + @mod_time[file] = time + coroutine.yield {"changedfile", file, output} + + process_directory_time: (directory) => + time = lfs.attributes directory, "modification" + unless time -- folder no longer exists + return + + directory = normalize_dir directory + unless @mod_time[directory] + -- new directory, register in path map for deletion tracking + @path_type_map[directory] = 'directory' - unless mod_time[file] -- file time scanned - mod_time[file] = time - continue + -- Check previously-registered paths for any that may have been deleted, and + -- delete the corresponding output paths. + remove_missing_paths: () => + for path, path_type in pairs @path_type_map + time = lfs.attributes path, "modification" + -- Skip if it still exists + continue if time - if time > mod_time[file] - mod_time[file] = time - coroutine.yield file + -- Unregister the deleted path + @path_type_map[path] = nil + @mod_time[path] = nil - sleep @polling_rate + coroutine.yield {"removed", path, (@output_for path, path_type), path_type} -{:Watcher, :SleepWatcher, :InotifyWacher} +{:Watcher, :SleepWatcher, :InotifyWatcher} diff --git a/spec/cmd_spec.moon b/spec/cmd_spec.moon index 039973d5..ee65efa4 100644 --- a/spec/cmd_spec.moon +++ b/spec/cmd_spec.moon @@ -1,59 +1,73 @@ - import with_dev from require "spec.helpers" -- TODO: add specs for windows equivalents -describe "moonc", -> - local moonc +describe "path_handling", -> + local path_handling dev_loaded = with_dev -> - moonc = require "moonscript.cmd.moonc" + path_handling = require "moonscript.cmd.path_handling" same = (fn, a, b) -> assert.same b, fn a it "should normalize dir", -> - same moonc.normalize_dir, "hello/world/", "hello/world/" - same moonc.normalize_dir, "hello/world//", "hello/world/" - same moonc.normalize_dir, "", "/" -- wrong - same moonc.normalize_dir, "hello", "hello/" + same path_handling.normalize_dir, "hello/world/", "hello/world/" + same path_handling.normalize_dir, "/hello/world/", "/hello/world/" + same path_handling.normalize_dir, "hello/world//", "hello/world/" + same path_handling.normalize_dir, "/hello/world//", "/hello/world/" + same path_handling.normalize_dir, "hello//world//", "hello/world/" + same path_handling.normalize_dir, "/hello//world//", "/hello/world/" + same path_handling.normalize_dir, "", "" + same path_handling.normalize_dir, "/", "/" + same path_handling.normalize_dir, "hello", "hello/" + same path_handling.normalize_dir, "/hello", "/hello/" it "should parse dir", -> - same moonc.parse_dir, "/hello/world/file", "/hello/world/" - same moonc.parse_dir, "/hello/world/", "/hello/world/" - same moonc.parse_dir, "world", "" - same moonc.parse_dir, "", "" + same path_handling.parse_dir, "/hello/world/file", "/hello/world/" + same path_handling.parse_dir, "/hello/world/", "/hello/world/" + same path_handling.parse_dir, "world", "" + same path_handling.parse_dir, "", "" it "should parse file", -> - same moonc.parse_file, "/hello/world/file", "file" - same moonc.parse_file, "/hello/world/", "" - same moonc.parse_file, "world", "world" - same moonc.parse_file, "", "" + same path_handling.parse_file, "/hello/world/file", "file" + same path_handling.parse_file, "/hello/world/", "" + same path_handling.parse_file, "world", "world" + same path_handling.parse_file, "", "" it "convert path", -> - same moonc.convert_path, "test.moon", "test.lua" - same moonc.convert_path, "/hello/file.moon", "/hello/file.lua" - same moonc.convert_path, "/hello/world/file", "/hello/world/file.lua" + same path_handling.convert_path, "test.moon", "test.lua" + same path_handling.convert_path, "/hello/file.moon", "/hello/file.lua" + same path_handling.convert_path, "/hello/world/file", "/hello/world/file.lua" - it "calculate target", -> - p = moonc.path_to_target + it "iterates paths", -> + single = {"foo"} + single_path = "foo" + nested = {"foo", "bar"} + nested_path = "foo/bar" + nested_file = {"foo", "bar.baz"} + nested_file_path = "foo/bar.baz" - assert.same "test.lua", p "test.moon" - assert.same "hello/world.lua", p "hello/world.moon" - assert.same "compiled/test.lua", p "test.moon", "compiled" + same_iterated_path = (path, comparison_tbl) -> + i = 0 + for path_element in path_handling.iterate_path(path) + i += 1 + assert.same comparison_tbl[i], path_element + assert.same #comparison_tbl, i - assert.same "/home/leafo/test.lua", p "/home/leafo/test.moon" - assert.same "compiled/test.lua", p "/home/leafo/test.moon", "compiled" - assert.same "/compiled/test.lua", p "/home/leafo/test.moon", "/compiled/" + same_iterated_path single_path, single + same_iterated_path nested_path, nested + same_iterated_path nested_file_path, nested_file - assert.same "moonscript/hello.lua", p "moonscript/hello.moon", nil, "moonscript" - assert.same "out/moonscript/hello.lua", p "moonscript/hello.moon", "out", "moonscript" +describe "moonc", -> + local moonc, path_handling - assert.same "out/moonscript/package/hello.lua", - p "moonscript/package/hello.moon", "out", "moonscript/" + dev_loaded = with_dev -> + path_handling = require "moonscript.cmd.path_handling" + moonc = require "moonscript.cmd.moonc" - assert.same "/out/moonscript/package/hello.lua", - p "/home/leafo/moonscript/package/hello.moon", "/out", "/home/leafo/moonscript" + same = (fn, a, b) -> + assert.same b, fn a it "should compile file text", -> assert.same { @@ -62,69 +76,496 @@ describe "moonc", -> moonc.compile_file_text "print'hello'", fname: "test.moon" } - describe "watcher", -> - describe "inotify watcher", -> - it "gets dirs", -> - import InotifyWacher from require "moonscript.cmd.watchers" - watcher = InotifyWacher { - {"hello.moon", "hello.lua"} - {"cool/no.moon", "cool/no.lua"} - } - - assert.same { - "./" - "cool/" - }, watcher\get_dirs! - - describe "parse args", -> - it "parses spec", -> - import parse_spec from require "moonscript.cmd.args" - spec = parse_spec "lt:o:X" - assert.same { - X: {} - o: {value: true} - t: {value: true} - l: {} - }, spec - - it "parses arguments", -> - import parse_arguments from require "moonscript.cmd.args" - out, res = parse_arguments { - "ga:p" - print: "p" - }, {"hello", "word", "-gap"} - - assert.same { - g: true - a: true - p: true - }, out - describe "stubbed lfs", -> - local dirs + local lfs, os_remove before_each -> - dirs = {} package.loaded.lfs = nil + os_remove = package.loaded.os_remove + package.loaded.os.remove = nil dev_loaded["moonscript.cmd.moonc"] = nil - package.loaded.lfs = { - mkdir: (dir) -> table.insert dirs, dir - attributes: -> "directory" - } + import create_io_stubs from require "spec.fs_stubs" + {:stubs, :fs_root} = create_io_stubs! + {lfs: stub_lfs, os: stub_os} = stubs + package.loaded.lfs = stub_lfs + package.loaded.os.remove = stub_os.remove moonc = require "moonscript.cmd.moonc" + lfs = package.loaded.lfs after_each -> package.loaded.lfs = nil + package.loaded.os.remove = os_remove dev_loaded["moonscript.cmd.moonc"] = nil moonc = require "moonscript.cmd.moonc" - it "should make directory", -> - moonc.mkdir "hello/world/directory" - assert.same { - "hello" - "hello/world" - "hello/world/directory" - }, dirs + describe "mkdir", -> + it "should make directory", -> + dirs_in_path = {"hello", "world", "directory"} + + moonc.mkdir "hello/world/directory" + + path = "" + for dir in *dirs_in_path + path ..= dir .. "/" + assert.are.same "directory", lfs.attributes(path, "mode") + + describe "process_filesystem_tree", -> + it "runs callbacks for nodes in a filesystem tree", -> + direct_file = "bar.moon" + lfs.touch direct_file + dir = "foo/" + lfs.mkdir dir + subdir = "foo/sub" + lfs.mkdir subdir + subfile = "foo/baz.moon" + lfs.touch subfile + another_dir = "nak/" + lfs.mkdir another_dir + another_subfile = "nak/baz.moon" + lfs.touch another_subfile + + files = {} + dirs = {} + file_cb = (file) -> + files[file] = true + dir_cb = (dir) -> + dirs[dir] = true + moonc.process_filesystem_tree dir, dir_cb, file_cb + + assert dirs[(path_handling.normalize_dir dir)] + assert dirs[(path_handling.normalize_dir subdir)] + assert files[(path_handling.normalize_path subfile)] + assert.is.Nil dirs[(path_handling.normalize_path another_dir)] + assert.is.Nil files[(path_handling.normalize_path another_subfile)] + + describe "parse_cli_paths", -> + it "errors if not given paths", -> + test_parse = () -> moonc.parse_cli_paths nil, nil + + assert.has_error test_parse, "No paths specified" + + it "errors on missing paths", -> + test_parse = () -> moonc.parse_cli_paths {"non_existent_path"} + + assert.error_matches test_parse, "Error code" + + it "accepts existing directories and files", -> + lfs.mkdir "foo" + lfs.touch "foo/bar" + lfs.mkdir "baz" + + test_parse = () -> moonc.parse_cli_paths {"foo", "foo/bar", "baz"} + + assert.has.no.error test_parse + + describe "output_for", -> + it "maps file output paths", -> + direct_file = "bar.moon" + lfs.touch direct_file + dir = "foo/" + lfs.mkdir dir + subfile = "foo/baz.moon" + lfs.touch subfile + + output_to, _cli_paths, prefix_map = moonc.parse_cli_paths {direct_file, "foo"} + test_output_for = (path, path_type) -> + moonc.output_for output_to, prefix_map, path, path_type + + assert.same "bar.lua", (test_output_for direct_file, "file") + assert.same "foo/baz.lua", (test_output_for subfile, "file") + assert.same "foo/", (test_output_for dir, "directory") + + it "maps file output paths with output-to set", -> + direct_file = "bar.moon" + lfs.touch direct_file + inclusive_dir = "foo/" + lfs.mkdir inclusive_dir + exclusive_dir = "foo_exclusive/" + lfs.mkdir exclusive_dir + inclusive_subfile = "foo/baz.moon" + lfs.touch inclusive_subfile + exclusive_subfile = "foo_exclusive/baz.moon" + lfs.touch exclusive_subfile + + output_to, _cli_paths, prefix_map = moonc.parse_cli_paths {direct_file, "foo", "foo_exclusive/"}, "nak" + test_output_for = (path, path_type) -> + moonc.output_for output_to, prefix_map, path, path_type + + assert.same "nak/bar.lua", (test_output_for direct_file, "file") + assert.same "nak/foo/baz.lua", (test_output_for inclusive_subfile, "file") + assert.same "nak/baz.lua", (test_output_for exclusive_subfile, "file") + assert.same "nak/foo/", (test_output_for inclusive_dir, "directory") + assert.same "nak/", (test_output_for exclusive_dir, "directory") + +describe "watcher", -> + local watchers, moonc, lfs, os_remove, fs_root, stubs + -- TODO why doesn't this declaration work if split over two lines, following a ,? + local direct_file, direct_file_sans_ext, inclusive_dir, exclusive_dir, inclusive_subfile, inclusive_nonmoon_subfile, exclusive_subfile, output_to, test_watcher, all_valid_files, all_files, all_dirs, subdir, valid_file_count, input_paths, prefix_map + + dev_loaded = with_dev -> + moonc = require "moonscript.cmd.moonc" + watchers = require "moonscript.cmd.watchers" + + -- Sets up filesystem stubs so we don't need to test against actual files + -- and directories + before_each -> + package.loaded.lfs = nil + os_remove = package.loaded.os_remove + package.loaded.os.remove = nil + dev_loaded["moonscript.cmd.moonc"] = nil + dev_loaded["moonscript.cmd.watchers"] = nil + + import create_io_stubs from require "spec.fs_stubs" + {:stubs, :fs_root} = create_io_stubs! + {lfs: stub_lfs, os: stub_os} = stubs + package.loaded.lfs = stub_lfs + package.loaded.os.remove = stub_os.remove + + moonc = require "moonscript.cmd.moonc" + watchers = require "moonscript.cmd.watchers" + lfs = package.loaded.lfs + + -- Setup 'filesystem' to use for the tests + direct_file = "bar.moon" + direct_file_sans_ext = "scriptfile" + inclusive_dir = "foo/" + subdir = "foo/sub" + exclusive_dir = "foo_exclusive/" + inclusive_subfile = "foo/baz.moon" + inclusive_nonmoon_subfile = "foo/not_a_moon_file" + exclusive_subfile = "foo_exclusive/baz.moon" + all_valid_files = {direct_file, direct_file_sans_ext, inclusive_subfile, + exclusive_subfile} -- inclusive_nonmoon_subfile is not valid + all_files = {direct_file, direct_file_sans_ext, inclusive_subfile, + inclusive_nonmoon_subfile, exclusive_subfile} + all_dirs = {inclusive_dir, subdir, exclusive_dir} + + for dir in *all_dirs + lfs.mkdir dir + + for file in *all_files + lfs.touch file + + output_to, input_paths, prefix_map = moonc.parse_cli_paths { + "bar.moon", "scriptfile", "foo", "foo_exclusive/" + }, "nak" + + after_each -> + package.loaded.lfs = nil + package.loaded.os.remove = os_remove + dev_loaded["moonscript.cmd.watchers"] = nil + + describe "polling watcher", -> + before_each -> + -- Stub sleep(); it's not actually called during the testing but needs to + -- be there for the import in SleepWatcher.new() + package.loaded.socket = + sleep: () -> nil + test_watcher = watchers.SleepWatcher output_to, input_paths, prefix_map + + after_each -> + package.loaded.socket = nil + + describe "process_file_time", -> + local test_process + before_each -> + test_process = (path) -> + co = coroutine.wrap test_watcher\process_file_time + return co path + + assert_expected_output_for = (file) -> + test_ret = test_process file + assert.not.Nil test_ret + {event_type, path, output} = test_ret + assert.are.same "changedfile", event_type + assert.are.same (test_watcher\output_for file, "file"), (output) + assert.are.same file, path + + it "yields for new files", -> + for file in *all_valid_files + assert_expected_output_for file + + it "does not yield for old files without a change", -> + -- Clear intial yield + test_process direct_file + + assert.is.Nil test_process direct_file + + it "yields for old files with a change", -> + -- Clear intial yield + test_process direct_file + + -- Bump the modification time; +5 because the granularity is only 1 + -- second, and 1 second likely has not passed since the before_each() + -- call created the file + lfs.touch direct_file, nil, os.time! + 5 + + assert_expected_output_for direct_file + + it "does not yield for non-existent files", -> + assert.is.Nil test_process "not a present file" + + it "does not yield for non-.moon files in watched directories", -> + assert.is.Nil test_process inclusive_nonmoon_subfile + + it "does yield for non-.moon files watched directly", -> + assert_expected_output_for direct_file_sans_ext + + describe "scan_path_times", -> + it "initially yields once per valid file", -> + co_scan = coroutine.wrap test_watcher\scan_path_times + + for i = 1, #all_valid_files + ret = co_scan! + {event_type, input, output} = ret + assert.are.same "changedfile", event_type + assert.is.not.Nil ret + assert.is.Nil co_scan! + + it "later yields once per file modification", -> + co_scan = coroutine.wrap test_watcher\scan_path_times + -- Clear initial yields + for i = 1, #all_valid_files + co_scan! + -- Update a file's timestamp + lfs.touch direct_file, nil, os.time! + 5 + -- Make a new coroutine + co_scan = coroutine.wrap test_watcher\scan_path_times + + {event_type, input_file, _output_file} = co_scan! + + -- We get the modified file back + assert.are.same direct_file, input_file + assert.are.same "changedfile", event_type + -- And we don't get any further yields + assert.is.Nil co_scan! + + describe "remove_missing_paths", -> + path_handling = require "moonscript.cmd.path_handling" + before_each -> + -- Prime the system by ensuring the watcher has recorded all the times + co_scan = coroutine.wrap test_watcher\scan_path_times + while co_scan! -- clear initial yields + nil + + remove_input_files = (files) -> + for file in *files + os.remove file + assert.is.Nil lfs.attributes file, "mode" + + remove_all_input_dirs = () -> + -- Reverse order, because children are listed after parents + for i = #all_dirs, 1, -1 + dir = all_dirs[i] + os.remove dir + assert.is.Nil lfs.attributes dir, "mode" + + test_removal = (files_removed) -> + removal_count = 0 + co = coroutine.wrap test_watcher\remove_missing_paths + + path_tuple = co! + while path_tuple != nil + {event_type, path, output, path_type} = path_tuple + assert.are.same "removed", event_type + files_removed[output] = true + removal_count += 1 + path_tuple = co! + + return removal_count + + it "does not generate removal events for output paths that have existing input paths", -> + files_removed = {} + + removal_count = test_removal files_removed + + assert.are.same 0, removal_count + + it "generates removal events for output files that been orphaned", -> + remove_input_files all_valid_files + files_removed = {} + + removal_count = test_removal files_removed + assert.are.same #all_valid_files, removal_count + for file in *all_valid_files + output = test_watcher\output_for file, "file" + assert.is_true files_removed[output] + + it "generates removal events for output directories that have been orphaned", -> + remove_input_files all_files + remove_all_input_dirs! + files_removed = {} + + -- Worst-case could take two passes to remove the dirs, as it does not + -- guarantee removing children before removing dirs + removal_count = test_removal files_removed + removal_count += test_removal files_removed + + assert.are.same #all_valid_files + #all_dirs, removal_count + for dir in *all_dirs + output = test_watcher\output_for dir, "directory" + output = path_handling.normalize_dir output + assert.is.True files_removed[output] + + describe "inotify watcher", -> + local inotify, test_watcher + before_each -> + {inotify: stub_inotify} = stubs + package.loaded.inotify = stub_inotify + inotify = package.loaded.inotify + test_watcher = watchers.InotifyWatcher output_to, input_paths, prefix_map + + after_each -> + package.loaded.inotify = nil + + describe "register_watcher", -> + local test_register + before_each -> + test_register = (path, path_type) -> + co = coroutine.wrap test_watcher\register_watcher + if path_type == "file" + return co path, path_type, test_watcher.file_event_types + elseif path_type == "directory" + return co path, path_type, test_watcher.dir_event_types + else error "Unrecognized type" + + it "yields if the path is a file", -> + ret = test_register direct_file, "file" + + assert.is.not.Nil ret + {event_type, path, output} = ret + assert.are.same "changedfile", event_type + assert.are.same direct_file, path + assert.are.same (test_watcher\output_for direct_file, "file"), output + + it "does not yield if the path is a directory", -> + ret = test_register inclusive_dir, "directory" + + assert.is.Nil ret + + describe "register_recursive_watchers", -> + local test_register_recursive + before_each -> + test_register_recursive = (path) -> + co = coroutine.wrap test_watcher\register_recursive_watchers + return () -> co path, test_watcher.dir_event_types + + it "does not yield for non-.moon files", -> + get_yield = test_register_recursive inclusive_dir + path_tuple = get_yield! + assert.is.not.Nil path_tuple + + while path_tuple + {_event_type, path, output} = path_tuple + assert.are.not.same inclusive_nonmoon_subfile, path + path_tuple = get_yield! + + it "does one yield per valid file", -> + get_yield = test_register_recursive inclusive_dir + valid_subfiles = {inclusive_subfile} + for i = 1, #valid_subfiles + path_tuple = get_yield! + assert.is.not.Nil path_tuple + + describe "register_initial_watchers", -> + it "does not yield for non-.moon files in the tree", -> + co = coroutine.wrap test_watcher\register_initial_watchers + + path_tuple = co! + assert.is.not.Nil path_tuple + while path_tuple + {_event_type, path, output} = path_tuple + assert.are.not.same inclusive_nonmoon_subfile, path + path_tuple = co! + + it "does yield for non-.moon files given directly", -> + yielded_scriptfile = false + co = coroutine.wrap test_watcher\register_initial_watchers + + path_tuple = co! + assert.is.not.Nil path_tuple + while path_tuple + {event_type, path, output} = path_tuple + assert.are.same "changedfile", event_type + if path == direct_file_sans_ext + yielded_scriptfile = true + path_tuple = co! + + assert yielded_scriptfile + + describe "handle_event", -> + local test_handle_event + path_handling = require "moonscript.cmd.path_handling" + before_each -> + -- Register initial watchers + co = coroutine.wrap test_watcher\register_initial_watchers + co_ret = co! + while co_ret != nil + co_ret = co! + + test_handle_event = coroutine.wrap () -> + ev = test_watcher.handle\read! + test_watcher\handle_event ev + + it "yields when directly watched files change", -> + inotify.generate_test_event direct_file, inotify.IN_CLOSE_WRITE + + path_tuple = test_handle_event! + + assert.is.not.Nil path_tuple + {event_type, path, output} = path_tuple + assert.are.same "changedfile", event_type + assert.are.same direct_file, path + assert.are.same (test_watcher\output_for direct_file), output + + it "yields when files in watched directories change", -> + inotify.generate_test_event inclusive_subfile, inotify.IN_CLOSE_WRITE + + path_tuple = test_handle_event! + + assert.is.not.Nil path_tuple + {event_type, path, output} = path_tuple + assert.are.same "changedfile", event_type + assert.are.same inclusive_subfile, path + assert.are.same (test_watcher\output_for inclusive_subfile), output + + it "yields for files in newly-created subdirectories of watched directories", -> + new_subdir = "#{inclusive_dir}new_subdir" + new_file = "#{new_subdir}/afile.moon" + moonc.mkdir new_subdir + lfs.touch new_file + inotify.generate_test_event new_subdir, inotify.IN_CREATE + + path_tuple = test_handle_event! + + assert.is.not.Nil path_tuple + {event_type, path, output} = path_tuple + assert.are.same "changedfile", event_type + assert.are.same new_file, path + assert.are.same (test_watcher\output_for new_file), output + + it "removes orphaned output files", -> + output_path = test_watcher\output_for inclusive_subfile, "file" + inotify.generate_test_event inclusive_subfile, inotify.IN_DELETE + + path_tuple = test_handle_event! + + {event_type, path, output, path_type} = path_tuple + assert.are.same "removed", event_type + assert.are.same inclusive_subfile, path + assert.are.same output_path, output + assert.are.same "file", path_type + + it "removes orphaned output directories", -> + output_path = test_watcher\output_for inclusive_dir, "directory" + inotify.generate_test_event inclusive_dir, inotify.IN_DELETE_SELF + + path_tuple = test_handle_event! + {event_type, path, output, path_type} = path_tuple + assert.are.same "removed", event_type + assert.are.same inclusive_dir, path + assert.are.same output_path, output + assert.are.same "directory", path_type diff --git a/spec/fs_stubs.moon b/spec/fs_stubs.moon new file mode 100644 index 00000000..6306a0e7 --- /dev/null +++ b/spec/fs_stubs.moon @@ -0,0 +1,227 @@ +-- Creates stubs for the subset of filesystem functionality needed/used by +-- tested functions. functions stubbed are from the lfs, os, and inotify +-- modules. +create_io_stubs = () -> + path_handling = require "moonscript.cmd.path_handling" + + new_node = (mode, mtime=os.time!) -> + if mode == "directory" + return { + :mode + children: {} + child_count: 0 + modification: mtime + } + else + return { + :mode + modification: mtime + } + + fs_root = new_node "directory" + + add_child = (fs_node, child_name, child_node) -> + assert fs_node.mode == "directory" + assert fs_node.children[child_name] == nil + fs_node.children[child_name] = child_node + fs_node.child_count += 1 + + remove_child = (fs_node, child_name) -> + assert fs_node.mode == "directory" and fs_node.child_count > 0 + child_node = fs_node.children[child_name] + assert child_node != nil + if child_node.mode == "directory" + if child_node.child_count == 0 + fs_node.children[child_name] = nil + fs_node.child_count -= 1 + return true + else + return nil, "Specified path is not empty", 2 -- TODO + else + fs_node.children[child_name] = nil + fs_node.child_count -= 1 + return true + + traverse_entire_path = (filepath, current_node=fs_root) -> + filepath = path_handling.normalize_path filepath + for path_element in path_handling.iterate_path filepath + if current_node.children != nil and current_node.mode == 'directory' + if next_node = current_node.children[path_element] + current_node = next_node + else + return nil, "No such path" + else + return nil, "Non-final element in path is a file" + return current_node + + -- Returns the parent node and the name of the child + traverse_parent = (filepath, current_node=fs_root) -> + filepath = path_handling.normalize_path filepath + path_elements = {} + for path_element in path_handling.iterate_path filepath + path_elements[#path_elements + 1] = path_element + + if #path_elements > 1 + for i = 1, #path_elements - 1 + path_element = path_elements[i] + if next_node = current_node.children[path_element] + if next_node.mode == 'directory' and next_node.children != nil + current_node = next_node + else + return nil, "Non-final element in path is a file" + else + return nil, "No such path" + return current_node, path_elements[#path_elements] + + lfs = + attributes: (filepath, aname) -> + assert aname == "mode" or aname == "modification", "lfs.attributes(): Not Yet Implemented: only 'mode' and 'modification' attributes currently work" + fs_node, err_msg = traverse_entire_path filepath + + unless fs_node + return nil, err_msg, 2 + + return fs_node[aname] + + rmdir: (dirpath) -> + parent_node, child_name = traverse_parent dirpath + + unless parent_node + return nil, "No such directory", 2 + + -- Remove the last node from its parent + if target_node = parent_node.children[child_name] + if target_node.mode == 'directory' + return remove_child parent_node, child_name + else + return nil, "Specified path is not a directory", 2 + else + return nil, "No such directory", 2 + + mkdir: (dirpath) -> + dirpath = path_handling.normalize_path dirpath + parent_node, child_name = traverse_parent dirpath + + unless parent_node + return nil, "No such directory", 2 + + -- Add a new node to the parent node + unless parent_node.children[child_name] + child_node = new_node "directory" + add_child parent_node, child_name, child_node + return true + else + return nil, "Cannot create a directory with a name identical to an existing path", 2 + + touch: (path, _atime, mtime=os.time!) -> + assert _atime == nil, "lfs.touch(): Not Yet Implemented: only supports setting mtime currently" + parent_node, child_name = traverse_parent path + + file_name = path_handling.parse_file path + + unless parent_node + return nil, "No such directory", 2 + + if target_node = parent_node.children[child_name] + -- Update modtime on existing node + target_node.modification = mtime + else + child_node = new_node "file", mtime + add_child parent_node, child_name, child_node + return true + + dir: (path) -> + dir_node, err_msg = traverse_entire_path path + unless dir_node + error err_msg + if dir_node.mode != "directory" + error "Not a directory" + + local name + dir_obj = + children: dir_node.children + next: () => + name, _val = next @children, name + return name + close: () -> nil + iter = (dir_obj) -> dir_obj\next! + return iter, dir_obj + + os = + remove: (path) -> + parent_node, child_name = traverse_parent path + + unless parent_node + return nil, "No such file or directory", 2 + + -- Remove the last node from its parent + if parent_node.children[child_name] + return remove_child parent_node, child_name + else + return nil, "No such file or directory", 2 + + local inotify + event_queue = + start: 1 + end: 1 + wd_list = + n: 1 + wds_by_path = {} + inotify = + init: () -> + return { + read: () => + if event_queue.start != event_queue.end + ev = event_queue[event_queue.start] + event_queue[event_queue.start] = nil + event_queue.start += 1 + return ev + else + return nil, "inotify handle.read! called with no test events queued", -1 + addwatch: (path, ...) => + event_types = {} + for _, ev_type in ipairs {...} + event_types[ev_type] = true + + wd = wd_list.n + wd_list.n += 1 + wd_list[wd] = {:path, :event_types} + wds_by_path[path] = wd + return wd + rmwatch: (wd) => + wd_list[wd] = nil + -- TODO error codes? + } + + :event_queue + generate_test_event: (path, ev_type) -> + wd = if wds_by_path[path] + -- Direct watcher + wds_by_path[path] + else + -- Parent watcher + wds_by_path[path_handling.parse_dir path] + unless wd and wd_list[wd] + error "Attempted to generate test event for unwatched path #{path}" + unless wd_list[wd].event_types[ev_type] + error "Attempted to generate test event type #{ev_type} for watcher that does not watch that type" + + event = + :wd + mask: ev_type + name: path_handling.parse_file path + event_queue[event_queue.end] = event + event_queue.end += 1 + + IN_CLOSE_WRITE: 1 + IN_MOVE_SELF: 2 + IN_DELETE_SELF: 3 + IN_MOVED_TO: 4 + IN_DELETE: 5 + IN_CREATE: 6 + + stubs = { :lfs, :os, :inotify } + + return { :stubs, :fs_root, :traverse_entire_path, :traverse_parent, :new_node, :add_child, :remove_child } + +{ :create_io_stubs } diff --git a/spec/fs_stubs_spec.moon b/spec/fs_stubs_spec.moon new file mode 100644 index 00000000..73a0cc85 --- /dev/null +++ b/spec/fs_stubs_spec.moon @@ -0,0 +1,191 @@ +import with_dev from require "spec.helpers" + +describe "filesystem stub helpers", -> + local create_io_stubs + + dev_loaded = with_dev -> + import create_io_stubs from require "spec.fs_stubs" + + describe "internal tests", -> + local fs_root, new_node, add_child, remove_child, traverse_entire_path, traverse_parent + + before_each -> + {:fs_root, :traverse_entire_path, :traverse_parent, :new_node, :add_child, :remove_child} = create_io_stubs! + + describe "add_child", -> + local child_name, child_node + before_each -> + child_node = new_node "file" + child_name = "foo" + + it "should increase child_count by 1", -> + old_count = fs_root.child_count + + add_child fs_root, child_name, child_node + + assert.are.same old_count + 1, fs_root.child_count + + it "should make the child reachable from the parent under the given name", -> + assert.is.nil fs_root.children[child_name] + + add_child fs_root, child_name, child_node + + assert.is.not.Nil fs_root.children[child_name] + assert.are.same child_node, fs_root.children[child_name] + + it "should error if there is already a child with that name", -> + add_dup = () -> + add_child fs_root, child_name, child_node + add_dup! + + assert.has.error add_dup + + it "should error if called on a file node", -> + parent_node = new_node "file" + + assert.has.error () -> + add_child parent_node, child_name, child_node + + describe "remove_child", -> + local child_name, child_node + before_each -> + child_node = new_node "file" + child_name = "foo" + add_child fs_root, child_name, child_node + + + it "should decrease child_count by 1", -> + old_count = fs_root.child_count + + remove_child fs_root, child_name + + assert.are.same old_count - 1, fs_root.child_count + + it "should mean no child is reachable from the parent under the given name", -> + assert.is.not.Nil fs_root.children[child_name] + assert.are.same child_node, fs_root.children[child_name] + + remove_child fs_root, child_name + + assert.is.nil fs_root.children[child_name] + + it "should error if there is no child with that name", -> + remove_dup = () -> + remove_child fs_root, child_name + remove_dup! + + assert.has.error remove_dup + + it "should error if called on a file node", -> + parent_node = new_node "file" + + assert.has.error () -> + remove_child parent_node, child_name + + describe "path traversal tests", -> + local valid_path, invalid_path, file_path, previous_node, last_node + + before_each -> + import iterate_path from require "moonscript.cmd.path_handling" + valid_path = "/foo/bar/baz" + invalid_path = "/foo/boo/baz" + file_path = "/foo/bar/file" + current_node = fs_root + previous_node = nil + + for element in iterate_path valid_path + child_node = new_node "directory" + add_child current_node, element, child_node + previous_node = current_node + current_node = child_node + last_node = current_node + + -- Add the file child in file_path + add_child fs_root.children["foo"].children["bar"], + "file", (new_node "file") + + describe "traverse_entire_path", -> + it "returns the node of the last element for a valid path", -> + fs_node, error_message = traverse_entire_path valid_path, fs_root + + assert.is.Nil error_message + assert.is.not.Nil fs_node + assert.are.same last_node, fs_node + + it "returns nil and error message if any element in the path does not exist", -> + fs_node, error_message = traverse_entire_path invalid_path, fs_root + + assert.is.nil fs_node + assert.are.same "No such path", error_message + + it "returns nil and error message if any element save the last in the path is not a directory", -> + fs_node, error_message = traverse_entire_path file_path .. "/nak", fs_root + + assert.is.nil fs_node + assert.are.same "Non-final element in path is a file", error_message + + describe "traverse_parent", -> + it "returns the node of the parent of the last element for a valid path, along with the last element's name", -> + fs_node, child_name = traverse_parent valid_path, fs_root + + assert.is.not.Nil fs_node + assert.are.same previous_node, fs_node + assert.are.same "baz", child_name + + it "returns nil and error message if any element in the path does not exist", -> + fs_node, error_message = traverse_entire_path invalid_path, fs_root + + assert.is.nil fs_node + assert.are.same "No such path", error_message + + it "returns nil and error message if any element save the last in the path is not a directory", -> + fs_node, error_message = traverse_entire_path file_path .. "/nak", fs_root + + assert.is.nil fs_node + assert.are.same "Non-final element in path is a file", error_message + + describe "stub tests", -> + local stubs, fs_root, new_node, add_child, remove_child, traverse_entire_path, traverse_parent + before_each -> + {:stubs, :fs_root, :traverse_entire_path, :traverse_parent, :new_node, :add_child, :remove_child} = create_io_stubs! + + describe "lfs", -> + local lfs + before_each -> + lfs = stubs.lfs + + describe "mkdir()", -> + local dir_name, parent_node, parent_name, dir_path, new_dir_name + + before_each -> + parent_node = new_node "directory" + parent_name = "foo" + add_child fs_root, parent_name, parent_node + new_dir_name = "bar" + dir_path = "/#{parent_name}/#{new_dir_name}" + + it "should make a new child reachable at the given path", -> + fs_node, error_message = traverse_entire_path dir_path + assert.is.Nil fs_node + assert.are.same "No such path", error_message + + is_ok, error_message, error_code = lfs.mkdir dir_path + + assert is_ok + assert.is.Nil error_message + assert.is.Nil error_code + fs_node, error_message = traverse_entire_path dir_path + assert.is.Nil error_message + assert.is.not.Nil fs_node + assert.are.same "table", type(fs_node) + assert.are.same parent_node.children[new_dir_name], fs_node + + it "should fail if there is no existing parent for the given path", -> + remove_child fs_root, parent_name + + is_ok, error_message, error_code = lfs.mkdir dir_path + + assert.is.Nil is_ok + assert.is.not.Nil error_message + assert.is.not.Nil error_code + assert.are.same "No such directory", error_message diff --git a/spec/helpers.moon b/spec/helpers.moon index 45545b91..26e3159f 100644 --- a/spec/helpers.moon +++ b/spec/helpers.moon @@ -1,4 +1,3 @@ - -- remove front indentation from a multiline string, making it suitable to be -- parsed unindent = (str) -> @@ -45,4 +44,5 @@ with_dev = (fn) -> dev_cache + { :unindent, :with_dev }