diff --git a/.gitignore b/.gitignore index 362d16f..9c2bce7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -venv/ -tmp_* -.vscode/ -__pycache__/ \ No newline at end of file +lua_install +luacov.stats.out +test* +.vscode/ \ No newline at end of file diff --git a/.luacheckrc b/.luacheckrc new file mode 100644 index 0000000..c6f086f --- /dev/null +++ b/.luacheckrc @@ -0,0 +1 @@ +ignore = {"631"} \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..73c6ac5 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,27 @@ +language: python +sudo: false + +env: + - LUA="lua=5.1" + - LUA="lua=5.2" + - LUA="lua=5.3" + - LUA="lua=5.4" + +before_install: + - pip install hererocks + - hererocks lua_install -r^ --$LUA + - export PATH=$PATH:$PWD/lua_install/bin + +install: + - luarocks make + - luarocks install luacheck + - luarocks install busted + - luarocks install luacov + - luarocks install luacov-coveralls + +script: + - luacheck --std max+busted src spec + - busted --verbose --coverage + +after_success: + - luacov-coveralls -e $TRAVIS_BUILD_DIR/lua_install diff --git a/README.md b/README.md index 4a68c9a..63fcb25 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,34 @@ -# Article uploader +# Markdown article online assets downloader -Tool for uploading articles to the server +This tool helps to download all images in markdown document and put them into one folder with changed paths + +## Usage: + +``` +md-parser [-o ] [-s ] [-c ] [-u] [-h] + + +Command line utility for saving markdown document online assets locally and upload it to server with scp + +Arguments: + input Input file. + +Options: + -o , Output directory. (default: ./) + --output + -s , Username and hostname in username@hostname notation. + --server + -c , Configuration file like + --config path= Output path on server + host= Server username and hostname in username@hostname notation + -u, --upload If sould upload to server + -h, --help Show this help message and exit. +``` + +It also has integrated functional to upload this article over ssh. As for MIT licence you can freely fork it and modify code for your own usage cases. + +## Example: + +``` +lua src/main.lua spec/assets/tmp_dir/some\ file.md -u -s dm1sh@localhost -o /tmp/art +``` \ No newline at end of file diff --git a/lib/argparse.lua b/lib/argparse.lua new file mode 100644 index 0000000..402f14a --- /dev/null +++ b/lib/argparse.lua @@ -0,0 +1,1527 @@ +-- The MIT License (MIT) + +-- Copyright (c) 2013 - 2018 Peter Melnichenko + +-- Permission is hereby granted, free of charge, to any person obtaining a copy of +-- this software and associated documentation files (the "Software"), to deal in +-- the Software without restriction, including without limitation the rights to +-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +-- the Software, and to permit persons to whom the Software is furnished to do so, +-- subject to the following conditions: + +-- The above copyright notice and this permission notice shall be included in all +-- copies or substantial portions of the Software. + +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +-- FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +-- COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +-- IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +-- CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +local function deep_update(t1, t2) + for k, v in pairs(t2) do + if type(v) == "table" then + v = deep_update({}, v) + end + + t1[k] = v + end + + return t1 +end + +-- A property is a tuple {name, callback}. +-- properties.args is number of properties that can be set as arguments +-- when calling an object. +local function class(prototype, properties, parent) + -- Class is the metatable of its instances. + local cl = {} + cl.__index = cl + + if parent then + cl.__prototype = deep_update(deep_update({}, parent.__prototype), prototype) + else + cl.__prototype = prototype + end + + if properties then + local names = {} + + -- Create setter methods and fill set of property names. + for _, property in ipairs(properties) do + local name, callback = property[1], property[2] + + cl[name] = function(self, value) + if not callback(self, value) then + self["_" .. name] = value + end + + return self + end + + names[name] = true + end + + function cl.__call(self, ...) + -- When calling an object, if the first argument is a table, + -- interpret keys as property names, else delegate arguments + -- to corresponding setters in order. + if type((...)) == "table" then + for name, value in pairs((...)) do + if names[name] then + self[name](self, value) + end + end + else + local nargs = select("#", ...) + + for i, property in ipairs(properties) do + if i > nargs or i > properties.args then + break + end + + local arg = select(i, ...) + + if arg ~= nil then + self[property[1]](self, arg) + end + end + end + + return self + end + end + + -- If indexing class fails, fallback to its parent. + local class_metatable = {} + class_metatable.__index = parent + + function class_metatable.__call(self, ...) + -- Calling a class returns its instance. + -- Arguments are delegated to the instance. + local object = deep_update({}, self.__prototype) + setmetatable(object, self) + return object(...) + end + + return setmetatable(cl, class_metatable) +end + +local function typecheck(name, types, value) + for _, type_ in ipairs(types) do + if type(value) == type_ then + return true + end + end + + error(("bad property '%s' (%s expected, got %s)"):format(name, table.concat(types, " or "), type(value))) +end + +local function typechecked(name, ...) + local types = {...} + return {name, function(_, value) typecheck(name, types, value) end} +end + +local multiname = {"name", function(self, value) + typecheck("name", {"string"}, value) + + for alias in value:gmatch("%S+") do + self._name = self._name or alias + table.insert(self._aliases, alias) + end + + -- Do not set _name as with other properties. + return true +end} + +local function parse_boundaries(str) + if tonumber(str) then + return tonumber(str), tonumber(str) + end + + if str == "*" then + return 0, math.huge + end + + if str == "+" then + return 1, math.huge + end + + if str == "?" then + return 0, 1 + end + + if str:match "^%d+%-%d+$" then + local min, max = str:match "^(%d+)%-(%d+)$" + return tonumber(min), tonumber(max) + end + + if str:match "^%d+%+$" then + local min = str:match "^(%d+)%+$" + return tonumber(min), math.huge + end +end + +local function boundaries(name) + return {name, function(self, value) + typecheck(name, {"number", "string"}, value) + + local min, max = parse_boundaries(value) + + if not min then + error(("bad property '%s'"):format(name)) + end + + self["_min" .. name], self["_max" .. name] = min, max + end} +end + +local actions = {} + +local option_action = {"action", function(_, value) + typecheck("action", {"function", "string"}, value) + + if type(value) == "string" and not actions[value] then + error(("unknown action '%s'"):format(value)) + end +end} + +local option_init = {"init", function(self) + self._has_init = true +end} + +local option_default = {"default", function(self, value) + if type(value) ~= "string" then + self._init = value + self._has_init = true + return true + end +end} + +local add_help = {"add_help", function(self, value) + typecheck("add_help", {"boolean", "string", "table"}, value) + + if self._has_help then + table.remove(self._options) + self._has_help = false + end + + if value then + local help = self:flag() + :description "Show this help message and exit." + :action(function() + print(self:get_help()) + os.exit(0) + end) + + if value ~= true then + help = help(value) + end + + if not help._name then + help "-h" "--help" + end + + self._has_help = true + end +end} + +local Parser = class({ + _arguments = {}, + _options = {}, + _commands = {}, + _mutexes = {}, + _groups = {}, + _require_command = true, + _handle_options = true +}, { + args = 3, + typechecked("name", "string"), + typechecked("description", "string"), + typechecked("epilog", "string"), + typechecked("usage", "string"), + typechecked("help", "string"), + typechecked("require_command", "boolean"), + typechecked("handle_options", "boolean"), + typechecked("action", "function"), + typechecked("command_target", "string"), + typechecked("help_vertical_space", "number"), + typechecked("usage_margin", "number"), + typechecked("usage_max_width", "number"), + typechecked("help_usage_margin", "number"), + typechecked("help_description_margin", "number"), + typechecked("help_max_width", "number"), + add_help +}) + +local Command = class({ + _aliases = {} +}, { + args = 3, + multiname, + typechecked("description", "string"), + typechecked("epilog", "string"), + typechecked("target", "string"), + typechecked("usage", "string"), + typechecked("help", "string"), + typechecked("require_command", "boolean"), + typechecked("handle_options", "boolean"), + typechecked("action", "function"), + typechecked("command_target", "string"), + typechecked("help_vertical_space", "number"), + typechecked("usage_margin", "number"), + typechecked("usage_max_width", "number"), + typechecked("help_usage_margin", "number"), + typechecked("help_description_margin", "number"), + typechecked("help_max_width", "number"), + typechecked("hidden", "boolean"), + add_help +}, Parser) + +local Argument = class({ + _minargs = 1, + _maxargs = 1, + _mincount = 1, + _maxcount = 1, + _defmode = "unused", + _show_default = true +}, { + args = 5, + typechecked("name", "string"), + typechecked("description", "string"), + option_default, + typechecked("convert", "function", "table"), + boundaries("args"), + typechecked("target", "string"), + typechecked("defmode", "string"), + typechecked("show_default", "boolean"), + typechecked("argname", "string", "table"), + typechecked("hidden", "boolean"), + option_action, + option_init +}) + +local Option = class({ + _aliases = {}, + _mincount = 0, + _overwrite = true +}, { + args = 6, + multiname, + typechecked("description", "string"), + option_default, + typechecked("convert", "function", "table"), + boundaries("args"), + boundaries("count"), + typechecked("target", "string"), + typechecked("defmode", "string"), + typechecked("show_default", "boolean"), + typechecked("overwrite", "boolean"), + typechecked("argname", "string", "table"), + typechecked("hidden", "boolean"), + option_action, + option_init +}, Argument) + +function Parser:_inherit_property(name, default) + local element = self + + while true do + local value = element["_" .. name] + + if value ~= nil then + return value + end + + if not element._parent then + return default + end + + element = element._parent + end +end + +function Argument:_get_argument_list() + local buf = {} + local i = 1 + + while i <= math.min(self._minargs, 3) do + local argname = self:_get_argname(i) + + if self._default and self._defmode:find "a" then + argname = "[" .. argname .. "]" + end + + table.insert(buf, argname) + i = i+1 + end + + while i <= math.min(self._maxargs, 3) do + table.insert(buf, "[" .. self:_get_argname(i) .. "]") + i = i+1 + + if self._maxargs == math.huge then + break + end + end + + if i < self._maxargs then + table.insert(buf, "...") + end + + return buf +end + +function Argument:_get_usage() + local usage = table.concat(self:_get_argument_list(), " ") + + if self._default and self._defmode:find "u" then + if self._maxargs > 1 or (self._minargs == 1 and not self._defmode:find "a") then + usage = "[" .. usage .. "]" + end + end + + return usage +end + +function actions.store_true(result, target) + result[target] = true +end + +function actions.store_false(result, target) + result[target] = false +end + +function actions.store(result, target, argument) + result[target] = argument +end + +function actions.count(result, target, _, overwrite) + if not overwrite then + result[target] = result[target] + 1 + end +end + +function actions.append(result, target, argument, overwrite) + result[target] = result[target] or {} + table.insert(result[target], argument) + + if overwrite then + table.remove(result[target], 1) + end +end + +function actions.concat(result, target, arguments, overwrite) + if overwrite then + error("'concat' action can't handle too many invocations") + end + + result[target] = result[target] or {} + + for _, argument in ipairs(arguments) do + table.insert(result[target], argument) + end +end + +function Argument:_get_action() + local action, init + + if self._maxcount == 1 then + if self._maxargs == 0 then + action, init = "store_true", nil + else + action, init = "store", nil + end + else + if self._maxargs == 0 then + action, init = "count", 0 + else + action, init = "append", {} + end + end + + if self._action then + action = self._action + end + + if self._has_init then + init = self._init + end + + if type(action) == "string" then + action = actions[action] + end + + return action, init +end + +-- Returns placeholder for `narg`-th argument. +function Argument:_get_argname(narg) + local argname = self._argname or self:_get_default_argname() + + if type(argname) == "table" then + return argname[narg] + else + return argname + end +end + +function Argument:_get_default_argname() + return "<" .. self._name .. ">" +end + +function Option:_get_default_argname() + return "<" .. self:_get_default_target() .. ">" +end + +-- Returns labels to be shown in the help message. +function Argument:_get_label_lines() + return {self._name} +end + +function Option:_get_label_lines() + local argument_list = self:_get_argument_list() + + if #argument_list == 0 then + -- Don't put aliases for simple flags like `-h` on different lines. + return {table.concat(self._aliases, ", ")} + end + + local longest_alias_length = -1 + + for _, alias in ipairs(self._aliases) do + longest_alias_length = math.max(longest_alias_length, #alias) + end + + local argument_list_repr = table.concat(argument_list, " ") + local lines = {} + + for i, alias in ipairs(self._aliases) do + local line = (" "):rep(longest_alias_length - #alias) .. alias .. " " .. argument_list_repr + + if i ~= #self._aliases then + line = line .. "," + end + + table.insert(lines, line) + end + + return lines +end + +function Command:_get_label_lines() + return {table.concat(self._aliases, ", ")} +end + +function Argument:_get_description() + if self._default and self._show_default then + if self._description then + return ("%s (default: %s)"):format(self._description, self._default) + else + return ("default: %s"):format(self._default) + end + else + return self._description or "" + end +end + +function Command:_get_description() + return self._description or "" +end + +function Option:_get_usage() + local usage = self:_get_argument_list() + table.insert(usage, 1, self._name) + usage = table.concat(usage, " ") + + if self._mincount == 0 or self._default then + usage = "[" .. usage .. "]" + end + + return usage +end + +function Argument:_get_default_target() + return self._name +end + +function Option:_get_default_target() + local res + + for _, alias in ipairs(self._aliases) do + if alias:sub(1, 1) == alias:sub(2, 2) then + res = alias:sub(3) + break + end + end + + res = res or self._name:sub(2) + return (res:gsub("-", "_")) +end + +function Option:_is_vararg() + return self._maxargs ~= self._minargs +end + +function Parser:_get_fullname() + local parent = self._parent + local buf = {self._name} + + while parent do + table.insert(buf, 1, parent._name) + parent = parent._parent + end + + return table.concat(buf, " ") +end + +function Parser:_update_charset(charset) + charset = charset or {} + + for _, command in ipairs(self._commands) do + command:_update_charset(charset) + end + + for _, option in ipairs(self._options) do + for _, alias in ipairs(option._aliases) do + charset[alias:sub(1, 1)] = true + end + end + + return charset +end + +function Parser:argument(...) + local argument = Argument(...) + table.insert(self._arguments, argument) + return argument +end + +function Parser:option(...) + local option = Option(...) + + if self._has_help then + table.insert(self._options, #self._options, option) + else + table.insert(self._options, option) + end + + return option +end + +function Parser:flag(...) + return self:option():args(0)(...) +end + +function Parser:command(...) + local command = Command():add_help(true)(...) + command._parent = self + table.insert(self._commands, command) + return command +end + +function Parser:mutex(...) + local elements = {...} + + for i, element in ipairs(elements) do + local mt = getmetatable(element) + assert(mt == Option or mt == Argument, ("bad argument #%d to 'mutex' (Option or Argument expected)"):format(i)) + end + + table.insert(self._mutexes, elements) + return self +end + +function Parser:group(name, ...) + assert(type(name) == "string", ("bad argument #1 to 'group' (string expected, got %s)"):format(type(name))) + + local group = {name = name, ...} + + for i, element in ipairs(group) do + local mt = getmetatable(element) + assert(mt == Option or mt == Argument or mt == Command, + ("bad argument #%d to 'group' (Option or Argument or Command expected)"):format(i + 1)) + end + + table.insert(self._groups, group) + return self +end + +local usage_welcome = "Usage: " + +function Parser:get_usage() + if self._usage then + return self._usage + end + + local usage_margin = self:_inherit_property("usage_margin", #usage_welcome) + local max_usage_width = self:_inherit_property("usage_max_width", 70) + local lines = {usage_welcome .. self:_get_fullname()} + + local function add(s) + if #lines[#lines]+1+#s <= max_usage_width then + lines[#lines] = lines[#lines] .. " " .. s + else + lines[#lines+1] = (" "):rep(usage_margin) .. s + end + end + + -- Normally options are before positional arguments in usage messages. + -- However, vararg options should be after, because they can't be reliable used + -- before a positional argument. + -- Mutexes come into play, too, and are shown as soon as possible. + -- Overall, output usages in the following order: + -- 1. Mutexes that don't have positional arguments or vararg options. + -- 2. Options that are not in any mutexes and are not vararg. + -- 3. Positional arguments - on their own or as a part of a mutex. + -- 4. Remaining mutexes. + -- 5. Remaining options. + + local elements_in_mutexes = {} + local added_elements = {} + local added_mutexes = {} + local argument_to_mutexes = {} + + local function add_mutex(mutex, main_argument) + if added_mutexes[mutex] then + return + end + + added_mutexes[mutex] = true + local buf = {} + + for _, element in ipairs(mutex) do + if not element._hidden and not added_elements[element] then + if getmetatable(element) == Option or element == main_argument then + table.insert(buf, element:_get_usage()) + added_elements[element] = true + end + end + end + + if #buf == 1 then + add(buf[1]) + elseif #buf > 1 then + add("(" .. table.concat(buf, " | ") .. ")") + end + end + + local function add_element(element) + if not element._hidden and not added_elements[element] then + add(element:_get_usage()) + added_elements[element] = true + end + end + + for _, mutex in ipairs(self._mutexes) do + local is_vararg = false + local has_argument = false + + for _, element in ipairs(mutex) do + if getmetatable(element) == Option then + if element:_is_vararg() then + is_vararg = true + end + else + has_argument = true + argument_to_mutexes[element] = argument_to_mutexes[element] or {} + table.insert(argument_to_mutexes[element], mutex) + end + + elements_in_mutexes[element] = true + end + + if not is_vararg and not has_argument then + add_mutex(mutex) + end + end + + for _, option in ipairs(self._options) do + if not elements_in_mutexes[option] and not option:_is_vararg() then + add_element(option) + end + end + + -- Add usages for positional arguments, together with one mutex containing them, if they are in a mutex. + for _, argument in ipairs(self._arguments) do + -- Pick a mutex as a part of which to show this argument, take the first one that's still available. + local mutex + + if elements_in_mutexes[argument] then + for _, argument_mutex in ipairs(argument_to_mutexes[argument]) do + if not added_mutexes[argument_mutex] then + mutex = argument_mutex + end + end + end + + if mutex then + add_mutex(mutex, argument) + else + add_element(argument) + end + end + + for _, mutex in ipairs(self._mutexes) do + add_mutex(mutex) + end + + for _, option in ipairs(self._options) do + add_element(option) + end + + if #self._commands > 0 then + if self._require_command then + add("") + else + add("[]") + end + + add("...") + end + + return table.concat(lines, "\n") +end + +local function split_lines(s) + if s == "" then + return {} + end + + local lines = {} + + if s:sub(-1) ~= "\n" then + s = s .. "\n" + end + + for line in s:gmatch("([^\n]*)\n") do + table.insert(lines, line) + end + + return lines +end + +local function autowrap_line(line, max_length) + -- Algorithm for splitting lines is simple and greedy. + local result_lines = {} + + -- Preserve original indentation of the line, put this at the beginning of each result line. + -- If the first word looks like a list marker ('*', '+', or '-'), add spaces so that starts + -- of the second and the following lines vertically align with the start of the second word. + local indentation = line:match("^ *") + + if line:find("^ *[%*%+%-]") then + indentation = indentation .. " " .. line:match("^ *[%*%+%-]( *)") + end + + -- Parts of the last line being assembled. + local line_parts = {} + + -- Length of the current line. + local line_length = 0 + + -- Index of the next character to consider. + local index = 1 + + while true do + local word_start, word_finish, word = line:find("([^ ]+)", index) + + if not word_start then + -- Ignore trailing spaces, if any. + break + end + + local preceding_spaces = line:sub(index, word_start - 1) + index = word_finish + 1 + + if (#line_parts == 0) or (line_length + #preceding_spaces + #word <= max_length) then + -- Either this is the very first word or it fits as an addition to the current line, add it. + table.insert(line_parts, preceding_spaces) -- For the very first word this adds the indentation. + table.insert(line_parts, word) + line_length = line_length + #preceding_spaces + #word + else + -- Does not fit, finish current line and put the word into a new one. + table.insert(result_lines, table.concat(line_parts)) + line_parts = {indentation, word} + line_length = #indentation + #word + end + end + + if #line_parts > 0 then + table.insert(result_lines, table.concat(line_parts)) + end + + if #result_lines == 0 then + -- Preserve empty lines. + result_lines[1] = "" + end + + return result_lines +end + +-- Automatically wraps lines within given array, +-- attempting to limit line length to `max_length`. +-- Existing line splits are preserved. +local function autowrap(lines, max_length) + local result_lines = {} + + for _, line in ipairs(lines) do + local autowrapped_lines = autowrap_line(line, max_length) + + for _, autowrapped_line in ipairs(autowrapped_lines) do + table.insert(result_lines, autowrapped_line) + end + end + + return result_lines +end + +function Parser:_get_element_help(element) + local label_lines = element:_get_label_lines() + local description_lines = split_lines(element:_get_description()) + + local result_lines = {} + + -- All label lines should have the same length (except the last one, it has no comma). + -- If too long, start description after all the label lines. + -- Otherwise, combine label and description lines. + + local usage_margin_len = self:_inherit_property("help_usage_margin", 3) + local usage_margin = (" "):rep(usage_margin_len) + local description_margin_len = self:_inherit_property("help_description_margin", 25) + local description_margin = (" "):rep(description_margin_len) + + local help_max_width = self:_inherit_property("help_max_width") + + if help_max_width then + local description_max_width = math.max(help_max_width - description_margin_len, 10) + description_lines = autowrap(description_lines, description_max_width) + end + + if #label_lines[1] >= (description_margin_len - usage_margin_len) then + for _, label_line in ipairs(label_lines) do + table.insert(result_lines, usage_margin .. label_line) + end + + for _, description_line in ipairs(description_lines) do + table.insert(result_lines, description_margin .. description_line) + end + else + for i = 1, math.max(#label_lines, #description_lines) do + local label_line = label_lines[i] + local description_line = description_lines[i] + + local line = "" + + if label_line then + line = usage_margin .. label_line + end + + if description_line and description_line ~= "" then + line = line .. (" "):rep(description_margin_len - #line) .. description_line + end + + table.insert(result_lines, line) + end + end + + return table.concat(result_lines, "\n") +end + +local function get_group_types(group) + local types = {} + + for _, element in ipairs(group) do + types[getmetatable(element)] = true + end + + return types +end + +function Parser:_add_group_help(blocks, added_elements, label, elements) + local buf = {label} + + for _, element in ipairs(elements) do + if not element._hidden and not added_elements[element] then + added_elements[element] = true + table.insert(buf, self:_get_element_help(element)) + end + end + + if #buf > 1 then + table.insert(blocks, table.concat(buf, ("\n"):rep(self:_inherit_property("help_vertical_space", 0) + 1))) + end +end + +function Parser:get_help() + if self._help then + return self._help + end + + local blocks = {self:get_usage()} + + local help_max_width = self:_inherit_property("help_max_width") + + if self._description then + local description = self._description + + if help_max_width then + description = table.concat(autowrap(split_lines(description), help_max_width), "\n") + end + + table.insert(blocks, description) + end + + -- 1. Put groups containing arguments first, then other arguments. + -- 2. Put remaining groups containing options, then other options. + -- 3. Put remaining groups containing commands, then other commands. + -- Assume that an element can't be in several groups. + local groups_by_type = { + [Argument] = {}, + [Option] = {}, + [Command] = {} + } + + for _, group in ipairs(self._groups) do + local group_types = get_group_types(group) + + for _, mt in ipairs({Argument, Option, Command}) do + if group_types[mt] then + table.insert(groups_by_type[mt], group) + break + end + end + end + + local default_groups = { + {name = "Arguments", type = Argument, elements = self._arguments}, + {name = "Options", type = Option, elements = self._options}, + {name = "Commands", type = Command, elements = self._commands} + } + + local added_elements = {} + + for _, default_group in ipairs(default_groups) do + local type_groups = groups_by_type[default_group.type] + + for _, group in ipairs(type_groups) do + self:_add_group_help(blocks, added_elements, group.name .. ":", group) + end + + local default_label = default_group.name .. ":" + + if #type_groups > 0 then + default_label = "Other " .. default_label:gsub("^.", string.lower) + end + + self:_add_group_help(blocks, added_elements, default_label, default_group.elements) + end + + if self._epilog then + local epilog = self._epilog + + if help_max_width then + epilog = table.concat(autowrap(split_lines(epilog), help_max_width), "\n") + end + + table.insert(blocks, epilog) + end + + return table.concat(blocks, "\n\n") +end + +local function get_tip(context, wrong_name) + local context_pool = {} + local possible_name + local possible_names = {} + + for name in pairs(context) do + if type(name) == "string" then + for i = 1, #name do + possible_name = name:sub(1, i - 1) .. name:sub(i + 1) + + if not context_pool[possible_name] then + context_pool[possible_name] = {} + end + + table.insert(context_pool[possible_name], name) + end + end + end + + for i = 1, #wrong_name + 1 do + possible_name = wrong_name:sub(1, i - 1) .. wrong_name:sub(i + 1) + + if context[possible_name] then + possible_names[possible_name] = true + elseif context_pool[possible_name] then + for _, name in ipairs(context_pool[possible_name]) do + possible_names[name] = true + end + end + end + + local first = next(possible_names) + + if first then + if next(possible_names, first) then + local possible_names_arr = {} + + for name in pairs(possible_names) do + table.insert(possible_names_arr, "'" .. name .. "'") + end + + table.sort(possible_names_arr) + return "\nDid you mean one of these: " .. table.concat(possible_names_arr, " ") .. "?" + else + return "\nDid you mean '" .. first .. "'?" + end + else + return "" + end +end + +local ElementState = class({ + invocations = 0 +}) + +function ElementState:__call(state, element) + self.state = state + self.result = state.result + self.element = element + self.target = element._target or element:_get_default_target() + self.action, self.result[self.target] = element:_get_action() + return self +end + +function ElementState:error(fmt, ...) + self.state:error(fmt, ...) +end + +function ElementState:convert(argument, index) + local converter = self.element._convert + + if converter then + local ok, err + + if type(converter) == "function" then + ok, err = converter(argument) + elseif type(converter[index]) == "function" then + ok, err = converter[index](argument) + else + ok = converter[argument] + end + + if ok == nil then + self:error(err and "%s" or "malformed argument '%s'", err or argument) + end + + argument = ok + end + + return argument +end + +function ElementState:default(mode) + return self.element._defmode:find(mode) and self.element._default +end + +local function bound(noun, min, max, is_max) + local res = "" + + if min ~= max then + res = "at " .. (is_max and "most" or "least") .. " " + end + + local number = is_max and max or min + return res .. tostring(number) .. " " .. noun .. (number == 1 and "" or "s") +end + +function ElementState:set_name(alias) + self.name = ("%s '%s'"):format(alias and "option" or "argument", alias or self.element._name) +end + +function ElementState:invoke() + self.open = true + self.overwrite = false + + if self.invocations >= self.element._maxcount then + if self.element._overwrite then + self.overwrite = true + else + local num_times_repr = bound("time", self.element._mincount, self.element._maxcount, true) + self:error("%s must be used %s", self.name, num_times_repr) + end + else + self.invocations = self.invocations + 1 + end + + self.args = {} + + if self.element._maxargs <= 0 then + self:close() + end + + return self.open +end + +function ElementState:pass(argument) + argument = self:convert(argument, #self.args + 1) + table.insert(self.args, argument) + + if #self.args >= self.element._maxargs then + self:close() + end + + return self.open +end + +function ElementState:complete_invocation() + while #self.args < self.element._minargs do + self:pass(self.element._default) + end +end + +function ElementState:close() + if self.open then + self.open = false + + if #self.args < self.element._minargs then + if self:default("a") then + self:complete_invocation() + else + if #self.args == 0 then + if getmetatable(self.element) == Argument then + self:error("missing %s", self.name) + elseif self.element._maxargs == 1 then + self:error("%s requires an argument", self.name) + end + end + + self:error("%s requires %s", self.name, bound("argument", self.element._minargs, self.element._maxargs)) + end + end + + local args + + if self.element._maxargs == 0 then + args = self.args[1] + elseif self.element._maxargs == 1 then + if self.element._minargs == 0 and self.element._mincount ~= self.element._maxcount then + args = self.args + else + args = self.args[1] + end + else + args = self.args + end + + self.action(self.result, self.target, args, self.overwrite) + end +end + +local ParseState = class({ + result = {}, + options = {}, + arguments = {}, + argument_i = 1, + element_to_mutexes = {}, + mutex_to_element_state = {}, + command_actions = {} +}) + +function ParseState:__call(parser, error_handler) + self.parser = parser + self.error_handler = error_handler + self.charset = parser:_update_charset() + self:switch(parser) + return self +end + +function ParseState:error(fmt, ...) + self.error_handler(self.parser, fmt:format(...)) +end + +function ParseState:switch(parser) + self.parser = parser + + if parser._action then + table.insert(self.command_actions, {action = parser._action, name = parser._name}) + end + + for _, option in ipairs(parser._options) do + option = ElementState(self, option) + table.insert(self.options, option) + + for _, alias in ipairs(option.element._aliases) do + self.options[alias] = option + end + end + + for _, mutex in ipairs(parser._mutexes) do + for _, element in ipairs(mutex) do + if not self.element_to_mutexes[element] then + self.element_to_mutexes[element] = {} + end + + table.insert(self.element_to_mutexes[element], mutex) + end + end + + for _, argument in ipairs(parser._arguments) do + argument = ElementState(self, argument) + table.insert(self.arguments, argument) + argument:set_name() + argument:invoke() + end + + self.handle_options = parser._handle_options + self.argument = self.arguments[self.argument_i] + self.commands = parser._commands + + for _, command in ipairs(self.commands) do + for _, alias in ipairs(command._aliases) do + self.commands[alias] = command + end + end +end + +function ParseState:get_option(name) + local option = self.options[name] + + if not option then + self:error("unknown option '%s'%s", name, get_tip(self.options, name)) + else + return option + end +end + +function ParseState:get_command(name) + local command = self.commands[name] + + if not command then + if #self.commands > 0 then + self:error("unknown command '%s'%s", name, get_tip(self.commands, name)) + else + self:error("too many arguments") + end + else + return command + end +end + +function ParseState:check_mutexes(element_state) + if self.element_to_mutexes[element_state.element] then + for _, mutex in ipairs(self.element_to_mutexes[element_state.element]) do + local used_element_state = self.mutex_to_element_state[mutex] + + if used_element_state and used_element_state ~= element_state then + self:error("%s can not be used together with %s", element_state.name, used_element_state.name) + else + self.mutex_to_element_state[mutex] = element_state + end + end + end +end + +function ParseState:invoke(option, name) + self:close() + option:set_name(name) + self:check_mutexes(option, name) + + if option:invoke() then + self.option = option + end +end + +function ParseState:pass(arg) + if self.option then + if not self.option:pass(arg) then + self.option = nil + end + elseif self.argument then + self:check_mutexes(self.argument) + + if not self.argument:pass(arg) then + self.argument_i = self.argument_i + 1 + self.argument = self.arguments[self.argument_i] + end + else + local command = self:get_command(arg) + self.result[command._target or command._name] = true + + if self.parser._command_target then + self.result[self.parser._command_target] = command._name + end + + self:switch(command) + end +end + +function ParseState:close() + if self.option then + self.option:close() + self.option = nil + end +end + +function ParseState:finalize() + self:close() + + for i = self.argument_i, #self.arguments do + local argument = self.arguments[i] + if #argument.args == 0 and argument:default("u") then + argument:complete_invocation() + else + argument:close() + end + end + + if self.parser._require_command and #self.commands > 0 then + self:error("a command is required") + end + + for _, option in ipairs(self.options) do + option.name = option.name or ("option '%s'"):format(option.element._name) + + if option.invocations == 0 then + if option:default("u") then + option:invoke() + option:complete_invocation() + option:close() + end + end + + local mincount = option.element._mincount + + if option.invocations < mincount then + if option:default("a") then + while option.invocations < mincount do + option:invoke() + option:close() + end + elseif option.invocations == 0 then + self:error("missing %s", option.name) + else + self:error("%s must be used %s", option.name, bound("time", mincount, option.element._maxcount)) + end + end + end + + for i = #self.command_actions, 1, -1 do + self.command_actions[i].action(self.result, self.command_actions[i].name) + end +end + +function ParseState:parse(args) + for _, arg in ipairs(args) do + local plain = true + + if self.handle_options then + local first = arg:sub(1, 1) + + if self.charset[first] then + if #arg > 1 then + plain = false + + if arg:sub(2, 2) == first then + if #arg == 2 then + if self.options[arg] then + local option = self:get_option(arg) + self:invoke(option, arg) + else + self:close() + end + + self.handle_options = false + else + local equals = arg:find "=" + if equals then + local name = arg:sub(1, equals - 1) + local option = self:get_option(name) + + if option.element._maxargs <= 0 then + self:error("option '%s' does not take arguments", name) + end + + self:invoke(option, name) + self:pass(arg:sub(equals + 1)) + else + local option = self:get_option(arg) + self:invoke(option, arg) + end + end + else + for i = 2, #arg do + local name = first .. arg:sub(i, i) + local option = self:get_option(name) + self:invoke(option, name) + + if i ~= #arg and option.element._maxargs > 0 then + self:pass(arg:sub(i + 1)) + break + end + end + end + end + end + end + + if plain then + self:pass(arg) + end + end + + self:finalize() + return self.result +end + +function Parser:error(msg) + io.stderr:write(("%s\n\nError: %s\n"):format(self:get_usage(), msg)) + os.exit(1) +end + +-- Compatibility with strict.lua and other checkers: +local default_cmdline = rawget(_G, "arg") or {} + +function Parser:_parse(args, error_handler) + return ParseState(self, error_handler):parse(args or default_cmdline) +end + +function Parser:parse(args) + return self:_parse(args, self.error) +end + +local function xpcall_error_handler(err) + return tostring(err) .. "\noriginal " .. debug.traceback("", 2):sub(2) +end + +function Parser:pparse(args) + local parse_error + + local ok, result = xpcall(function() + return self:_parse(args, function(_, err) + parse_error = err + error(err, 0) + end) + end, xpcall_error_handler) + + if ok then + return true, result + elseif not parse_error then + error(result, 0) + else + return false, parse_error + end +end + +local argparse = {} + +argparse.version = "0.6.0" + +setmetatable(argparse, {__call = function(_, ...) + return Parser(default_cmdline[0]):add_help(true)(...) +end}) + +return argparse diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 4261e68..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -requests==2.24.0 \ No newline at end of file diff --git a/spec/assets/Readme.txt b/spec/assets/Readme.txt new file mode 100644 index 0000000..83664a5 --- /dev/null +++ b/spec/assets/Readme.txt @@ -0,0 +1,3 @@ + * Welcome to The Triangle programm. + *** It would help you to draw some simple, but beautiful triangles. +***** Hope you'll enjoy using it! \ No newline at end of file diff --git a/spec/assets/Rref.txt b/spec/assets/Rref.txt new file mode 100644 index 0000000..83664a5 --- /dev/null +++ b/spec/assets/Rref.txt @@ -0,0 +1,3 @@ + * Welcome to The Triangle programm. + *** It would help you to draw some simple, but beautiful triangles. +***** Hope you'll enjoy using it! \ No newline at end of file diff --git a/src/__init__.py b/spec/assets/list.db similarity index 100% rename from src/__init__.py rename to spec/assets/list.db diff --git a/spec/assets/ref.html b/spec/assets/ref.html new file mode 100644 index 0000000..b329f57 --- /dev/null +++ b/spec/assets/ref.html @@ -0,0 +1,239 @@ + + + + + + +LuaSocket: Index to reference manual + + + + + + + +
+
+
+ + + +
+LuaSocket +
Network support for the Lua language +
+

+home · +download · +installation · +introduction · +reference +

+
+
+
+ + + +

Reference

+ +
+DNS (in socket) +
+toip, +tohostname, +gethostname. +
+
+ + + +
+FTP +
+get, +put. +
+
+ + + +
+HTTP +
+request. +
+
+ + + +
+LTN12 +
+filter: +chain, +cycle. +
+
+pump: +all, +step. +
+
+sink: +chain, +error, +file, +null, +simplify, +table. +
+
+source: +cat, +chain, +empty, +error, +file, +simplify, +string. +
+
+ + + +
+MIME +
+high-level: +normalize, +decode, +encode, +stuff, +wrap. +
+
+low-level: +b64, +dot, +eol, +qp, +wrp, +qpwrp. +unb64, +unqp, +
+
+ + + +
+SMTP +
+message, +send. +
+
+ + + +
+Socket +
+_DEBUG, +dns, +gettime, +newtry, +protect, +select, +sink, +skip, +sleep, +source, +tcp, +try, +udp, +_VERSION. +
+
+ + + +
+TCP (in socket) +
+accept, +bind, +close, +connect, +getpeername, +getsockname, +getstats, +receive, +send, +setoption, +setstats, +settimeout, +shutdown. +
+
+ + + +
+UDP (in socket) +
+close, +getpeername, +getsockname, +receive, +receivefrom, +send, +sendto, +setpeername, +setsockname, +setoption, +settimeout. +
+
+ + + +
+URL +
+absolute, +build, +build_path, +escape, +parse, +parse_path, +unescape. +
+
+ + + +
+
+
+

+home · +download · +installation · +introduction · +reference +

+

+ +Last modified by Diego Nehab on
+Thu Apr 20 00:25:47 EDT 2006 + +

+
+
+ + + diff --git a/spec/assets/tmp_dir/and one more.md b/spec/assets/tmp_dir/and one more.md new file mode 100644 index 0000000..06d02d5 --- /dev/null +++ b/spec/assets/tmp_dir/and one more.md @@ -0,0 +1 @@ +Tasty content \ No newline at end of file diff --git a/spec/assets/tmp_dir/some file.md b/spec/assets/tmp_dir/some file.md new file mode 100644 index 0000000..a444519 --- /dev/null +++ b/spec/assets/tmp_dir/some file.md @@ -0,0 +1,7 @@ +# Header :D + +Some additional content, don't mind + +![Tux &>](https://d33wubrfki0l68.cloudfront.net/e7ed9fe4bafe46e275c807d63591f85f9ab246ba/e2d28/assets/images/tux.png) + +And even more content diff --git a/spec/netw_ops_spec.lua b/spec/netw_ops_spec.lua new file mode 100644 index 0000000..1072a96 --- /dev/null +++ b/spec/netw_ops_spec.lua @@ -0,0 +1,193 @@ +local NetwOps = require("src.netw_ops") +local Utils = require("spec.utils") + +local assets_dir = './spec/assets/' + +describe('netw_ops', function() + describe('download_to', function() + local function clean() + Utils.clean_assets({'reference.html', 'Readme.html', 'tux.png'}) + end + + before_each(clean) + after_each(clean) + + it('Saves file into output', function() + NetwOps.download_to('http://w3.impa.br/~diego/software/luasocket/reference.html', assets_dir) + + local f = io.open(assets_dir .. 'reference.html', 'r') + local reference = io.open(assets_dir .. 'ref.html', 'r') + + local content = f:read('a') + local ref_content = reference:read('a') + + assert.truthy(f) + assert.equal(ref_content, content) + + f:close() + reference:close() + end) + + it('Works with HTTPS protocol', function() + NetwOps.download_to('https://raw.githubusercontent.com/Dm1tr1y147/thetriangle/master/Readme.txt', + assets_dir) + + local f = io.open(assets_dir .. 'Readme.txt', 'r') + local reference = io.open(assets_dir .. 'Rref.txt', 'r') + + local content = f:read('a') + local ref_content = reference:read('a') + + assert.truthy(f) + assert.equal(ref_content, content) + + f:close() + reference:close() + end) + + it('Works with binary files', function() + NetwOps.download_to( + 'https://d33wubrfki0l68.cloudfront.net/e7ed9fe4bafe46e275c807d63591f85f9ab246ba/e2d28/assets/images/tux.png', + assets_dir) + + local f = io.open(assets_dir .. 'tux.png', "r") + + assert.truthy(f) + + f:close() + end) + end) + + describe('get_file_name', function() + it('Gets last element of path', function() + assert.equal('tux.png', NetwOps.get_file_name('https://test.com/tux.png')) + end) + + it("Doesn't return anything if no filename", function() + assert.equal('', NetwOps.get_file_name('https://test.com/')) + end) + end) + + describe('scp_wrap', function() + local function clean() + Utils.clean_assets({'downloaded.txt'}) + end + + before_each(clean) + after_each(clean) + + it('Downloads file from server', function() + local res = NetwOps.scp_wrap( + 'dm1sh@192.168.0.18:/mnt/hdd/Work/Development/Lua/md-offliner/spec/assets/Rref.txt', + assets_dir .. 'downloaded.txt') + + assert.is_not_nil(res) + + local orig_f = io.open(assets_dir .. 'Rref.txt') + local dest_f = io.open(assets_dir .. 'downloaded.txt') + + assert.is_not_nil(dest_f) + + local orig_content = orig_f:read('a') + local dest_content = dest_f:read('a') + + orig_f:close() + dest_f:close() + + assert.equal(orig_content, dest_content) + end) + end) + + describe('upload_dir', function() + local function clean() + Utils.clean_assets({'output_dir/'}) + end + + before_each(clean) + after_each(clean) + + it('Uploads all files in directory to server', function() + NetwOps.upload_dir(assets_dir .. 'tmp_dir/', 'dm1sh@192.168.0.18', + '/mnt/hdd/Work/Development/Lua/md-offliner/spec/assets/output_dir/') + + local ref_dir = Utils.list_dir(assets_dir .. 'tmp_dir') + local dest_dir = Utils.list_dir(assets_dir .. 'output_dir') + + assert.are.same(ref_dir, dest_dir) + end) + end) + + describe('download_config', function() + local function clean() + Utils.clean_assets({'tmp_dir/list.db'}) + end + + before_each(clean) + after_each(clean) + teardown(clean) + + it('Downloads database file from server', function() + NetwOps.download_db('dm1sh@192.168.0.18', '/mnt/hdd/Work/Development/Lua/md-offliner/spec/assets/', + assets_dir .. '/tmp_dir') + + local ref_file = io.open(assets_dir .. 'list.db', 'r') + local file = io.open(assets_dir .. 'tmp_dir/list.db', 'r') + + assert.is_not_nil(file) + + local ref_content = ref_file:read('a') + local content = file:read('a') + + assert.equal(ref_content, content) + + ref_file:close() + file:close() + end) + end) + + describe('insert_article', function() + local function clean() + Utils.clean_assets({'ref_list.db'}) + end + + setup(function() + Utils.copy_file(assets_dir .. 'list.db', assets_dir .. 'ref_list.db') + end) + + before_each(function() + Utils.copy_file(assets_dir .. 'ref_list.db', assets_dir .. 'list.db') + end) + + teardown(function() + Utils.copy_file(assets_dir .. 'ref_list.db', assets_dir .. 'list.db') + clean() + end) + + it('Adds article to file', function() + NetwOps.insert_article(assets_dir, 'Test_article') + + local ref_list = io.open(assets_dir .. 'ref_list.db', 'r') + local list = io.open(assets_dir .. 'list.db', 'r') + + assert.equal(ref_list:read('a') .. math.floor(os.time()) .. ' Test_article', list:read('a')) + + ref_list:close() + list:close() + end) + it('Updates if entry exists', function() + local list = io.open(assets_dir .. 'list.db', 'a') + list:write('5647546 Test_article\n') + list:close() + + NetwOps.insert_article(assets_dir, 'Test_article') + + local ref_list = io.open(assets_dir .. 'ref_list.db', 'r') + list = io.open(assets_dir .. 'list.db', 'r') + + assert.equal(ref_list:read('a') .. math.floor(os.time()) .. ' Test_article', list:read('a')) + + ref_list:close() + list:close() + end) + end) +end) diff --git a/spec/process_md_spec.lua b/spec/process_md_spec.lua new file mode 100644 index 0000000..279f5bc --- /dev/null +++ b/spec/process_md_spec.lua @@ -0,0 +1,103 @@ +local ProcessMD = require("src.process_md") +local Utils = require("spec.utils") + +local assets_dir = './spec/assets/' + +describe("process_md", function() + describe("get_file_content", function() + it("Gets content of file test.txt", function() + assert.equal( + "![Tux, the Linux mascot](http://test.ml/tux.png) and even ![](http://w3.impa.br/~diego/software/luasocket/luasocket.png)", + ProcessMD.get_file_content("test.txt")) + end) + it("Throws error on wrong filename", function() + assert.has_error(function() + ProcessMD.get_file_content("foo.bar") + end, "foo.bar: No such file or directory") + end) + end) + + describe("get_web_imgs_path", function() + it("Gets img path from markdown img string", function() + assert.are.same({'https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png'}, + ProcessMD.get_web_imgs_path( + '![alt text](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png)')) + end) + it("Doen't return local paths", function() + assert.are.same({}, ProcessMD.get_web_imgs_path('![alt text](logo)')) + end) + it("Gets img path from makrdown text", function() + assert.are.same({'https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png'}, + ProcessMD.get_web_imgs_path( + 'Tesing text, only simple ![img](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png) img passed')) + end) + it("Doesn't return links paths", function() + assert.are.same({}, ProcessMD.get_web_imgs_path( + 'My favorite search engine is [Duck Duck Go](https://duckduckgo.com).')) + end) + it("Works for multiple images links", function() + assert.are.same({'http://test.ml/tux.png', 'http://w3.impa.br/~diego/software/luasocket/luasocket.png'}, + ProcessMD.get_web_imgs_path( + '![Tux, the Linux mascot](http://test.ml/tux.png) and even ![](http://w3.impa.br/~diego/software/luasocket/luasocket.png)')) + end) + end) + + describe("replace_paths", function() + it("Replaces original paths with passed", function() + assert.equal('![Tux, the Linux mascot](./tux.png) and even ![](./luasocket.png)', ProcessMD.replace_paths( + '![Tux, the Linux mascot](http://test.ml/tux.png) and even ![](http://w3.impa.br/~diego/software/luasocket/luasocket.png)', + { + ["http://test.ml/tux.png"] = "./tux.png", + ["http://w3.impa.br/~diego/software/luasocket/luasocket.png"] = "./luasocket.png" + })) + end) + end) + + describe("get_header", function() + it('Gets header of md file', function() + local f = io.open(assets_dir .. 'tmp_dir/some file.md', 'r') + + local content = f:read('a') + + local header + content, header = ProcessMD.get_header(content, assets_dir .. 'tmp_dir/some file.md', true) + f:close() + + assert.equal('Header_:D', header) + end) + + it("Returns empty line if no header", function() + local f = io.open(assets_dir .. 'tmp_dir/and one more.md', 'r') + + local content = f:read('a') + + local header + _, header = ProcessMD.get_header(content, assets_dir .. 'tmp_dir/and one more.md', true) + f:close() + + assert.equal('and_one_more', header) + end) + end) + + describe("save_document", function() + local function clean() + Utils.clean_assets({'Article_name.md'}) + end + + setup(clean) + teardown(clean) + + it('Saves markdown document into file', function() + local temp_content = 'Test document' + ProcessMD.save_document(assets_dir, 'Article_name', temp_content) + + local f = io.open(assets_dir .. 'Article_name.md', 'r') + assert.is_not_nil(f) + + local f_content = f:read('a') + assert.equal(temp_content, f_content) + + f:close() + end) + end) +end) diff --git a/spec/utils.lua b/spec/utils.lua new file mode 100644 index 0000000..1713d94 --- /dev/null +++ b/spec/utils.lua @@ -0,0 +1,34 @@ +local utils = {} + +local assets_dir = './spec/assets/' + +function utils.list_dir(path) + local files = {} + local pfile = io.popen('ls -a "' .. path .. '"') + for file in pfile:lines() do + table.insert(files, file) + end + + return files +end + +function utils.clean_assets(files) + local am = 0 + for _, file in ipairs(files) do + os.execute('rm -rf ' .. assets_dir .. file) + am = am + 1 + end + + return am +end + +function utils.copy_file(from_path, dest_path) + local from, dest = io.open(from_path, 'r'), io.open(dest_path, 'w') + + dest:write(from:read('a')) + + from:close() + dest:close() +end + +return utils diff --git a/src/app.py b/src/app.py deleted file mode 100644 index 7d13e5d..0000000 --- a/src/app.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -''' Main module ''' - -import os -import sys -import shutil - -from get_args import parse_args -from article_process import get_article_title, get_paths, replace_article_paths, rm_first_line -from netw_ops import download_and_save, upload_to_server, add_to_list_on_server - - -def main(argv): - '''Main function''' - - input_file = '' - server_cred = '' - output_directory = '' - - res = parse_args(argv) - - if len(res) == 2: - input_file, output_directory = res # pylint: disable=unbalanced-tuple-unpacking - elif len(res) == 3: - input_file, server_cred, output_directory = res - - try: - ifile = open(input_file, "r") - except IOError as ex: - print("Couldn't open input file") - print(ex) - sys.exit(2) - - text = ifile.read() - - if output_directory[-1] != '/': - output_directory += '/' - - article_filename = get_article_title(text) - - if not article_filename: - article_filename = input_file.split('/')[-1] - else: - text = rm_first_line(text) - - article_folder = article_filename.split('.')[0] + '/' - - if not os.path.exists(article_folder): - os.makedirs(article_folder) - - paths = get_paths(text) - - res_paths = [] - - for url in paths: - try: - res_paths.append(download_and_save(url, article_folder)) - except Exception as ex: - paths.remove(url) - print("Couldn't process image:", ex, '\nurl:', url) - raise "Couldn't process image" - - text = replace_article_paths(text, paths, res_paths) - - open(article_folder + article_filename, "w").write(text) - - if server_cred: - upload_to_server(server_cred, article_folder, output_directory) - - shutil.rmtree(article_folder) - - add_to_list_on_server( - server_cred, article_folder, output_directory) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/src/arg_proc.lua b/src/arg_proc.lua new file mode 100644 index 0000000..60241d9 --- /dev/null +++ b/src/arg_proc.lua @@ -0,0 +1,67 @@ +local args = {} + +function args.read_config(argv, path, print_err) + local function file_exists(file_path) + local f = io.open(file_path, "rb") + + if f then + f:close() + end + return f ~= nil + end + + if file_exists(path) then + for line in io.lines(path) do + local key, value = line:match("([^=]+)=([^=]+)") + if not (key and value) then + print_err:error("Wrong config file syntax") + end + + argv[key] = value + end + end + + return argv +end + +function args.parse() + local Argparse = require("lib.argparse") + + local parser = Argparse("md-parser", + "Command line utility for saving markdown document online assets locally and upload it to server with scp") + + parser:argument("input", "Input file.") + + parser:option("-o --output", "Output directory.", "./") + parser:option("-s --server", "Username and hostname in username@hostname notation.") + parser:option("-c --config", + "Configuration file like\n\t\t\tpath= Output path on server\n\t\t\thost= Server username and hostname in username@hostname notation") + + parser:flag("-u --upload", "If sould upload to server") + + local arguments = parser:parse() + + if not arguments.input:find('[^%.]+%.md$') then + parser:error("You can't process non-markdown file") + end + + if ((arguments.output or arguments.server) and arguments.config) then + parser:error("You can't use both command line parameters and configuration file") + end + + if (arguments.config) then + arguments = args.read_config(arguments, arguments.config, parser) + end + + if arguments.upload and not (arguments.output and arguments.server) then + parser:error("You should specify output directory and server credentials for upload") + end + + if arguments.output[arguments.output:len()] ~= '/' then + arguments.output = arguments.output .. '/' + end + + return arguments +end + +return args diff --git a/src/article_process.py b/src/article_process.py deleted file mode 100644 index 867e8c0..0000000 --- a/src/article_process.py +++ /dev/null @@ -1,52 +0,0 @@ -'''Provides some article content operations''' - -import re - - -def get_paths(string): - '''Gets images paths in article''' - - img_reg = re.compile(r'!\[.*?\]\(.*?\)') - path_reg = re.compile(r'(?<=\()http[s]{0,1}.*?(?=\))') - - imgs = img_reg.findall(string) - - paths = [] - - for img in imgs: - res = path_reg.search(img) - if res: - paths.append(res.group()) - - return paths - - -def rm_first_line(string): - '''Removes first line from string''' - - return string[string.find('\n') + 1:] - - -def get_article_title(string): - '''Gets article title''' - - header = '' - - if string[0] == '#': - header = string.split('\n')[0] - while header[0] == '#' or header[0] == ' ': - header = header[1:] - header += '.md' - header = header.replace(' ', '_') - - return header - - -def replace_article_paths(string, orig_paths, res_paths): - '''Replaces all web links with downloaded ones''' - - for i, val in enumerate(res_paths): - print(val[2:]) - string = string.replace(orig_paths[i], '/articles/' + val[2:]) - - return string diff --git a/src/get_args.py b/src/get_args.py deleted file mode 100644 index c7860d4..0000000 --- a/src/get_args.py +++ /dev/null @@ -1,78 +0,0 @@ -''' Module to get arguments passed into Main module''' - -import getopt -import sys - - -def read_cfg_file(path): - '''Reads config file''' - - cfg = open(path, 'r') - buff = cfg.read() - for line in buff.split('\n'): - if line.split('=')[0] == 'output': - output_directory = line.split('=')[1] - elif line.split('=')[0] == 'host': - server_cred = line.split('=')[1] - if not (output_directory and server_cred): - print("No config file provided") - sys.exit(2) - - return output_directory, server_cred - - -def usage(): - '''Prints usage instructions''' - - print(''''Usage: ./article_uploader -i -o - or ./article_uploader -u -o -s - or ./article_uploader -u -c - with configuration file such as - path= - host=''') - - -def parse_args(argv): - '''Parses arguments provided by user''' - - input_file = '' - output_directory = '' - upload_to_server = False - server_cred = '' - cfg_path = '' - - try: - opts = getopt.getopt(argv, 'hi:o:us:c:')[0] - except getopt.GetoptError: - usage() - sys.exit(2) - - for opt, arg in opts: - if opt == '-h': - usage() - sys.exit() - elif opt == '-i': - input_file = arg - elif opt == '-o': - output_directory = arg - elif opt == '-u': - upload_to_server = True - elif opt == '-s': - server_cred = arg - elif opt == '-c': - cfg_path = arg - - if not (input_file and (output_directory or upload_to_server) or cfg_path): - usage() - sys.exit(2) - - if upload_to_server and not (server_cred and output_directory) and cfg_path: - output_directory, server_cred = read_cfg_file(cfg_path) - else: - usage() - sys.exit(2) - - if server_cred and output_directory: - return input_file, server_cred, output_directory - else: - return input_file, output_directory diff --git a/src/main.lua b/src/main.lua new file mode 100644 index 0000000..ff2fc51 --- /dev/null +++ b/src/main.lua @@ -0,0 +1,85 @@ +local Arguments = require("src.arg_proc") +local ProcessMD = require("src.process_md") +local NetwOps = require("src.netw_ops") + +local function get_document_info(filename, upload) + local status, content = pcall(ProcessMD.get_file_content, filename) + if not status then + print('Error: ' .. content) + os.exit(1) + end + + local document_name + content, document_name = ProcessMD.get_header(content, upload) + + local urls = ProcessMD.get_web_imgs_path(content) + + return document_name, content, urls +end + +local function compose_document_destination_folder(upload, path, document_name) + if upload or not path then + path = './' + end + path = path .. document_name + + return path +end + +local function download_netw_assets(urls, dest) + local dict = {} + for _, path in ipairs(urls) do + local status, res_file = pcall(NetwOps.download_to, path, dest) + if not status then + print('Error: ' .. res_file) + os.exit(1) + end + + dict[path] = res_file + end + return dict +end + +local function upload_to_server(local_article_directory, server_cred, server_path, document_name) + local status, err = pcall(NetwOps.upload_dir, local_article_directory, server_cred, server_path .. document_name) + if not status then + print('Error: ' .. err) + os.exit(1) + end + + local status, err = pcall(NetwOps.download_db, server_cred, server_path, local_article_directory) + if not status then + print('Error: ' .. err) + os.exit(1) + end + + NetwOps.insert_article(local_article_directory, document_name) + + local status, err = pcall(NetwOps.upload_db, server_cred, server_path, local_article_directory) + if not status then + print('Error: ' .. err) + os.exit(1) + end + + os.execute('rm -rf "' .. local_article_directory .. '"') +end + +local params = Arguments.parse() + +local document_name, content, urls = get_document_info(params.input, params.upload) + +local local_article_directory = compose_document_destination_folder(params.upload, params.output, document_name) + +local dict = download_netw_assets(urls, local_article_directory) + +content = ProcessMD.replace_paths(content, dict) + +local status, err = pcall(ProcessMD.save_document, local_article_directory, document_name, content) +if not status then + print('Error: ' .. err) + os.exit(1) +end + +if params.upload then + upload_to_server(local_article_directory, params.server, params.output, document_name) +end diff --git a/src/netw_ops.lua b/src/netw_ops.lua new file mode 100644 index 0000000..1c8a358 --- /dev/null +++ b/src/netw_ops.lua @@ -0,0 +1,107 @@ +local netw_ops = {} +local https = require("ssl.https") +local http = require("socket.http") + +function netw_ops.download_to(url, output) + local res = os.execute('mkdir -p "' .. output .. '"') + + if not res then + error("Couldn't create output directory " .. output) + end + + local filename = netw_ops.get_file_name(url) + if not filename then + error("Wrong url, if it is really an image, try to download it on your own: " .. url) + end + + local response, code + local _ + if url:match("^https://") then + response, code, _, _ = https.request(url) + else + response, code, _, _ = http.request(url) + end + + local ofile, err + if code == 200 then + ofile, err = io.open(output .. '/' .. filename, "wb") + else + error("Error downloading file. Server returned " .. code .. ' code') + end + + if not ofile then + error(err, 2) + end + + ofile:write(response) + ofile:close() + + return './' .. filename +end + +function netw_ops.get_file_name(str) + return str:match("/([^/]*)$") +end + +function netw_ops.scp_wrap(from_path, to_path) + return os.execute('scp -prq ' .. from_path .. ' ' .. to_path) +end + +function netw_ops.upload_dir(local_path, server_cred, server_path) + os.execute('ssh ' .. server_cred .. ' "rm -rf ' .. server_path .. '"') + local res = netw_ops.scp_wrap(local_path .. '/', server_cred .. ':' .. server_path .. '/') + + if res then + return res + else + error("Could't upload directory to server " .. server_cred) + end +end + +function netw_ops.download_db(server_cred, server_path, local_path) + local res = netw_ops.scp_wrap(server_cred .. ':' .. server_path .. 'list.db', local_path) + + if res then + return res + else + error("Could't download server" .. server_cred .. " articles database") + end +end + +function netw_ops.insert_article(local_path, document_name) + local lines = '' + local not_in = true + local new_line = math.floor(os.time()) .. ' ' .. document_name + local file_name = local_path .. '/list.db' + local file = io.open(file_name, 'r') + + for line in file:lines() do + if line:find(document_name) then + line = new_line + not_in = false + end + lines = lines .. (lines:len() > 0 and '\n' or '') .. line + end + + if not_in then + lines = lines .. (lines:len() > 0 and '\n' or '') .. new_line + end + + file:close() + file = io.open(file_name, 'w') + file:write(lines) + + file:close() +end + +function netw_ops.upload_db(server_cred, server_path, local_path) + local res = netw_ops.scp_wrap(local_path .. '/' .. 'list.db', server_cred .. ':' .. server_path) + + if res then + return res + else + error("Could't upload server " .. server_cred .. " articles database") + end +end + +return netw_ops diff --git a/src/netw_ops.py b/src/netw_ops.py deleted file mode 100644 index 3fe0a56..0000000 --- a/src/netw_ops.py +++ /dev/null @@ -1,71 +0,0 @@ -'''Provides network related operations''' - -import subprocess -import os -import time -import requests - - -def download_and_save(url, out_dir): - '''Downloads file from url and saves it into out_dir''' - - file = requests.get(url) - - out_path = './' + out_dir + url.split('/')[-1] - - try: - open(out_path, 'wb').write(file.content) - return out_path - except IOError as ex: - print(ex) - raise 'Couldn\'t open file for write' - - -def scp_wrap(recursively, from_path, to_path): - '''Downloads/uploads files from/to server using scp''' - - if recursively: - proc = subprocess.Popen(["scp", "-r", from_path, to_path]) - else: - proc = subprocess.Popen(["scp", from_path, to_path]) - - sts = os.waitpid(proc.pid, 0) # pylint: disable=unused-variable - - -def upload_to_server(server_cred, local_path, server_path): - '''Uploads selected folder to server using scp''' - - scp_wrap(True, local_path, server_cred + ':' + server_path) - - -def add_to_list_on_server(server_cred, local_path, server_path): - '''Reads list of articles on server and add new article to it''' - - article_name = local_path[:-1] - - scp_wrap(False, server_cred + ':' + server_path + 'list.db', './') - - articles_list_file = open('list.db', 'r+') - articles_list = articles_list_file.read() - - articles_list_s = articles_list.split('\n') - - flag = True - for i, val in enumerate(articles_list_s): - if article_name in val: - line_s = val.split(' ') - line_s[0] = str(int(time.time())) - articles_list_s[i] = ' '.join(line_s) - - flag = False - - if flag: - articles_list_s.append(str(int(time.time())) + ' ' + article_name) - - articles_list = '\n'.join(filter(None, articles_list_s)) - - articles_list_file.seek(0) - articles_list_file.write(articles_list) - articles_list_file.close() - - scp_wrap(False, 'list.db', server_cred + ':' + server_path) diff --git a/src/process_md.lua b/src/process_md.lua new file mode 100644 index 0000000..f9b8fb1 --- /dev/null +++ b/src/process_md.lua @@ -0,0 +1,65 @@ +local process_md = {} + +function process_md.get_file_content(path) + local f, err = io.open(path, "r+") + assert(f, err) + + local content = f:read("a") + + f:close() + return content +end + +function process_md.get_web_imgs_path(str) + local urls = {} + + for link in string.gmatch(str, '!%[[^%]]*%]%((http[^%)]+)%)') do + table.insert(urls, link) + end + + return urls +end + +function process_md.replace_paths(str, dict) + for key, value in pairs(dict) do + str = string.gsub(str, key, value) + end + + return str +end + +function process_md.get_header(content, filename, upload) + local first_line = content:match('([^\n]-)\n') + + local header + if first_line then + header = first_line:match("# (.*)") + end + + if not header then + return content, filename:match('/([^/]-).md$'):gsub(' ', '_') + else + if upload then + content = process_md.remove_first_line(content) + end + return content, header:gsub(' ', '_') + end +end + +function process_md.remove_first_line(content) + return content:match('[^\n]-\n+(.*)') +end + +function process_md.save_document(dest, header, content) + local filename = dest .. '/' .. header .. '.md' + + local output_file = io.open(filename, "w") + if not output_file then + error("Couldn't open output file: " .. filename) + end + + output_file:write(content) + output_file:close() +end + +return process_md