Rewrote with lua, added tests

This commit is contained in:
Dm1tr1y147 2020-07-23 19:18:27 +05:00
parent d4c48016ee
commit 4eabc25d87
23 changed files with 2499 additions and 285 deletions

8
.gitignore vendored
View File

@ -1,4 +1,4 @@
venv/
tmp_*
.vscode/
__pycache__/
lua_install
luacov.stats.out
test*
.vscode/

1
.luacheckrc Normal file
View File

@ -0,0 +1 @@
ignore = {"631"}

27
.travis.yml Normal file
View File

@ -0,0 +1,27 @@
language: python
sudo: false
env:
- LUA="lua=5.1"
- LUA="lua=5.2"
- LUA="lua=5.3"
- LUA="lua=5.4"
before_install:
- pip install hererocks
- hererocks lua_install -r^ --$LUA
- export PATH=$PATH:$PWD/lua_install/bin
install:
- luarocks make
- luarocks install luacheck
- luarocks install busted
- luarocks install luacov
- luarocks install luacov-coveralls
script:
- luacheck --std max+busted src spec
- busted --verbose --coverage
after_success:
- luacov-coveralls -e $TRAVIS_BUILD_DIR/lua_install

View File

@ -1,3 +1,34 @@
# Article uploader
# Markdown article online assets downloader
Tool for uploading articles to the server
This tool helps to download all images in markdown document and put them into one folder with changed paths
## Usage:
```
md-parser [-o <output>] [-s <server>] [-c <config>] [-u] [-h]
<input>
Command line utility for saving markdown document online assets locally and upload it to server with scp
Arguments:
input Input file.
Options:
-o <output>, Output directory. (default: ./)
--output <output>
-s <server>, Username and hostname in username@hostname notation.
--server <server>
-c <config>, Configuration file like
--config <config> path= Output path on server
host= Server username and hostname in username@hostname notation
-u, --upload If sould upload to server
-h, --help Show this help message and exit.
```
It also has integrated functional to upload this article over ssh. As for MIT licence you can freely fork it and modify code for your own usage cases.
## Example:
```
lua src/main.lua spec/assets/tmp_dir/some\ file.md -u -s dm1sh@localhost -o /tmp/art
```

1527
lib/argparse.lua Normal file

File diff suppressed because it is too large Load Diff

View File

@ -1 +0,0 @@
requests==2.24.0

3
spec/assets/Readme.txt Normal file
View File

@ -0,0 +1,3 @@
* Welcome to The Triangle programm.
*** It would help you to draw some simple, but beautiful triangles.
***** Hope you'll enjoy using it!

3
spec/assets/Rref.txt Normal file
View File

@ -0,0 +1,3 @@
* Welcome to The Triangle programm.
*** It would help you to draw some simple, but beautiful triangles.
***** Hope you'll enjoy using it!

239
spec/assets/ref.html Normal file
View File

@ -0,0 +1,239 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
"http://www.w3.org/TR/html4/strict.dtd">
<html>
<head>
<meta name="description" content="LuaSocket: Index to reference manual">
<meta name="keywords" content="Lua, LuaSocket, Index, Manual, Network, Library,
Support, Manual">
<title>LuaSocket: Index to reference manual</title>
<link rel="stylesheet" href="reference.css" type="text/css">
</head>
<body>
<!-- header ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<div class=header>
<hr>
<center>
<table summary="LuaSocket logo">
<tr><td align=center><a href="http://www.lua.org">
<img width=128 height=128 border=0 alt="LuaSocket" src="luasocket.png">
</a></td></tr>
<tr><td align=center valign=top>Network support for the Lua language
</td></tr>
</table>
<p class=bar>
<a href="home.html">home</a> &middot;
<a href="home.html#download">download</a> &middot;
<a href="installation.html">installation</a> &middot;
<a href="introduction.html">introduction</a> &middot;
<a href="reference.html">reference</a>
</p>
</center>
<hr>
</div>
<!-- reference +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<h2>Reference</h2>
<blockquote>
<a href="dns.html">DNS (in socket)</a>
<blockquote>
<a href="dns.html#toip">toip</a>,
<a href="dns.html#tohostname">tohostname</a>,
<a href="dns.html#gethostname">gethostname</a>.
</blockquote>
</blockquote>
<!-- ftp ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<blockquote>
<a href="ftp.html">FTP</a>
<blockquote>
<a href="ftp.html#get">get</a>,
<a href="ftp.html#put">put</a>.
</blockquote>
</blockquote>
<!-- http +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<blockquote>
<a href="http.html">HTTP</a>
<blockquote>
<a href="http.html#request">request</a>.
</blockquote>
</blockquote>
<!-- ltn12 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<blockquote>
<a href="ltn12.html">LTN12</a>
<blockquote>
<a href="ltn12.html#filter">filter</a>:
<a href="ltn12.html#filter.chain">chain</a>,
<a href="ltn12.html#filter.cycle">cycle</a>.
</blockquote>
<blockquote>
<a href="ltn12.html#pump">pump</a>:
<a href="ltn12.html#pump.all">all</a>,
<a href="ltn12.html#pump.step">step</a>.
</blockquote>
<blockquote>
<a href="ltn12.html#sink">sink</a>:
<a href="ltn12.html#sink.chain">chain</a>,
<a href="ltn12.html#sink.error">error</a>,
<a href="ltn12.html#sink.file">file</a>,
<a href="ltn12.html#sink.null">null</a>,
<a href="ltn12.html#sink.simplify">simplify</a>,
<a href="ltn12.html#sink.table">table</a>.
</blockquote>
<blockquote>
<a href="ltn12.html#source">source</a>:
<a href="ltn12.html#source.cat">cat</a>,
<a href="ltn12.html#source.chain">chain</a>,
<a href="ltn12.html#source.empty">empty</a>,
<a href="ltn12.html#source.error">error</a>,
<a href="ltn12.html#source.file">file</a>,
<a href="ltn12.html#source.simplify">simplify</a>,
<a href="ltn12.html#source.string">string</a>.
</blockquote>
</blockquote>
<!-- mime +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<blockquote>
<a href="mime.html">MIME</a>
<blockquote>
<a href="mime.html#high">high-level</a>:
<a href="mime.html#normalize">normalize</a>,
<a href="mime.html#decode">decode</a>,
<a href="mime.html#encode">encode</a>,
<a href="mime.html#stuff">stuff</a>,
<a href="mime.html#wrap">wrap</a>.
</blockquote>
<blockquote>
<a href="mime.html#low">low-level</a>:
<a href="mime.html#b64">b64</a>,
<a href="mime.html#dot">dot</a>,
<a href="mime.html#eol">eol</a>,
<a href="mime.html#qp">qp</a>,
<a href="mime.html#wrp">wrp</a>,
<a href="mime.html#qpwrp">qpwrp</a>.
<a href="mime.html#unb64">unb64</a>,
<a href="mime.html#unqp">unqp</a>,
</blockquote>
</blockquote>
<!-- smtp +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<blockquote>
<a href="smtp.html">SMTP</a>
<blockquote>
<a href="smtp.html#message">message</a>,
<a href="smtp.html#send">send</a>.
</blockquote>
</blockquote>
<!-- socket +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<blockquote>
<a href="socket.html">Socket</a>
<blockquote>
<a href="socket.html#debug">_DEBUG</a>,
<a href="dns.html#dns">dns</a>,
<a href="socket.html#gettime">gettime</a>,
<a href="socket.html#newtry">newtry</a>,
<a href="socket.html#protect">protect</a>,
<a href="socket.html#select">select</a>,
<a href="socket.html#sink">sink</a>,
<a href="socket.html#skip">skip</a>,
<a href="socket.html#sleep">sleep</a>,
<a href="socket.html#source">source</a>,
<a href="tcp.html#tcp">tcp</a>,
<a href="socket.html#try">try</a>,
<a href="udp.html#udp">udp</a>,
<a href="socket.html#version">_VERSION</a>.
</blockquote>
</blockquote>
<!-- tcp +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<blockquote>
<a href="tcp.html">TCP (in socket)</a>
<blockquote>
<a href="tcp.html#accept">accept</a>,
<a href="tcp.html#bind">bind</a>,
<a href="tcp.html#close">close</a>,
<a href="tcp.html#connect">connect</a>,
<a href="tcp.html#getpeername">getpeername</a>,
<a href="tcp.html#getsockname">getsockname</a>,
<a href="tcp.html#getstats">getstats</a>,
<a href="tcp.html#receive">receive</a>,
<a href="tcp.html#send">send</a>,
<a href="tcp.html#setoption">setoption</a>,
<a href="tcp.html#setstats">setstats</a>,
<a href="tcp.html#settimeout">settimeout</a>,
<a href="tcp.html#shutdown">shutdown</a>.
</blockquote>
</blockquote>
<!-- udp +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<blockquote>
<a href="udp.html">UDP (in socket)</a>
<blockquote>
<a href="udp.html#close">close</a>,
<a href="udp.html#getpeername">getpeername</a>,
<a href="udp.html#getsockname">getsockname</a>,
<a href="udp.html#receive">receive</a>,
<a href="udp.html#receivefrom">receivefrom</a>,
<a href="udp.html#send">send</a>,
<a href="udp.html#sendto">sendto</a>,
<a href="udp.html#setpeername">setpeername</a>,
<a href="udp.html#setsockname">setsockname</a>,
<a href="udp.html#setoption">setoption</a>,
<a href="udp.html#settimeout">settimeout</a>.
</blockquote>
</blockquote>
<!-- url ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<blockquote>
<a href="url.html">URL</a>
<blockquote>
<a href="url.html#absolute">absolute</a>,
<a href="url.html#build">build</a>,
<a href="url.html#build_path">build_path</a>,
<a href="url.html#escape">escape</a>,
<a href="url.html#parse">parse</a>,
<a href="url.html#parse_path">parse_path</a>,
<a href="url.html#unescape">unescape</a>.
</blockquote>
</blockquote>
<!-- footer ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -->
<div class=footer>
<hr>
<center>
<p class=bar>
<a href="home.html">home</a> &middot;
<a href="home.html#down">download</a> &middot;
<a href="installation.html">installation</a> &middot;
<a href="introduction.html">introduction</a> &middot;
<a href="reference.html">reference</a>
</p>
<p>
<small>
Last modified by Diego Nehab on <br>
Thu Apr 20 00:25:47 EDT 2006
</small>
</p>
</center>
</div>
</body>
</html>

View File

@ -0,0 +1 @@
Tasty content

View File

@ -0,0 +1,7 @@
# Header :D
Some additional content, don't mind
![Tux &>](https://d33wubrfki0l68.cloudfront.net/e7ed9fe4bafe46e275c807d63591f85f9ab246ba/e2d28/assets/images/tux.png)
And even more content

193
spec/netw_ops_spec.lua Normal file
View File

@ -0,0 +1,193 @@
local NetwOps = require("src.netw_ops")
local Utils = require("spec.utils")
local assets_dir = './spec/assets/'
describe('netw_ops', function()
describe('download_to', function()
local function clean()
Utils.clean_assets({'reference.html', 'Readme.html', 'tux.png'})
end
before_each(clean)
after_each(clean)
it('Saves file into output', function()
NetwOps.download_to('http://w3.impa.br/~diego/software/luasocket/reference.html', assets_dir)
local f = io.open(assets_dir .. 'reference.html', 'r')
local reference = io.open(assets_dir .. 'ref.html', 'r')
local content = f:read('a')
local ref_content = reference:read('a')
assert.truthy(f)
assert.equal(ref_content, content)
f:close()
reference:close()
end)
it('Works with HTTPS protocol', function()
NetwOps.download_to('https://raw.githubusercontent.com/Dm1tr1y147/thetriangle/master/Readme.txt',
assets_dir)
local f = io.open(assets_dir .. 'Readme.txt', 'r')
local reference = io.open(assets_dir .. 'Rref.txt', 'r')
local content = f:read('a')
local ref_content = reference:read('a')
assert.truthy(f)
assert.equal(ref_content, content)
f:close()
reference:close()
end)
it('Works with binary files', function()
NetwOps.download_to(
'https://d33wubrfki0l68.cloudfront.net/e7ed9fe4bafe46e275c807d63591f85f9ab246ba/e2d28/assets/images/tux.png',
assets_dir)
local f = io.open(assets_dir .. 'tux.png', "r")
assert.truthy(f)
f:close()
end)
end)
describe('get_file_name', function()
it('Gets last element of path', function()
assert.equal('tux.png', NetwOps.get_file_name('https://test.com/tux.png'))
end)
it("Doesn't return anything if no filename", function()
assert.equal('', NetwOps.get_file_name('https://test.com/'))
end)
end)
describe('scp_wrap', function()
local function clean()
Utils.clean_assets({'downloaded.txt'})
end
before_each(clean)
after_each(clean)
it('Downloads file from server', function()
local res = NetwOps.scp_wrap(
'dm1sh@192.168.0.18:/mnt/hdd/Work/Development/Lua/md-offliner/spec/assets/Rref.txt',
assets_dir .. 'downloaded.txt')
assert.is_not_nil(res)
local orig_f = io.open(assets_dir .. 'Rref.txt')
local dest_f = io.open(assets_dir .. 'downloaded.txt')
assert.is_not_nil(dest_f)
local orig_content = orig_f:read('a')
local dest_content = dest_f:read('a')
orig_f:close()
dest_f:close()
assert.equal(orig_content, dest_content)
end)
end)
describe('upload_dir', function()
local function clean()
Utils.clean_assets({'output_dir/'})
end
before_each(clean)
after_each(clean)
it('Uploads all files in directory to server', function()
NetwOps.upload_dir(assets_dir .. 'tmp_dir/', 'dm1sh@192.168.0.18',
'/mnt/hdd/Work/Development/Lua/md-offliner/spec/assets/output_dir/')
local ref_dir = Utils.list_dir(assets_dir .. 'tmp_dir')
local dest_dir = Utils.list_dir(assets_dir .. 'output_dir')
assert.are.same(ref_dir, dest_dir)
end)
end)
describe('download_config', function()
local function clean()
Utils.clean_assets({'tmp_dir/list.db'})
end
before_each(clean)
after_each(clean)
teardown(clean)
it('Downloads database file from server', function()
NetwOps.download_db('dm1sh@192.168.0.18', '/mnt/hdd/Work/Development/Lua/md-offliner/spec/assets/',
assets_dir .. '/tmp_dir')
local ref_file = io.open(assets_dir .. 'list.db', 'r')
local file = io.open(assets_dir .. 'tmp_dir/list.db', 'r')
assert.is_not_nil(file)
local ref_content = ref_file:read('a')
local content = file:read('a')
assert.equal(ref_content, content)
ref_file:close()
file:close()
end)
end)
describe('insert_article', function()
local function clean()
Utils.clean_assets({'ref_list.db'})
end
setup(function()
Utils.copy_file(assets_dir .. 'list.db', assets_dir .. 'ref_list.db')
end)
before_each(function()
Utils.copy_file(assets_dir .. 'ref_list.db', assets_dir .. 'list.db')
end)
teardown(function()
Utils.copy_file(assets_dir .. 'ref_list.db', assets_dir .. 'list.db')
clean()
end)
it('Adds article to file', function()
NetwOps.insert_article(assets_dir, 'Test_article')
local ref_list = io.open(assets_dir .. 'ref_list.db', 'r')
local list = io.open(assets_dir .. 'list.db', 'r')
assert.equal(ref_list:read('a') .. math.floor(os.time()) .. ' Test_article', list:read('a'))
ref_list:close()
list:close()
end)
it('Updates if entry exists', function()
local list = io.open(assets_dir .. 'list.db', 'a')
list:write('5647546 Test_article\n')
list:close()
NetwOps.insert_article(assets_dir, 'Test_article')
local ref_list = io.open(assets_dir .. 'ref_list.db', 'r')
list = io.open(assets_dir .. 'list.db', 'r')
assert.equal(ref_list:read('a') .. math.floor(os.time()) .. ' Test_article', list:read('a'))
ref_list:close()
list:close()
end)
end)
end)

103
spec/process_md_spec.lua Normal file
View File

@ -0,0 +1,103 @@
local ProcessMD = require("src.process_md")
local Utils = require("spec.utils")
local assets_dir = './spec/assets/'
describe("process_md", function()
describe("get_file_content", function()
it("Gets content of file test.txt", function()
assert.equal(
"![Tux, the Linux mascot](http://test.ml/tux.png) and even ![](http://w3.impa.br/~diego/software/luasocket/luasocket.png)",
ProcessMD.get_file_content("test.txt"))
end)
it("Throws error on wrong filename", function()
assert.has_error(function()
ProcessMD.get_file_content("foo.bar")
end, "foo.bar: No such file or directory")
end)
end)
describe("get_web_imgs_path", function()
it("Gets img path from markdown img string", function()
assert.are.same({'https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png'},
ProcessMD.get_web_imgs_path(
'![alt text](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png)'))
end)
it("Doen't return local paths", function()
assert.are.same({}, ProcessMD.get_web_imgs_path('![alt text](logo)'))
end)
it("Gets img path from makrdown text", function()
assert.are.same({'https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png'},
ProcessMD.get_web_imgs_path(
'Tesing text, only simple ![img](https://github.com/adam-p/markdown-here/raw/master/src/common/images/icon48.png) img passed'))
end)
it("Doesn't return links paths", function()
assert.are.same({}, ProcessMD.get_web_imgs_path(
'My favorite search engine is [Duck Duck Go](https://duckduckgo.com).'))
end)
it("Works for multiple images links", function()
assert.are.same({'http://test.ml/tux.png', 'http://w3.impa.br/~diego/software/luasocket/luasocket.png'},
ProcessMD.get_web_imgs_path(
'![Tux, the Linux mascot](http://test.ml/tux.png) and even ![](http://w3.impa.br/~diego/software/luasocket/luasocket.png)'))
end)
end)
describe("replace_paths", function()
it("Replaces original paths with passed", function()
assert.equal('![Tux, the Linux mascot](./tux.png) and even ![](./luasocket.png)', ProcessMD.replace_paths(
'![Tux, the Linux mascot](http://test.ml/tux.png) and even ![](http://w3.impa.br/~diego/software/luasocket/luasocket.png)',
{
["http://test.ml/tux.png"] = "./tux.png",
["http://w3.impa.br/~diego/software/luasocket/luasocket.png"] = "./luasocket.png"
}))
end)
end)
describe("get_header", function()
it('Gets header of md file', function()
local f = io.open(assets_dir .. 'tmp_dir/some file.md', 'r')
local content = f:read('a')
local header
content, header = ProcessMD.get_header(content, assets_dir .. 'tmp_dir/some file.md', true)
f:close()
assert.equal('Header_:D', header)
end)
it("Returns empty line if no header", function()
local f = io.open(assets_dir .. 'tmp_dir/and one more.md', 'r')
local content = f:read('a')
local header
_, header = ProcessMD.get_header(content, assets_dir .. 'tmp_dir/and one more.md', true)
f:close()
assert.equal('and_one_more', header)
end)
end)
describe("save_document", function()
local function clean()
Utils.clean_assets({'Article_name.md'})
end
setup(clean)
teardown(clean)
it('Saves markdown document into file', function()
local temp_content = 'Test document'
ProcessMD.save_document(assets_dir, 'Article_name', temp_content)
local f = io.open(assets_dir .. 'Article_name.md', 'r')
assert.is_not_nil(f)
local f_content = f:read('a')
assert.equal(temp_content, f_content)
f:close()
end)
end)
end)

34
spec/utils.lua Normal file
View File

@ -0,0 +1,34 @@
local utils = {}
local assets_dir = './spec/assets/'
function utils.list_dir(path)
local files = {}
local pfile = io.popen('ls -a "' .. path .. '"')
for file in pfile:lines() do
table.insert(files, file)
end
return files
end
function utils.clean_assets(files)
local am = 0
for _, file in ipairs(files) do
os.execute('rm -rf ' .. assets_dir .. file)
am = am + 1
end
return am
end
function utils.copy_file(from_path, dest_path)
local from, dest = io.open(from_path, 'r'), io.open(dest_path, 'w')
dest:write(from:read('a'))
from:close()
dest:close()
end
return utils

View File

@ -1,77 +0,0 @@
#!/usr/bin/env python3
''' Main module '''
import os
import sys
import shutil
from get_args import parse_args
from article_process import get_article_title, get_paths, replace_article_paths, rm_first_line
from netw_ops import download_and_save, upload_to_server, add_to_list_on_server
def main(argv):
'''Main function'''
input_file = ''
server_cred = ''
output_directory = ''
res = parse_args(argv)
if len(res) == 2:
input_file, output_directory = res # pylint: disable=unbalanced-tuple-unpacking
elif len(res) == 3:
input_file, server_cred, output_directory = res
try:
ifile = open(input_file, "r")
except IOError as ex:
print("Couldn't open input file")
print(ex)
sys.exit(2)
text = ifile.read()
if output_directory[-1] != '/':
output_directory += '/'
article_filename = get_article_title(text)
if not article_filename:
article_filename = input_file.split('/')[-1]
else:
text = rm_first_line(text)
article_folder = article_filename.split('.')[0] + '/'
if not os.path.exists(article_folder):
os.makedirs(article_folder)
paths = get_paths(text)
res_paths = []
for url in paths:
try:
res_paths.append(download_and_save(url, article_folder))
except Exception as ex:
paths.remove(url)
print("Couldn't process image:", ex, '\nurl:', url)
raise "Couldn't process image"
text = replace_article_paths(text, paths, res_paths)
open(article_folder + article_filename, "w").write(text)
if server_cred:
upload_to_server(server_cred, article_folder, output_directory)
shutil.rmtree(article_folder)
add_to_list_on_server(
server_cred, article_folder, output_directory)
if __name__ == '__main__':
main(sys.argv[1:])

67
src/arg_proc.lua Normal file
View File

@ -0,0 +1,67 @@
local args = {}
function args.read_config(argv, path, print_err)
local function file_exists(file_path)
local f = io.open(file_path, "rb")
if f then
f:close()
end
return f ~= nil
end
if file_exists(path) then
for line in io.lines(path) do
local key, value = line:match("([^=]+)=([^=]+)")
if not (key and value) then
print_err:error("Wrong config file syntax")
end
argv[key] = value
end
end
return argv
end
function args.parse()
local Argparse = require("lib.argparse")
local parser = Argparse("md-parser",
"Command line utility for saving markdown document online assets locally and upload it to server with scp")
parser:argument("input", "Input file.")
parser:option("-o --output", "Output directory.", "./")
parser:option("-s --server", "Username and hostname in username@hostname notation.")
parser:option("-c --config",
"Configuration file like\n\t\t\tpath= Output path on server\n\t\t\thost= Server username and hostname in username@hostname notation")
parser:flag("-u --upload", "If sould upload to server")
local arguments = parser:parse()
if not arguments.input:find('[^%.]+%.md$') then
parser:error("You can't process non-markdown file")
end
if ((arguments.output or arguments.server) and arguments.config) then
parser:error("You can't use both command line parameters and configuration file")
end
if (arguments.config) then
arguments = args.read_config(arguments, arguments.config, parser)
end
if arguments.upload and not (arguments.output and arguments.server) then
parser:error("You should specify output directory and server credentials for upload")
end
if arguments.output[arguments.output:len()] ~= '/' then
arguments.output = arguments.output .. '/'
end
return arguments
end
return args

View File

@ -1,52 +0,0 @@
'''Provides some article content operations'''
import re
def get_paths(string):
'''Gets images paths in article'''
img_reg = re.compile(r'!\[.*?\]\(.*?\)')
path_reg = re.compile(r'(?<=\()http[s]{0,1}.*?(?=\))')
imgs = img_reg.findall(string)
paths = []
for img in imgs:
res = path_reg.search(img)
if res:
paths.append(res.group())
return paths
def rm_first_line(string):
'''Removes first line from string'''
return string[string.find('\n') + 1:]
def get_article_title(string):
'''Gets article title'''
header = ''
if string[0] == '#':
header = string.split('\n')[0]
while header[0] == '#' or header[0] == ' ':
header = header[1:]
header += '.md'
header = header.replace(' ', '_')
return header
def replace_article_paths(string, orig_paths, res_paths):
'''Replaces all web links with downloaded ones'''
for i, val in enumerate(res_paths):
print(val[2:])
string = string.replace(orig_paths[i], '/articles/' + val[2:])
return string

View File

@ -1,78 +0,0 @@
''' Module to get arguments passed into Main module'''
import getopt
import sys
def read_cfg_file(path):
'''Reads config file'''
cfg = open(path, 'r')
buff = cfg.read()
for line in buff.split('\n'):
if line.split('=')[0] == 'output':
output_directory = line.split('=')[1]
elif line.split('=')[0] == 'host':
server_cred = line.split('=')[1]
if not (output_directory and server_cred):
print("No config file provided")
sys.exit(2)
return output_directory, server_cred
def usage():
'''Prints usage instructions'''
print(''''Usage: ./article_uploader -i <inputfile> -o <output directory>
or ./article_uploader -u -o <output_directory> -s <server username and hostname in username@hostname notation>
or ./article_uploader -u -c <configuration file>
with configuration file such as
path=<output path on server>
host=<server username and hostname in username@hostname notation>''')
def parse_args(argv):
'''Parses arguments provided by user'''
input_file = ''
output_directory = ''
upload_to_server = False
server_cred = ''
cfg_path = ''
try:
opts = getopt.getopt(argv, 'hi:o:us:c:')[0]
except getopt.GetoptError:
usage()
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
usage()
sys.exit()
elif opt == '-i':
input_file = arg
elif opt == '-o':
output_directory = arg
elif opt == '-u':
upload_to_server = True
elif opt == '-s':
server_cred = arg
elif opt == '-c':
cfg_path = arg
if not (input_file and (output_directory or upload_to_server) or cfg_path):
usage()
sys.exit(2)
if upload_to_server and not (server_cred and output_directory) and cfg_path:
output_directory, server_cred = read_cfg_file(cfg_path)
else:
usage()
sys.exit(2)
if server_cred and output_directory:
return input_file, server_cred, output_directory
else:
return input_file, output_directory

85
src/main.lua Normal file
View File

@ -0,0 +1,85 @@
local Arguments = require("src.arg_proc")
local ProcessMD = require("src.process_md")
local NetwOps = require("src.netw_ops")
local function get_document_info(filename, upload)
local status, content = pcall(ProcessMD.get_file_content, filename)
if not status then
print('Error: ' .. content)
os.exit(1)
end
local document_name
content, document_name = ProcessMD.get_header(content, upload)
local urls = ProcessMD.get_web_imgs_path(content)
return document_name, content, urls
end
local function compose_document_destination_folder(upload, path, document_name)
if upload or not path then
path = './'
end
path = path .. document_name
return path
end
local function download_netw_assets(urls, dest)
local dict = {}
for _, path in ipairs(urls) do
local status, res_file = pcall(NetwOps.download_to, path, dest)
if not status then
print('Error: ' .. res_file)
os.exit(1)
end
dict[path] = res_file
end
return dict
end
local function upload_to_server(local_article_directory, server_cred, server_path, document_name)
local status, err = pcall(NetwOps.upload_dir, local_article_directory, server_cred, server_path .. document_name)
if not status then
print('Error: ' .. err)
os.exit(1)
end
local status, err = pcall(NetwOps.download_db, server_cred, server_path, local_article_directory)
if not status then
print('Error: ' .. err)
os.exit(1)
end
NetwOps.insert_article(local_article_directory, document_name)
local status, err = pcall(NetwOps.upload_db, server_cred, server_path, local_article_directory)
if not status then
print('Error: ' .. err)
os.exit(1)
end
os.execute('rm -rf "' .. local_article_directory .. '"')
end
local params = Arguments.parse()
local document_name, content, urls = get_document_info(params.input, params.upload)
local local_article_directory = compose_document_destination_folder(params.upload, params.output, document_name)
local dict = download_netw_assets(urls, local_article_directory)
content = ProcessMD.replace_paths(content, dict)
local status, err = pcall(ProcessMD.save_document, local_article_directory, document_name, content)
if not status then
print('Error: ' .. err)
os.exit(1)
end
if params.upload then
upload_to_server(local_article_directory, params.server, params.output, document_name)
end

107
src/netw_ops.lua Normal file
View File

@ -0,0 +1,107 @@
local netw_ops = {}
local https = require("ssl.https")
local http = require("socket.http")
function netw_ops.download_to(url, output)
local res = os.execute('mkdir -p "' .. output .. '"')
if not res then
error("Couldn't create output directory " .. output)
end
local filename = netw_ops.get_file_name(url)
if not filename then
error("Wrong url, if it is really an image, try to download it on your own: " .. url)
end
local response, code
local _
if url:match("^https://") then
response, code, _, _ = https.request(url)
else
response, code, _, _ = http.request(url)
end
local ofile, err
if code == 200 then
ofile, err = io.open(output .. '/' .. filename, "wb")
else
error("Error downloading file. Server returned " .. code .. ' code')
end
if not ofile then
error(err, 2)
end
ofile:write(response)
ofile:close()
return './' .. filename
end
function netw_ops.get_file_name(str)
return str:match("/([^/]*)$")
end
function netw_ops.scp_wrap(from_path, to_path)
return os.execute('scp -prq ' .. from_path .. ' ' .. to_path)
end
function netw_ops.upload_dir(local_path, server_cred, server_path)
os.execute('ssh ' .. server_cred .. ' "rm -rf ' .. server_path .. '"')
local res = netw_ops.scp_wrap(local_path .. '/', server_cred .. ':' .. server_path .. '/')
if res then
return res
else
error("Could't upload directory to server " .. server_cred)
end
end
function netw_ops.download_db(server_cred, server_path, local_path)
local res = netw_ops.scp_wrap(server_cred .. ':' .. server_path .. 'list.db', local_path)
if res then
return res
else
error("Could't download server" .. server_cred .. " articles database")
end
end
function netw_ops.insert_article(local_path, document_name)
local lines = ''
local not_in = true
local new_line = math.floor(os.time()) .. ' ' .. document_name
local file_name = local_path .. '/list.db'
local file = io.open(file_name, 'r')
for line in file:lines() do
if line:find(document_name) then
line = new_line
not_in = false
end
lines = lines .. (lines:len() > 0 and '\n' or '') .. line
end
if not_in then
lines = lines .. (lines:len() > 0 and '\n' or '') .. new_line
end
file:close()
file = io.open(file_name, 'w')
file:write(lines)
file:close()
end
function netw_ops.upload_db(server_cred, server_path, local_path)
local res = netw_ops.scp_wrap(local_path .. '/' .. 'list.db', server_cred .. ':' .. server_path)
if res then
return res
else
error("Could't upload server " .. server_cred .. " articles database")
end
end
return netw_ops

View File

@ -1,71 +0,0 @@
'''Provides network related operations'''
import subprocess
import os
import time
import requests
def download_and_save(url, out_dir):
'''Downloads file from url and saves it into out_dir'''
file = requests.get(url)
out_path = './' + out_dir + url.split('/')[-1]
try:
open(out_path, 'wb').write(file.content)
return out_path
except IOError as ex:
print(ex)
raise 'Couldn\'t open file for write'
def scp_wrap(recursively, from_path, to_path):
'''Downloads/uploads files from/to server using scp'''
if recursively:
proc = subprocess.Popen(["scp", "-r", from_path, to_path])
else:
proc = subprocess.Popen(["scp", from_path, to_path])
sts = os.waitpid(proc.pid, 0) # pylint: disable=unused-variable
def upload_to_server(server_cred, local_path, server_path):
'''Uploads selected folder to server using scp'''
scp_wrap(True, local_path, server_cred + ':' + server_path)
def add_to_list_on_server(server_cred, local_path, server_path):
'''Reads list of articles on server and add new article to it'''
article_name = local_path[:-1]
scp_wrap(False, server_cred + ':' + server_path + 'list.db', './')
articles_list_file = open('list.db', 'r+')
articles_list = articles_list_file.read()
articles_list_s = articles_list.split('\n')
flag = True
for i, val in enumerate(articles_list_s):
if article_name in val:
line_s = val.split(' ')
line_s[0] = str(int(time.time()))
articles_list_s[i] = ' '.join(line_s)
flag = False
if flag:
articles_list_s.append(str(int(time.time())) + ' ' + article_name)
articles_list = '\n'.join(filter(None, articles_list_s))
articles_list_file.seek(0)
articles_list_file.write(articles_list)
articles_list_file.close()
scp_wrap(False, 'list.db', server_cred + ':' + server_path)

65
src/process_md.lua Normal file
View File

@ -0,0 +1,65 @@
local process_md = {}
function process_md.get_file_content(path)
local f, err = io.open(path, "r+")
assert(f, err)
local content = f:read("a")
f:close()
return content
end
function process_md.get_web_imgs_path(str)
local urls = {}
for link in string.gmatch(str, '!%[[^%]]*%]%((http[^%)]+)%)') do
table.insert(urls, link)
end
return urls
end
function process_md.replace_paths(str, dict)
for key, value in pairs(dict) do
str = string.gsub(str, key, value)
end
return str
end
function process_md.get_header(content, filename, upload)
local first_line = content:match('([^\n]-)\n')
local header
if first_line then
header = first_line:match("# (.*)")
end
if not header then
return content, filename:match('/([^/]-).md$'):gsub(' ', '_')
else
if upload then
content = process_md.remove_first_line(content)
end
return content, header:gsub(' ', '_')
end
end
function process_md.remove_first_line(content)
return content:match('[^\n]-\n+(.*)')
end
function process_md.save_document(dest, header, content)
local filename = dest .. '/' .. header .. '.md'
local output_file = io.open(filename, "w")
if not output_file then
error("Couldn't open output file: " .. filename)
end
output_file:write(content)
output_file:close()
end
return process_md