From d813bcb5b038e07db230813b084d24b6f1edc5ca Mon Sep 17 00:00:00 2001 From: Nao Ueda Date: Tue, 3 Nov 2020 19:21:03 +0900 Subject: change structure to distribute a package --- .gitignore | 7 +++ ppad.py | 135 ------------------------------------------------------- ppad/__init__.py | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++++ ppad/__main__.py | 4 ++ pyproject.toml | 3 ++ setup.cfg | 27 +++++++++++ tests/.gitkeep | 0 7 files changed, 170 insertions(+), 135 deletions(-) delete mode 100755 ppad.py create mode 100755 ppad/__init__.py create mode 100644 ppad/__main__.py create mode 100644 pyproject.toml create mode 100644 setup.cfg create mode 100644 tests/.gitkeep diff --git a/.gitignore b/.gitignore index c5edab1..d5258fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,9 @@ +__pycache__/ + +build/ +dist/ +*.eggs +*.egg-info/ + .envrc .direnv/ diff --git a/ppad.py b/ppad.py deleted file mode 100755 index 338b37c..0000000 --- a/ppad.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import time -import datetime -from concurrent import futures -import pytz -from dateutil import parser -import requests -import progressbar - -PAPERTRAIL_API_TOKEN = os.environ.get('PAPERTRAIL_API_TOKEN', None) -ARCHIVES_URL = 'https://papertrailapp.com/api/v1/archives.json' -DEFAULT_REMAIN_SIZE = 25 -HEADERS = {'X-Papertrail-Token': PAPERTRAIL_API_TOKEN} -MIN_INTERVAL_SEC = 0.1 - - -def get_ppheader(response): - hist = [response] + response.history - for h in hist: - if 'X-Rate-Limit-Remaining' not in h.headers: - continue - - return ( - int(h.headers['X-Rate-Limit-Limit']), - int(h.headers['X-Rate-Limit-Remaining']), - int(h.headers['X-Rate-Limit-Reset']) - ) - - -def do_download(url, filename, index): - while True: - try: - with requests.Session() as s: - res = s.get(url, headers=HEADERS) - finishtime = time.time_ns() - (limit, rem, reset) = get_ppheader(res) - - if 200 <= res.status_code < 300: - with open(filename, "wb") as f: - for chunk in res.iter_content(chunk_size=128): - f.write(chunk) - return (limit, rem, reset, finishtime, index) - except requests.ConnectionError: - time.sleep(1) - - -def parse_span(): - if len(sys.argv) == 1: - return None, None - - f = t = None - fromstr = tostr = '' - span = sys.argv[1].split('~') - if len(span) == 1: - fromstr = tostr = span[0] - else: - [fromstr, tostr, *_] = span - - if fromstr: - f = parser.isoparse(fromstr) - if not f.tzname(): - utc = pytz.timezone('UTC') - f = utc.localize(f) - - if tostr: - t = parser.isoparse(tostr) - if not t.tzname(): - utc = pytz.timezone('UTC') - t = utc.localize(t) - - if tostr and fromstr == tostr: - t = t + datetime.timedelta(days=1) - - return f, t - - -def main(): - if not PAPERTRAIL_API_TOKEN: - print('Not set the environment variable `PAPERTRAIL_API_TOKEN`', - file=sys.stderr) - sys.exit(1) - - _from, to = parse_span() - - r = requests.get(ARCHIVES_URL, headers=HEADERS) - r.raise_for_status() - - archives = [ar for ar in r.json() - if (not _from or _from <= parser.isoparse(ar["start"])) - and (not to or parser.isoparse(ar["end"]) < to) - ] - - with futures.ThreadPoolExecutor(max_workers=10) as executor: - future_list = [] - remain = DEFAULT_REMAIN_SIZE - until_reset_sec = 0 - lasttime = time.time_ns() - with progressbar.ProgressBar(max_value=len(archives)) as bar: - for i, ar in enumerate(archives): - future_list.append( - executor.submit( - do_download, - ar['_links']['download']['href'], - ar['filename'], - i - ) - ) - - if len(future_list) < remain: - time.sleep(MIN_INTERVAL_SEC) - continue - - for future in future_list: - (_, rem, reset, finishtime, index) = future.result() - if finishtime > lasttime: - remain = rem - until_reset_sec = reset - lasttime = finishtime - bar.update(index) - - future_list = [] - - if remain <= 0: - time.sleep(until_reset_sec) - remain = DEFAULT_REMAIN_SIZE - continue - - time.sleep(MIN_INTERVAL_SEC) - - -if __name__ == "__main__": - main() diff --git a/ppad/__init__.py b/ppad/__init__.py new file mode 100755 index 0000000..39be363 --- /dev/null +++ b/ppad/__init__.py @@ -0,0 +1,129 @@ +import sys +import os +import time +import datetime +from concurrent import futures +import pytz +from dateutil import parser +import requests +import progressbar + +PAPERTRAIL_API_TOKEN = os.environ.get('PAPERTRAIL_API_TOKEN', None) +ARCHIVES_URL = 'https://papertrailapp.com/api/v1/archives.json' +DEFAULT_REMAIN_SIZE = 25 +HEADERS = {'X-Papertrail-Token': PAPERTRAIL_API_TOKEN} +MIN_INTERVAL_SEC = 0.1 + + +def get_ppheader(response): + hist = [response] + response.history + for h in hist: + if 'X-Rate-Limit-Remaining' not in h.headers: + continue + + return ( + int(h.headers['X-Rate-Limit-Limit']), + int(h.headers['X-Rate-Limit-Remaining']), + int(h.headers['X-Rate-Limit-Reset']) + ) + + +def do_download(url, filename, index): + while True: + try: + with requests.Session() as s: + res = s.get(url, headers=HEADERS) + finishtime = time.time_ns() + (limit, rem, reset) = get_ppheader(res) + + if 200 <= res.status_code < 300: + with open(filename, "wb") as f: + for chunk in res.iter_content(chunk_size=128): + f.write(chunk) + return (limit, rem, reset, finishtime, index) + except requests.ConnectionError: + time.sleep(1) + + +def parse_span(): + if len(sys.argv) == 1: + return None, None + + f = t = None + fromstr = tostr = '' + span = sys.argv[1].split('~') + if len(span) == 1: + fromstr = tostr = span[0] + else: + [fromstr, tostr, *_] = span + + if fromstr: + f = parser.isoparse(fromstr) + if not f.tzname(): + utc = pytz.timezone('UTC') + f = utc.localize(f) + + if tostr: + t = parser.isoparse(tostr) + if not t.tzname(): + utc = pytz.timezone('UTC') + t = utc.localize(t) + + if tostr and fromstr == tostr: + t = t + datetime.timedelta(days=1) + + return f, t + + +def main(): + if not PAPERTRAIL_API_TOKEN: + print('Not set the environment variable `PAPERTRAIL_API_TOKEN`', + file=sys.stderr) + sys.exit(1) + + _from, to = parse_span() + + r = requests.get(ARCHIVES_URL, headers=HEADERS) + r.raise_for_status() + + archives = [ar for ar in r.json() + if (not _from or _from <= parser.isoparse(ar["start"])) + and (not to or parser.isoparse(ar["end"]) < to) + ] + + with futures.ThreadPoolExecutor(max_workers=10) as executor: + future_list = [] + remain = DEFAULT_REMAIN_SIZE + until_reset_sec = 0 + lasttime = time.time_ns() + with progressbar.ProgressBar(max_value=len(archives)) as bar: + for i, ar in enumerate(archives): + future_list.append( + executor.submit( + do_download, + ar['_links']['download']['href'], + ar['filename'], + i + ) + ) + + if len(future_list) < remain: + time.sleep(MIN_INTERVAL_SEC) + continue + + for future in future_list: + (_, rem, reset, finishtime, index) = future.result() + if finishtime > lasttime: + remain = rem + until_reset_sec = reset + lasttime = finishtime + bar.update(index) + + future_list = [] + + if remain <= 0: + time.sleep(until_reset_sec) + remain = DEFAULT_REMAIN_SIZE + continue + + time.sleep(MIN_INTERVAL_SEC) diff --git a/ppad/__main__.py b/ppad/__main__.py new file mode 100644 index 0000000..868d99e --- /dev/null +++ b/ppad/__main__.py @@ -0,0 +1,4 @@ +from . import main + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9787c3b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..d0a6241 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,27 @@ +[metadata] +name = ppad +version = 0.0.7 +url = https://github.com/nao0x2c6/ppad +author = Nao Ueda +author_email = nao.uedder@gmail.com +license_file = LICENSE +description = Papertrail log archives downloader +long_description = file: README.md +long_description_content_type = text/markdown +keywords = papertrail +classifiers = + License :: OSI Approved :: GNU General Public License v2 (GPLv2) + Programming Language :: Python :: 3.6 + +[options] +packages = find: +install_requires = + progressbar2 >= 3 + python-dateutil >= 2 + pytz >= 2020 + requests >= 2 + + +[options.entry_points] +console_scripts = + ppad = ppad:main diff --git a/tests/.gitkeep b/tests/.gitkeep new file mode 100644 index 0000000..e69de29 -- cgit v1.2.3-54-g00ecf