From f0c0188d58c5acf48ec43c8db78385df24503565 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 16:50:42 +0200 Subject: [PATCH 01/24] fix(model/paste): interpolate expiration paste model now consists of expiration instead of lifetime and timestamp. This is to ensure, that man-in-the-middle attackers cannot derive the origin of a paste through observing transport layer server request times. --- src/httpaste/backend/__init__.py | 2 +- src/httpaste/backend/file/paste.py | 28 ++++++++++------------- src/httpaste/backend/sqlite/paste.py | 12 ++++------ src/httpaste/backend/sqlite/paste.sql | 9 ++++---- src/httpaste/controller/paste/__init__.py | 4 ++-- src/httpaste/model/__init__.py | 13 ++++++++--- src/httpaste/model/paste.py | 26 +++++++++++++-------- 7 files changed, 50 insertions(+), 44 deletions(-) diff --git a/src/httpaste/backend/__init__.py b/src/httpaste/backend/__init__.py index 2e176d9..48978a0 100644 --- a/src/httpaste/backend/__init__.py +++ b/src/httpaste/backend/__init__.py @@ -26,7 +26,7 @@ class SQLite(Backend): def __init__(self, parameters: SqliteParameters): - parameters['connection'] = get_sqlite_connection(parameters) + parameters = SqliteParameters(parameters.path, get_sqlite_connection(parameters)) self.user = SqliteUser(parameters, User) self.paste = SqlitePaste(parameters, Paste) diff --git a/src/httpaste/backend/file/paste.py b/src/httpaste/backend/file/paste.py index d27b7ca..faefaf0 100644 --- a/src/httpaste/backend/file/paste.py +++ b/src/httpaste/backend/file/paste.py @@ -7,6 +7,15 @@ from pathlib import Path from ast import literal_eval +COLUMNS = [ + 'data', + 'data_hash', + 'sub', + 'expiration', + 'encoding' +] + + def load( proto: object, path: Path, @@ -22,13 +31,7 @@ def load( return None cells = {} - for column in [ - 'data', - 'data_hash', - 'sub', - 'timestamp', - 'lifetime', - 'encoding']: + for column in COLUMNS: cell = row.joinpath(column) @@ -56,8 +59,7 @@ def load( cells['sub'], cells['data'], cells['data_hash'], - cells['timestamp'], - cells['lifetime'], + cells['expiration'], cells['encoding']) @@ -68,13 +70,7 @@ def dump(model: object, path: Path, model_schema: type) -> None: row = path.joinpath(model.pid.hex()) row.mkdir(parents=True, exist_ok=True) - for column in [ - 'data', - 'data_hash', - 'sub', - 'timestamp', - 'lifetime', - 'encoding']: + for column in COLUMNS: cell = row.joinpath(column) cell_schema = getattr(model_schema, column) diff --git a/src/httpaste/backend/sqlite/paste.py b/src/httpaste/backend/sqlite/paste.py index 61f8cd1..150e3cb 100644 --- a/src/httpaste/backend/sqlite/paste.py +++ b/src/httpaste/backend/sqlite/paste.py @@ -11,7 +11,7 @@ def load(proto: object, connection: Connection, model_class: type): cur = connection.cursor() cur.execute( - 'SELECT pid, data, data_hash, sub, timestamp, lifetime, encoding FROM pastes WHERE pid=?', + 'SELECT pid, data, data_hash, sub, expiration, encoding FROM pastes WHERE pid=?', (proto.pid, )) @@ -24,8 +24,7 @@ def load(proto: object, connection: Connection, model_class: type): result['sub'], result['data'], result['data_hash'], - result['timestamp'], - result['lifetime'], + result['expiration'], result['encoding']) return None @@ -38,14 +37,13 @@ def dump(model: object, connection: Connection): cur = connection.cursor() cur.execute( - '''INSERT INTO pastes (pid, data, data_hash, sub, timestamp, lifetime, encoding) - VALUES (?,?,?,?,?,?,?)''', + '''INSERT INTO pastes (pid, data, data_hash, sub, expiration, encoding) + VALUES (?,?,?,?,?,?)''', (model.pid, model.data, model.data_hash, model.sub, - model.timestamp, - model.lifetime, + model.expiration, model.encoding)) connection.commit() diff --git a/src/httpaste/backend/sqlite/paste.sql b/src/httpaste/backend/sqlite/paste.sql index 5788d5b..7f9bb46 100644 --- a/src/httpaste/backend/sqlite/paste.sql +++ b/src/httpaste/backend/sqlite/paste.sql @@ -1,10 +1,9 @@ CREATE TABLE IF NOT EXISTS "pastes" ( - "id" BLOB NOT NULL UNIQUE, + "pid" BLOB NOT NULL UNIQUE, "data" BLOB NOT NULL, "data_hash" BLOB NOT NULL, "sub" BLOB UNIQUE, - "timestamp" INTEGER NOT NULL, - "lifetime" INTEGER NOT NULL, - "encoding" TEXT - PRIMARY KEY("id") + "expiration" INTEGER NOT NULL, + "encoding" TEXT, + PRIMARY KEY("pid") ); \ No newline at end of file diff --git a/src/httpaste/controller/paste/__init__.py b/src/httpaste/controller/paste/__init__.py index fcd6ab6..c8f4b87 100644 --- a/src/httpaste/controller/paste/__init__.py +++ b/src/httpaste/controller/paste/__init__.py @@ -71,7 +71,7 @@ def get(**kwargs): config.salt, config.hmac_iterations) try: - data, lifetime, encoding = call() + data, expiration, encoding = call() except paste_model.LifetimeError as e: if kwargs.get('user') is not None: paste_model.remove_safe(pid, sub, pkey, config.backend.paste, @@ -85,7 +85,7 @@ def get(**kwargs): raise ForbiddenError(str(e)) # burn after read - if lifetime < 0: + if expiration < 0: if kwargs.get('user') is not None: paste_model.remove_safe(pid, sub, pkey, config.backend.paste, config.salt, config.hmac_iterations) diff --git a/src/httpaste/model/__init__.py b/src/httpaste/model/__init__.py index 955ded4..b9d1b4f 100644 --- a/src/httpaste/model/__init__.py +++ b/src/httpaste/model/__init__.py @@ -12,6 +12,7 @@ class PasteDataSchema: sub = bytes timestamp = int lifetime = int + expiration = int encoding = str @@ -54,6 +55,14 @@ class PasteEncoding(PasteDataSchema.encoding): """ +class PasteExpiration(PasteDataSchema.expiration): + """Paste Expiration + + < 0: after first acccess + 0: never + """ + + class PasteLifetime(PasteDataSchema.lifetime): """Paste Lifetime """ @@ -128,8 +137,6 @@ class Paste(NamedTuple): #: paste data hash data_hash: Optional[PasteHash] = None #: paste timestamp - timestamp: Optional[PasteTimestamp] = None - #: paste lifetime - lifetime: Optional[PasteLifetime] = None + expiration: Optional[PasteExpiration] = None #: paste encoding encoding: Optional[PasteEncoding] = None diff --git a/src/httpaste/model/paste.py b/src/httpaste/model/paste.py index 5ad9dc8..04ea6a5 100755 --- a/src/httpaste/model/paste.py +++ b/src/httpaste/model/paste.py @@ -10,7 +10,7 @@ from httpaste.helper.crypto import dhash, shash, encrypt, decrypt from httpaste.helper.common import generate_random_string from httpaste.model import (Paste, PasteId, Sub, MasterKey, PasteKey, Salt, PasteData, PasteHash, PasteTimestamp, PasteSub, - PasteLifetime, PasteEncoding) + PasteLifetime, PasteEncoding, PasteExpiration) class NotFoundError(Exception): @@ -87,8 +87,7 @@ def load(proto: Paste, backend: object) -> Optional[Paste]: raise SubError('Paste not owned by user') - if model.lifetime >= 0 and model.timestamp + \ - (60 * model.lifetime) < int(time.time()): + if model.expiration > 0 and model.expiration < int(time.time()): raise LifetimeError('Paste expired') @@ -121,8 +120,7 @@ def load_safe( proto.sub, data, model.data_hash, - model.timestamp, - model.lifetime, + model.expiration, model.encoding) @@ -195,6 +193,11 @@ def create( sub = None timestamp = PasteTimestamp(int(time.time())) + if lifetime < 0: + expiration = -1 + else: + expiration = PasteExpiration(timestamp + (lifetime * 60)) + safe_data = PasteData(encrypt(data, pid, salt, hmac_iter)) model = Paste( @@ -202,8 +205,7 @@ def create( sub, safe_data, data_hash, - timestamp, - lifetime, + expiration, encoding) dump(model, backend) @@ -234,6 +236,11 @@ def create_safe(data: PasteData, safe_sub = PasteSub(shash(sub, data_hash, pid)) timestamp = PasteTimestamp(int(time.time())) + if lifetime < 0: + expiration = -1 + else: + expiration = PasteExpiration(timestamp + (lifetime * 60)) + safe_data = PasteData(encrypt(data, pkey, salt, hmac_iter)) dump(Paste( @@ -241,8 +248,7 @@ def create_safe(data: PasteData, safe_sub, safe_data, data_hash, - timestamp, - lifetime, + expiration, encoding ), backend) @@ -279,7 +285,7 @@ def get(pid: PasteId, backend: object, salt: Salt = Config.salt, hmac_iter: int data = decrypt(model.data, pid, salt, hmac_iter) - return PasteData(data), model.lifetime, model.encoding + return PasteData(data), model.expiration, model.encoding def get_safe( From bd7af0585011ccbce7d7a105d6c913de0ed3579a Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 17:59:04 +0200 Subject: [PATCH 02/24] feat(mode/user): implement complex index to accomodate more personalized user data, the index can now hold more than just paste constraints. It was planned to implement an authentication expiration, however, this does not make sense in an HTTP basic auth context. --- src/httpaste/backend/file/user.py | 18 ++++++--- src/httpaste/model/__init__.py | 6 ++- src/httpaste/model/user.py | 61 +++++++++++++++++++++---------- 3 files changed, 58 insertions(+), 27 deletions(-) diff --git a/src/httpaste/backend/file/user.py b/src/httpaste/backend/file/user.py index 02255b2..72c9256 100644 --- a/src/httpaste/backend/file/user.py +++ b/src/httpaste/backend/file/user.py @@ -6,6 +6,12 @@ acting as cells. from pathlib import Path from ast import literal_eval +COLUMNS = [ + 'sub', + 'key_hash', + 'index', +] + def load( proto: object, @@ -22,15 +28,17 @@ def load( return None cells = {} - for column in ['key_hash', 'index']: + for column in COLUMNS[1:]: cell = row.joinpath(column) - if getattr(model_schema, column) == bytes: - + if not cell.exists(): + cells[column] = None + elif getattr(model_schema, column) == bytes: cells[column] = cell.read_bytes() + elif getattr(model_schema, column) == str: + cells[column] = cell.read_text() else: - cells[column] = literal_eval(cell.read_text()) return model_class( @@ -46,7 +54,7 @@ def dump(model: object, path: Path, model_schema: object) -> None: row = path.joinpath(model.sub.hex()) row.mkdir(parents=True, exist_ok=True) - for column in ['key_hash', 'index']: + for column in COLUMNS[1:]: cell = row.joinpath(column) diff --git a/src/httpaste/model/__init__.py b/src/httpaste/model/__init__.py index 955ded4..e41e4c1 100644 --- a/src/httpaste/model/__init__.py +++ b/src/httpaste/model/__init__.py @@ -1,6 +1,6 @@ """Model """ -from typing import NamedTuple, Optional, Dict, Union +from typing import NamedTuple, Optional, Dict, Union, Any, TypedDict class PasteDataSchema: @@ -89,9 +89,11 @@ class Sub(UserDataSchema.sub): """ -class Index(Dict[PasteId, PasteKey]): +class Index(TypedDict): """User Paste Index """ + auth_expires: int + pastes: Dict[str, Dict[str, Any]] class SerializedIndex(UserDataSchema.index): diff --git a/src/httpaste/model/user.py b/src/httpaste/model/user.py index 7639127..b925063 100755 --- a/src/httpaste/model/user.py +++ b/src/httpaste/model/user.py @@ -2,6 +2,7 @@ """user model interface """ import json +from time import time from typing import Optional from httpaste import Config @@ -34,7 +35,7 @@ class IndexError(Exception): """ -def load( +def _load( proto: User, master_key: str, backend: object, @@ -54,16 +55,19 @@ def load( return None try: - return User( - *model[:-1], - Index(**json.loads(decrypt(model.index, master_key, salt, hmac_iter))) - ) + serialized_data = decrypt(model.index, master_key, salt, hmac_iter) except DecryptionError as e: - raise IndexError('unable to decrypt user index') from e + else: + data = json.loads(serialized_data) + + return User( + *model[:-1], + Index(**data) + ) -def dump( +def _dump( model: User, key: MasterKey, backend: object, @@ -77,7 +81,7 @@ def dump( :param salt: randomization salt """ - if not isinstance(model.index, Index): + if model.index is not None and not isinstance(model.index, dict): raise BaseException('index serialization pre-processing not allowed.') @@ -102,11 +106,13 @@ def load_paste_key( :param salt: randomization salt """ - for k, v in load(User(sub), key, backend, salt, hmac_iter).index.items(): + model = _load(User(sub), key, backend, salt, hmac_iter) + + for k, v in model.index.get('pastes').items(): if bytes.fromhex(k) == pid: - return PasteKey(bytes.fromhex(v)) + return PasteKey(bytes.fromhex(v.get('key'))) return None @@ -128,12 +134,13 @@ def dump_paste_key( :param backend: user model backend """ - model = load(User(sub), key, backend, salt, hmac_iter) + model = _load(User(sub), key, backend, salt, hmac_iter) - dump(User(*model[:-1], Index({ - **model.index, - **{pid.hex(): pkey.hex()} - })), key, backend, salt, hmac_iter) + model.index.setdefault('pastes', {})[pid.hex()] = { + 'key': pkey.hex() + } + + _dump(model, key, backend, salt, hmac_iter) def authenticate( @@ -154,22 +161,36 @@ def authenticate( proto = User(sub) + bogus_decline_msg = 'unable to authenticate' + try: - model = load(proto, key, backend, salt, hmac_iter) + model = _load(proto, key, backend, salt, hmac_iter) except IndexError as e: - raise AuthenticationError('you dun goofed') + raise AuthenticationError(bogus_decline_msg) from e if not model: - model = User(sub, key_hash, Index({})) - dump(model, key, backend, salt, hmac_iter) + data = { + 'auth_expires': int(time()) + (1 * 60) + } + + model = User(sub, key_hash, Index(data)) + _dump(model, key, backend, salt, hmac_iter) else: if model.key_hash != key_hash: - raise AuthenticationError('you dun goofed') + raise AuthenticationError(bogus_decline_msg) return { 'sub': sub, 'master_key': key } + + +__all__ = [ + AuthenticationError, + load_paste_key, + dump_paste_key, + authenticate +] \ No newline at end of file From 27bc790d5d860e06d18e319f2a497f16f0ff8ee5 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 18:47:11 +0200 Subject: [PATCH 03/24] feat: add backend sanitizing --- src/httpaste/__main__.py | 22 +++++++++++++++++++++- src/httpaste/backend/file/__init__.py | 14 ++++++++++++++ src/httpaste/backend/file/paste.py | 17 +++++++++++++++++ src/httpaste/backend/file/user.py | 5 +++++ src/httpaste/backend/sqlite/__init__.py | 8 ++++++++ src/httpaste/backend/sqlite/paste.py | 12 ++++++++++++ src/httpaste/backend/sqlite/user.py | 5 +++++ 7 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/httpaste/__main__.py b/src/httpaste/__main__.py index 399417f..4a6aae4 100644 --- a/src/httpaste/__main__.py +++ b/src/httpaste/__main__.py @@ -92,6 +92,20 @@ def command_init_backend(**kwargs): config.backend.paste.init() +def command_sanitize_backend(**kwargs): + """sanitize the backend + """ + + from httpaste import load_config + + config, _ = load_config(kwargs.get('config')) + + config, _ = load_config(kwargs.get('config')) + + config.backend.user.sanitize() + config.backend.paste.sanitize() + + def parser(): p = argparse.ArgumentParser(description='Process some integers.') @@ -121,6 +135,11 @@ def parser(): help=command_init_backend.__doc__) p_init_backend.add_argument('--config', '-c', required=True) + p_sanitize_backend = sp.add_parser( + 'sanitize-backend', + help=command_sanitize_backend.__doc__) + p_sanitize_backend.add_argument('--config', '-c', required=True) + return p @@ -136,7 +155,8 @@ def main(): 'cgi': command_cgi, 'fcgi': command_fcgi, 'default-config': command_default_config, - 'init-backend': command_init_backend + 'init-backend': command_init_backend, + 'sanitize-backend': command_sanitize_backend }[kwargs.pop('command')](**kwargs) diff --git a/src/httpaste/backend/file/__init__.py b/src/httpaste/backend/file/__init__.py index a8d1126..9352f8f 100644 --- a/src/httpaste/backend/file/__init__.py +++ b/src/httpaste/backend/file/__init__.py @@ -58,6 +58,13 @@ class User(object): return user.init(self.path) + def sanitize(self): + + if self.path.exists(): + return user.sanitize(self.path, self.model_class, self.model_schema) + + return None + class Paste(object): """Filesystem paste model backend @@ -96,3 +103,10 @@ class Paste(object): def init(self): return paste.init(self.path) + + def sanitize(self): + + if self.path.exists(): + return paste.sanitize(self.path, self.model_class, self.model_schema) + + return None \ No newline at end of file diff --git a/src/httpaste/backend/file/paste.py b/src/httpaste/backend/file/paste.py index faefaf0..dbeb341 100644 --- a/src/httpaste/backend/file/paste.py +++ b/src/httpaste/backend/file/paste.py @@ -5,6 +5,7 @@ acting as cells. """ from pathlib import Path from ast import literal_eval +from time import time COLUMNS = [ @@ -98,6 +99,22 @@ def init(path: Path): return None +def sanitize(path: Path, model_class: type, model_schema: type): + + for row in path.iterdir(): + + expiration_cell = row.joinpath('expiration') + + if not expiration_cell.exists(): + continue + + expiration = literal_eval(expiration_cell.read_text()) + + if expiration < int(time()) and expiration > 0: + + delete(model_class(bytes.fromhex(row.name)), path) + + def _rm_tree(pth: Path): for child in pth.iterdir(): if child.is_file(): diff --git a/src/httpaste/backend/file/user.py b/src/httpaste/backend/file/user.py index 72c9256..21e32d0 100644 --- a/src/httpaste/backend/file/user.py +++ b/src/httpaste/backend/file/user.py @@ -82,6 +82,11 @@ def init(path: Path): return None +def sanitize(path: Path, model_class: type, model_schema: type): + + return None + + def _rm_tree(pth: Path): for child in pth.iterdir(): if child.is_file(): diff --git a/src/httpaste/backend/sqlite/__init__.py b/src/httpaste/backend/sqlite/__init__.py index 8bd14e5..5680f5c 100644 --- a/src/httpaste/backend/sqlite/__init__.py +++ b/src/httpaste/backend/sqlite/__init__.py @@ -45,6 +45,10 @@ class User(object): return user.init(self.connection) + def sanitize(self): + + return user.sanitize(self.connection, self.model_class) + class Paste(object): """SQLite paste model backend @@ -74,6 +78,10 @@ class Paste(object): return paste.init(self.connection) + def sanitize(self): + + return paste.sanitize(self.connection, self.model_class) + def get_connection(parameters: Parameters): """get an sqlite connection object diff --git a/src/httpaste/backend/sqlite/paste.py b/src/httpaste/backend/sqlite/paste.py index 150e3cb..bb0eb17 100644 --- a/src/httpaste/backend/sqlite/paste.py +++ b/src/httpaste/backend/sqlite/paste.py @@ -2,6 +2,7 @@ """ from os import path from sqlite3 import Connection +from time import time def load(proto: object, connection: Connection, model_class: type): @@ -67,3 +68,14 @@ def init(connection: Connection): cur.execute(fh.read()) connection.commit() + + +def sanitize(connection: Connection, model_class: type) -> bool: + + cur = connection.cursor() + + cur.execute('''SELECT pid FROM pastes WHERE expiration < ? AND expiration > 0''', (int(time()),)) + + for row in cur.fetchall(): + + delete(model_class(row['pid'])) \ No newline at end of file diff --git a/src/httpaste/backend/sqlite/user.py b/src/httpaste/backend/sqlite/user.py index 423b797..11fc53e 100644 --- a/src/httpaste/backend/sqlite/user.py +++ b/src/httpaste/backend/sqlite/user.py @@ -53,3 +53,8 @@ def init(connection: Connection): cur.execute(fh.read()) connection.commit() + + +def sanitize(connection: Connection, model_class) -> bool: + + return None \ No newline at end of file From dc5c644f29ab52621cbe77a99e41c4dc29a56a81 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 18:50:19 +0200 Subject: [PATCH 04/24] fix(model/paste): remove faulty var from get_safe() --- src/httpaste/model/paste.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpaste/model/paste.py b/src/httpaste/model/paste.py index 04ea6a5..f7f856d 100755 --- a/src/httpaste/model/paste.py +++ b/src/httpaste/model/paste.py @@ -300,4 +300,4 @@ def get_safe( model = load_safe(Paste(pid, sub), pkey, backend, salt, hmac_iter) - return PasteData(model.data), model.lifetime, model.encoding + return PasteData(model.data), model.expiration, model.encoding From 59823fe6d742984c122c80a08d892b88002b24ff Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 19:00:13 +0200 Subject: [PATCH 05/24] chore: update package version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index bc3179b..ebd2764 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = httpaste-victorykit -version = 1.0.0-alpha.1 +version = 1.0.0-alpha author = Tiara Rodney author_email = t.rodney@victoryk.it description = a versatile HTTP pastebin From b3273ee9e29eb2a283a561a4715befdcc87dde60 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 19:15:31 +0200 Subject: [PATCH 06/24] chore: update setup.cfg classifiers --- setup.cfg | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index ebd2764..a74a667 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,10 +10,14 @@ url = https://victorykit.bitbucket.io/httpaste project_urls = Bug Tracker = https://bitbucket.org/victorykit/httpaste/jira classifiers = - topic = : Software Development :: Libraries :: Python Modules - Programming Language :: Python :: 3 + Development Status :: 3 - Alpha + Intended Audience :: Developers + Topic :: Internet :: WWW/HTTP :: WSGI :: Server + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 Operating System :: OS Independent - License :: Other/Proprietary License + License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+) [options] install_requires = From e670f0be274310301d88f59a3dac36a2f7b81d35 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 19:35:20 +0200 Subject: [PATCH 07/24] fix(helper): initialize module chore: upgrade version --- setup.cfg | 2 +- src/httpaste/helper/__init__.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 src/httpaste/helper/__init__.py diff --git a/setup.cfg b/setup.cfg index a74a667..d330606 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = httpaste-victorykit -version = 1.0.0-alpha +version = 1.0.1-alpha author = Tiara Rodney author_email = t.rodney@victoryk.it description = a versatile HTTP pastebin diff --git a/src/httpaste/helper/__init__.py b/src/httpaste/helper/__init__.py new file mode 100644 index 0000000..e69de29 From 678935bd0e8b45c528c80a97ddb8551ec3428187 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 19:51:05 +0200 Subject: [PATCH 08/24] fix: catch standalone command exception gevent will not be installed by default, therefore print a message explaining required steps --- src/httpaste/__main__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/httpaste/__main__.py b/src/httpaste/__main__.py index 399417f..24763f9 100644 --- a/src/httpaste/__main__.py +++ b/src/httpaste/__main__.py @@ -30,7 +30,14 @@ def command_standalone(**kwargs): """ from httpaste import load_config, get_flask_app - from gevent.pywsgi import WSGIServer + + try: + from gevent.pywsgi import WSGIServer + except ImportError as e: + raise ImportError(' '.join(( + 'gevent is currently not installed.', + 'Please install it by running \'python3 -m pip install gevent\'.' + ))) from e config, server_config = load_config(kwargs.get('config_path')) From 50af1a4587d1a6f965e1fbdb60601755857cfebb Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 19:51:52 +0200 Subject: [PATCH 09/24] fix(__main__): require command argument --- src/httpaste/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpaste/__main__.py b/src/httpaste/__main__.py index 24763f9..7b6de2f 100644 --- a/src/httpaste/__main__.py +++ b/src/httpaste/__main__.py @@ -103,7 +103,7 @@ def parser(): p = argparse.ArgumentParser(description='Process some integers.') - sp = p.add_subparsers(dest='command') + sp = p.add_subparsers(dest='command', required=True) p_standalone = sp.add_parser('standalone', help=command_standalone.__doc__) p_standalone.add_argument('--config-path', '-c', required=True) From c88e3ab17ee0672236794735ea28793a467fd130 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 19:51:05 +0200 Subject: [PATCH 10/24] fix: catch standalone command exception gevent will not be installed by default, therefore print a message explaining required steps --- src/httpaste/__main__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/httpaste/__main__.py b/src/httpaste/__main__.py index 4a6aae4..f019a95 100644 --- a/src/httpaste/__main__.py +++ b/src/httpaste/__main__.py @@ -30,7 +30,14 @@ def command_standalone(**kwargs): """ from httpaste import load_config, get_flask_app - from gevent.pywsgi import WSGIServer + + try: + from gevent.pywsgi import WSGIServer + except ImportError as e: + raise ImportError(' '.join(( + 'gevent is currently not installed.', + 'Please install it by running \'python3 -m pip install gevent\'.' + ))) from e config, server_config = load_config(kwargs.get('config_path')) From 6bef30f1f345f4b66b1d0044bb68b66e820ff0ea Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 19:51:52 +0200 Subject: [PATCH 11/24] fix(__main__): require command argument --- src/httpaste/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpaste/__main__.py b/src/httpaste/__main__.py index f019a95..436a46c 100644 --- a/src/httpaste/__main__.py +++ b/src/httpaste/__main__.py @@ -117,7 +117,7 @@ def parser(): p = argparse.ArgumentParser(description='Process some integers.') - sp = p.add_subparsers(dest='command') + sp = p.add_subparsers(dest='command', required=True) p_standalone = sp.add_parser('standalone', help=command_standalone.__doc__) p_standalone.add_argument('--config-path', '-c', required=True) From d367d43524c07501dba679fceaebcf38be1da812 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 20:06:44 +0200 Subject: [PATCH 12/24] fix(schema): initialize module chore: upgrade version --- setup.cfg | 2 +- src/httpaste/schema/__init__.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 src/httpaste/schema/__init__.py diff --git a/setup.cfg b/setup.cfg index d330606..f081dbf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = httpaste-victorykit -version = 1.0.1-alpha +version = 1.0.3-alpha author = Tiara Rodney author_email = t.rodney@victoryk.it description = a versatile HTTP pastebin diff --git a/src/httpaste/schema/__init__.py b/src/httpaste/schema/__init__.py new file mode 100644 index 0000000..e69de29 From dfff5f15237e6ffb05941d7a67597714fadf12cb Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 20:20:09 +0200 Subject: [PATCH 13/24] fix(setup.cfg): add schema data path --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index f081dbf..802dfa7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,3 +37,6 @@ console_scripts = [options.packages.find] where = src + +[options.package_data] +schema = src/httpaste/schema/httpaste.openapi.json \ No newline at end of file From 6e0229f34f2ac5f035dc3e8e6c9950884c5e32ac Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 01:29:42 +0200 Subject: [PATCH 14/24] fix(schema): add parameters to paste/private route chore: upgrade version resolves: HTTPASTE-32 --- setup.cfg | 4 ++-- src/httpaste/schema/httpaste.openapi.json | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 802dfa7..8c8ebc5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = httpaste-victorykit -version = 1.0.3-alpha +version = 1.0.4-alpha author = Tiara Rodney author_email = t.rodney@victoryk.it description = a versatile HTTP pastebin @@ -39,4 +39,4 @@ console_scripts = where = src [options.package_data] -schema = src/httpaste/schema/httpaste.openapi.json \ No newline at end of file +schema = src/httpaste/schema/httpaste.openapi.json diff --git a/src/httpaste/schema/httpaste.openapi.json b/src/httpaste/schema/httpaste.openapi.json index 163a57e..8f55586 100644 --- a/src/httpaste/schema/httpaste.openapi.json +++ b/src/httpaste/schema/httpaste.openapi.json @@ -161,6 +161,15 @@ }, { "$ref": "#/components/parameters/syntax" + }, + { + "$ref": "#/components/parameters/format" + }, + { + "$ref": "#/components/parameters/linenos" + }, + { + "$ref": "#/components/parameters/mime" } ], "responses": { From bdefd7e592d9bf7ed44fbbfa73649147ddea5ca8 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 03:48:18 +0200 Subject: [PATCH 15/24] refactor(setuptools): include sql schema fix(backend/sqlite): load files through importlib is required since package will be distributed as python egg chore: upgrade version --- setup.cfg | 5 +++-- src/httpaste/__main__.py | 9 +++++---- src/httpaste/backend/sqlite/paste.py | 3 ++- src/httpaste/backend/sqlite/user.py | 3 ++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/setup.cfg b/setup.cfg index 8c8ebc5..62ebeae 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = httpaste-victorykit -version = 1.0.4-alpha +version = 1.0.5-alpha author = Tiara Rodney author_email = t.rodney@victoryk.it description = a versatile HTTP pastebin @@ -39,4 +39,5 @@ console_scripts = where = src [options.package_data] -schema = src/httpaste/schema/httpaste.openapi.json +openapi_schema = src/httpaste/schema/httpaste.openapi.json +sql_schema = *.sql diff --git a/src/httpaste/__main__.py b/src/httpaste/__main__.py index 436a46c..ec35404 100644 --- a/src/httpaste/__main__.py +++ b/src/httpaste/__main__.py @@ -2,6 +2,7 @@ """ import argparse import os +from importlib.resources import open_text def _this_dir(basename: str) -> str: @@ -20,7 +21,7 @@ def _path_output(path, echo: bool = False) -> str: return path else: - with open(path, 'r') as fh: + with open_text('httpaste', path) as fh: return fh.read() @@ -51,21 +52,21 @@ def command_wsgi(**kwargs): """get WSGI script """ - print(_path_output(_this_dir('wsgi.py'), kwargs.get('echo'))) + print(_path_output('wsgi.py', kwargs.get('echo'))) def command_cgi(**kwargs): """get CGI script """ - print(_path_output(_this_dir('cgi.py'), kwargs.get('echo'))) + print(_path_output('cgi.py', kwargs.get('echo'))) def command_fcgi(**kwargs): """get FastCGI script """ - print(_path_output(_this_dir('fcgi.py'), kwargs.get('echo'))) + print(_path_output('fcgi.py', kwargs.get('echo'))) def command_default_config(**kwargs): diff --git a/src/httpaste/backend/sqlite/paste.py b/src/httpaste/backend/sqlite/paste.py index bb0eb17..51c9fc6 100644 --- a/src/httpaste/backend/sqlite/paste.py +++ b/src/httpaste/backend/sqlite/paste.py @@ -3,6 +3,7 @@ from os import path from sqlite3 import Connection from time import time +from importlib.resources import open_text def load(proto: object, connection: Connection, model_class: type): @@ -63,7 +64,7 @@ def init(connection: Connection): cur = connection.cursor() - with open(path.join(path.dirname(__file__), 'paste.sql'), 'r') as fh: + with open_text('httpaste.backend.sqlite', 'paste.sql') as fh: cur.execute(fh.read()) diff --git a/src/httpaste/backend/sqlite/user.py b/src/httpaste/backend/sqlite/user.py index 11fc53e..e4f712d 100644 --- a/src/httpaste/backend/sqlite/user.py +++ b/src/httpaste/backend/sqlite/user.py @@ -3,6 +3,7 @@ from os import path from sqlite3 import Connection from httpaste.model import User +from importlib.resources import open_text def load(proto: User, connection: Connection): @@ -48,7 +49,7 @@ def init(connection: Connection): cur = connection.cursor() - with open(path.join(path.dirname(__file__), 'user.sql'), 'r') as fh: + with open_text('httpaste.backend.sqlite', 'user.sql') as fh: cur.execute(fh.read()) From fc367ed4cb25da544ca119509c956f88eb52b8b6 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 04:29:23 +0200 Subject: [PATCH 16/24] fix(wsgi+config): fault environment setup chore: upgrade version --- setup.cfg | 2 +- src/httpaste/__init__.py | 17 +++++++++++++---- src/httpaste/wsgi.py | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/setup.cfg b/setup.cfg index 62ebeae..af12e9a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = httpaste-victorykit -version = 1.0.5-alpha +version = 1.0.6-alpha author = Tiara Rodney author_email = t.rodney@victoryk.it description = a versatile HTTP pastebin diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 6eea5ba..3a4cb21 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -143,6 +143,7 @@ from inspect import isclass from configparser import ConfigParser from ast import literal_eval from io import StringIO +from os import environ from connexion import FlaskApp from connexion.resolver import RestyResolver @@ -158,7 +159,7 @@ from httpaste.helper.http import ( UnauthorizedError) -CONFIGPATH_ENVIRON = 'HTTPASTE_CONFIG' +CONFIGPATH_ENVIRON = 'HTTPASTE_CONFIGPATH' def get_sanitized_config_charset(charset: str): @@ -198,17 +199,17 @@ class ServerConfig: bind_address = None -def get_config_path(environ: str = CONFIGPATH_ENVIRON): +def get_config_path(var_name: str = CONFIGPATH_ENVIRON): """ """ try: - return os.environ[environ] + return environ[var_name] except KeyError as e: raise ConfigError( - 'environment variable \'{environ}\' not set.') from e + f'environment variable \'{var_name}\' not set.') from e def load_config(path: str) -> Tuple[Config, ServerConfig]: @@ -322,6 +323,14 @@ def get_flask_app( with application.app.app_context(): application.app.httpaste = config + #add header for browsers to present a sign-in prompt + @application.app.after_request + def rewrite_forbidden_request(response): + + if response.status_code in [401]: + response.headers['WWW-Authenticate'] = 'Basic realm="private"' + return response + return application diff --git a/src/httpaste/wsgi.py b/src/httpaste/wsgi.py index e45f821..9e43ff0 100755 --- a/src/httpaste/wsgi.py +++ b/src/httpaste/wsgi.py @@ -3,6 +3,6 @@ """ from httpaste import load_config, get_flask_app, get_config_path -config, server_config = load_config(get_config_path) +config, server_config = load_config(get_config_path()) application = get_flask_app(config, server_config) From 65f50770959c0d877d8f138d44f10884cc973905 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 16:15:14 +0200 Subject: [PATCH 17/24] fix(init): add importlib context to connexion init connexion openapi spec reference now provided through importlib context manager, since file will exist inside of python egg, therefore not being accesible through package file path chore: upgrade version --- setup.cfg | 2 +- src/httpaste/__init__.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/setup.cfg b/setup.cfg index af12e9a..0382db4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = httpaste-victorykit -version = 1.0.6-alpha +version = 1.0.7-alpha author = Tiara Rodney author_email = t.rodney@victoryk.it description = a versatile HTTP pastebin diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 3a4cb21..ca5aeff 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -144,6 +144,7 @@ from configparser import ConfigParser from ast import literal_eval from io import StringIO from os import environ +from importlib.resources import path as pkg_resource_path from connexion import FlaskApp from connexion.resolver import RestyResolver @@ -301,13 +302,15 @@ def get_flask_app( options = {"swagger_ui": server_config.swagger_ui} - application = FlaskApp(__name__, specification_dir='schema/') + #context manager returns a pathlib.Path object + with pkg_resource_path('httpaste.schema', 'httpaste.openapi.json') as path: + application = FlaskApp(__name__, specification_dir=path.parent) - application.add_api( - 'httpaste.openapi.json', - options=options, - resolver=RestyResolver('httpaste.controller') - ) + application.add_api( + path.name, + options=options, + resolver=RestyResolver('httpaste.controller') + ) for err_cls in [ BadRequestError, From e330bbf70a67ee62cfeca77a8d3a77800a2220a1 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 16:47:47 +0200 Subject: [PATCH 18/24] fix(init): add custom context manager for pkg resources chore: upgrade version --- setup.cfg | 2 +- src/httpaste/__init__.py | 6 +++--- src/httpaste/helper/common.py | 32 +++++++++++++++++++++++++++++++- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index 0382db4..d05333e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = httpaste-victorykit -version = 1.0.7-alpha +version = 1.0.8-alpha author = Tiara Rodney author_email = t.rodney@victoryk.it description = a versatile HTTP pastebin diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index ca5aeff..999721b 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -144,14 +144,13 @@ from configparser import ConfigParser from ast import literal_eval from io import StringIO from os import environ -from importlib.resources import path as pkg_resource_path from connexion import FlaskApp from connexion.resolver import RestyResolver from httpaste.model import Backend from httpaste.backend import get_backend_map -from httpaste.helper.common import generate_random_string +from httpaste.helper.common import (generate_random_string, tmp_pkg_resource_text_path) from httpaste.helper.http import ( BadRequestError, ForbiddenError, @@ -303,7 +302,8 @@ def get_flask_app( options = {"swagger_ui": server_config.swagger_ui} #context manager returns a pathlib.Path object - with pkg_resource_path('httpaste.schema', 'httpaste.openapi.json') as path: + with tmp_pkg_resource_text_path('httpaste.schema', 'httpaste.openapi.json') as path: + application = FlaskApp(__name__, specification_dir=path.parent) application.add_api( diff --git a/src/httpaste/helper/common.py b/src/httpaste/helper/common.py index 92cc6d7..8edacd1 100644 --- a/src/httpaste/helper/common.py +++ b/src/httpaste/helper/common.py @@ -1,6 +1,11 @@ from random import choice from base64 import b64decode from urllib.parse import urljoin +from importlib.resources import read_text +from tempfile import mkdtemp +from pathlib import Path +from contextlib import contextmanager + class DecodeError(Exception): """ @@ -29,4 +34,29 @@ def decode(data: str, encoding: str) -> bytes: def join_url(base:str, url: str) -> str: - return urljoin(base, url, True) \ No newline at end of file + return urljoin(base, url, True) + + +@contextmanager +def tmp_pkg_resource_text_path(package:str, resource:str) -> Path: + """context manager for accessing package resources from a real path + + this applies to the circumstance of the package living inside of an + egg and therefore is unable to provide real existing paths to any + module that may require it. + + :param package: dot seperated package name + :param resource: basename of resource inside package + + :returns: a Path-like object + """ + data = read_text(package, resource) + tmp_dirname = mkdtemp() + tmp_dirpath = Path(tmp_dirname) + tmp_file = tmp_dirpath.joinpath(resource) + tmp_file.write_text(data) + try: + yield tmp_file + finally: + tmp_file.unlink() + tmp_dirpath.rmdir() \ No newline at end of file From ecd7d03a20e5f9cd9dc57835d053bcaa93b0cd60 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 17:11:15 +0200 Subject: [PATCH 19/24] fix(init): again, distutils... is confusing non-script files should now be, hopefully, included in bdist and sdist chore: upgrade version --- MANIFEST.in | 2 ++ setup.cfg | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..75d6607 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include src/httpaste/schema *.json +recursive-incude src/httpaste/backend *.sql diff --git a/setup.cfg b/setup.cfg index d05333e..915d670 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = httpaste-victorykit -version = 1.0.8-alpha +version = 1.0.9-alpha author = Tiara Rodney author_email = t.rodney@victoryk.it description = a versatile HTTP pastebin @@ -38,6 +38,3 @@ console_scripts = [options.packages.find] where = src -[options.package_data] -openapi_schema = src/httpaste/schema/httpaste.openapi.json -sql_schema = *.sql From a4116832e7a7819358e05eac3bb71d91fe77c6c9 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 8 Apr 2022 20:49:39 +0200 Subject: [PATCH 20/24] fix(build-toolchain): migrate versioning to setuptools_scm versioning of package now handled through setuptools_scm. Git tag therefore takes precedence and manually setting a version in setup.cfg is redundant. In addition, setuptools_scm handles proper inclusion of sdist non-python files and simplifies bdist packaging. --- .gitignore | 4 +++- MANIFEST.in | 2 -- pyproject.toml | 7 +++++-- setup.cfg | 5 ++++- 4 files changed, 12 insertions(+), 6 deletions(-) delete mode 100644 MANIFEST.in diff --git a/.gitignore b/.gitignore index 0ec6dc9..e2bd56f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,6 @@ **/__pycache__/ .DS_Store .coverage -/*.md \ No newline at end of file +/*.md +/.eggs/ +/devel/ \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 75d6607..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -recursive-include src/httpaste/schema *.json -recursive-incude src/httpaste/backend *.sql diff --git a/pyproject.toml b/pyproject.toml index 9111d02..cb91714 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,8 @@ [build-system] requires = [ "setuptools", - "wheel" + "wheel", + "setuptools-scm[toml]" ] build-backend = "setuptools.build_meta" @@ -9,4 +10,6 @@ build-backend = "setuptools.build_meta" max_line_length = 80 aggressive = 3 recursive = true -in-place = true \ No newline at end of file +in-place = true + +[tool.setuptools_scm] diff --git a/setup.cfg b/setup.cfg index 915d670..465a26b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,5 @@ [metadata] name = httpaste-victorykit -version = 1.0.9-alpha author = Tiara Rodney author_email = t.rodney@victoryk.it description = a versatile HTTP pastebin @@ -38,3 +37,7 @@ console_scripts = [options.packages.find] where = src +[options.package_data] +* = + *.json + *.sql \ No newline at end of file From c8dffbbdf893961a988421bd122089dcae0d1268 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 8 Apr 2022 21:01:33 +0200 Subject: [PATCH 21/24] refactor(__init__): reset context manager for flask app with proper bdist setup, it shouldn't be necessary to wrap path of importlib, since the context manager will handle egg extraction itself. Hopefully... --- src/httpaste/__init__.py | 3 ++- src/httpaste/helper/common.py | 26 -------------------------- 2 files changed, 2 insertions(+), 27 deletions(-) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 999721b..92c55a5 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -144,6 +144,7 @@ from configparser import ConfigParser from ast import literal_eval from io import StringIO from os import environ +from importlib.resources import path as resource_path from connexion import FlaskApp from connexion.resolver import RestyResolver @@ -302,7 +303,7 @@ def get_flask_app( options = {"swagger_ui": server_config.swagger_ui} #context manager returns a pathlib.Path object - with tmp_pkg_resource_text_path('httpaste.schema', 'httpaste.openapi.json') as path: + with resource_path('httpaste.schema', 'httpaste.openapi.json') as path: application = FlaskApp(__name__, specification_dir=path.parent) diff --git a/src/httpaste/helper/common.py b/src/httpaste/helper/common.py index 8edacd1..2e6d1c7 100644 --- a/src/httpaste/helper/common.py +++ b/src/httpaste/helper/common.py @@ -1,7 +1,6 @@ from random import choice from base64 import b64decode from urllib.parse import urljoin -from importlib.resources import read_text from tempfile import mkdtemp from pathlib import Path from contextlib import contextmanager @@ -35,28 +34,3 @@ def decode(data: str, encoding: str) -> bytes: def join_url(base:str, url: str) -> str: return urljoin(base, url, True) - - -@contextmanager -def tmp_pkg_resource_text_path(package:str, resource:str) -> Path: - """context manager for accessing package resources from a real path - - this applies to the circumstance of the package living inside of an - egg and therefore is unable to provide real existing paths to any - module that may require it. - - :param package: dot seperated package name - :param resource: basename of resource inside package - - :returns: a Path-like object - """ - data = read_text(package, resource) - tmp_dirname = mkdtemp() - tmp_dirpath = Path(tmp_dirname) - tmp_file = tmp_dirpath.joinpath(resource) - tmp_file.write_text(data) - try: - yield tmp_file - finally: - tmp_file.unlink() - tmp_dirpath.rmdir() \ No newline at end of file From 2762d6d67ff8a5984178aafd332f0fb1b369815c Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 8 Apr 2022 21:03:35 +0200 Subject: [PATCH 22/24] docs(guide/getting-started): init --- docs/guide/getting-started.rst | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 docs/guide/getting-started.rst diff --git a/docs/guide/getting-started.rst b/docs/guide/getting-started.rst new file mode 100644 index 0000000..431fa2b --- /dev/null +++ b/docs/guide/getting-started.rst @@ -0,0 +1,73 @@ +Getting Started +=============== + +Install +""""""" + +.. code-block:: shell + + $ python3 -m pip install httpaste-victorykit + $ httpaste --help + +.. note:: + httpaste is publicly available at `https://httpaste.it`_, and can be accessed + over the TOR network via `https://pastefao6mwyafs3cznoe2u2a6iizw5laulrznla3dytcnvaizte73yd.onion`_ aswell. Both are hosted on different servers of different service providers. + +Create Configuration +"""""""""""""""""""" + +.. code-block:: shell + + $ httpaste default-config --dump myconfig.ini + +.. note:: + The default configuration creates an in-memory SQLite backend, which is not + suitable for WWW deployments. Visit `backend`, for more + information on configuring the backend. + + +Run a Local Evaluation Server +""""""""""""""""""""""""""""" + +.. code-block:: shell + + $ httpaste standalone --config myconfig.ini --port 8080 + + +Publish a Private Paste +""""""""""""""""""""""" + +.. code-block:: shell + + $ echo 'My first private paste' | curl -F 'data=<-' -u myusername:mypassword http://localhost:8080/paste/private + http://localhost:8080/paste/private/UALUA9 + +.. note:: + If the user does not exist, they will be created upon authentication. + + +Retrieve a Private Paste +"""""""""""""""""""""""" + +.. code-block:: shell + + $ curl -u myusername:mypassword http://localhost:8080/paste/private/UALUA9 + My first private paste + + +Publish a Public Paste +"""""""""""""""""""""" + +.. code-block:: shell + + $ echo 'My first public paste' | curl -F 'data=<-' http://localhost:8080/paste/public + http://localhost:8080/paste/public/X4L39J + + +Retrieve a Public Paste +"""""""""""""""""""""""" + +.. code-block:: shell + + $ curl http://localhost:8080/paste/public/X4L39J + My first public paste \ No newline at end of file From f3a3e9516387fc8f8c23d32c2c29418b341c89a3 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 8 Apr 2022 21:25:36 +0200 Subject: [PATCH 23/24] fix(init): remove non-existing common import --- src/httpaste/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 92c55a5..14f2385 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -151,7 +151,7 @@ from connexion.resolver import RestyResolver from httpaste.model import Backend from httpaste.backend import get_backend_map -from httpaste.helper.common import (generate_random_string, tmp_pkg_resource_text_path) +from httpaste.helper.common import generate_random_string from httpaste.helper.http import ( BadRequestError, ForbiddenError, From 89c1a67a47d6d797538abd73774a7e86d7faeb57 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 9 Apr 2022 04:16:52 +0200 Subject: [PATCH 24/24] docs: sound less like a smart-ass sales person --- README.md | 46 +++++++++++++--------- docs/README.rst | 53 ++++++++++++++++--------- docs/guide/get-started.rst | 70 ---------------------------------- docs/guide/getting-started.rst | 3 -- docs/index.rst | 2 +- 5 files changed, 65 insertions(+), 109 deletions(-) delete mode 100644 docs/guide/get-started.rst diff --git a/README.md b/README.md index 030d58b..8be40cf 100644 --- a/README.md +++ b/README.md @@ -2,28 +2,40 @@ ![](docs/_assets/images/favpng_parrot-royalty-free-cartoon.png) -httpaste is a pastebin application for easily pasting and retrieving data over -HTTP from shell environments and web browsers. It is inspired by [sprunge.us](http://sprunge.us) -and [ix.io](http://ix.io/), but focuses on extendability, advanced security, with little to -no trade-off to simplicity. It can be hosted through WSGI, CGI, Fast CGI, or -as a standalone evaluation server. It offers multiple storage backends, such as -a filesystem backend, SQLite backend, MySQL backend, or MongoDB backend. +**NOTE**: httpaste is publicly hosted at [httpaste.it](http://httpaste.it) and as a hidden Tor service ([https://paste77ubkwxy4fqezffsmthxdh3xerwi72tlsw2mch7ecjhw2xn7iyd.onion](https://paste77ubkwxy4fqezffsmthxdh3xerwi72tlsw2mch7ecjhw2xn7iyd.onion)). +Both services are to be considered evaluatory, as long as the source code +is in pre-release. Regarding voidance of pre-release status, see [Open Issues](https://victorykit.atlassian.net/issues/?jql=project%20%3D%20HTTPASTE%20AND%20fixVersion%20in%20(1.1.0-beta%2C%201.2.0-beta%2C%201.3.0)), for more information. -All pastes are being encrypted on the fly and can only be retrieved by an -authorized user, either through knowing the paste id of a public paste, or -having authentication credentials, as well as the paste id of a private paste. -This makes httpaste ideal as a pastebin for sensitive environments such as the -Tor network. Authentication credentials are created on-the-fly and don’t require a sign-up process. +This program offers an HTTP interface for storing public and private data +(a.k.a. pastes), commonly referred to as a pastebin application. It is inspired by [sprunge.us](http://sprunge.us) and [ix.io](http://ix.io/). It can be hosted through WSGI, CGI, Fast +CGI, or as a standalone evaluation server. It offers multiple storage backends, +such as a filesystem backend, SQLite backend, or MySQL backend. -httpaste supports output formatting for syntax highlighting (powered by +Public data can be accessed through an URL, where as private pastes +additionally require HTTP basic authentication. Creation of authentication +credentials happens on the fly, there is no sign-up process. Public pastes can +only be accessed by knowing their paste ids, they are not listed on any index, +since it isn’t technically possible (by design). + +All pastes are symetrically encrypted server-side with an HMAC derived key and +SHA-256 hashing, a server-side salt and a randomly generated password. Public +paste’s passwords are derived from their ids. Private paste’s passwords are +randomly generated and stored inside a symetrically encrypted personal +database, with the encryption key also being derived through the same HMAC +mechanism, where the HTTP basic authentication credentials act as the master +password. + +Paste ids, usernames, and any other identifiable attributes are only stored +inside storage backends as keyed and salted BLAKE2 hashes. + +The program supports output formatting for syntax highlighting (powered by [pygments](https://pygments.org/)), as well as MIME type output manipulation, and input encoding. -Therefore httpaste can server as an anonymous object storage for small data. +The program can therefore serve as a minimalist, anonymous object storage for +small data. -Minute-based and ‘burn-after-read’ paste expiration are supported. +Minute-based and ‘burn-after-read’ paste expiration are also supported. -httpaste focuses on security through cryptography, making it a computationally intensive application. - -# Get Started +# Getting Started ## Install diff --git a/docs/README.rst b/docs/README.rst index 0bf034c..96f45f3 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -9,28 +9,41 @@ httpaste - versatile HTTP pastebin .. image:: _assets/images/favpng_parrot-royalty-free-cartoon.png -httpaste is a pastebin application for easily pasting and retrieving data over -HTTP from shell environments and web browsers. It is inspired by `sprunge.us`_ -and `ix.io`_, but focuses on extendability, advanced security, with little to -no trade-off to simplicity. It can be hosted through WSGI, CGI, Fast CGI, or -as a standalone evaluation server. It offers multiple storage backends, such as -a filesystem backend, SQLite backend, MySQL backend, or MongoDB backend. +.. note:: + httpaste is publicly hosted at `httpaste.it`_ and as a hidden Tor service (``_). + Both services are to be considered evaluatory, as long as the source code + is in pre-release. Regarding voidance of pre-release status, see `Open Issues`_, for more information. -All pastes are being encrypted on the fly and can only be retrieved by an -authorized user, either through knowing the paste id of a public paste, or -having authentication credentials, as well as the paste id of a private paste. -This makes httpaste ideal as a pastebin for sensitive environments such as the -Tor network. Authentication credentials are created on-the-fly and don't require a sign-up process. +This program offers an HTTP interface for storing public and private data +(a.k.a. pastes), commonly referred to as a pastebin application. It is inspired by `sprunge.us`_ and `ix.io`_. It can be hosted through WSGI, CGI, Fast +CGI, or as a standalone evaluation server. It offers multiple storage backends, +such as a filesystem backend, SQLite backend, or MySQL backend. -httpaste supports output formatting for syntax highlighting (powered by +Public data can be accessed through an URL, where as private pastes +additionally require HTTP basic authentication. Creation of authentication +credentials happens on the fly, there is no sign-up process. Public pastes can +only be accessed by knowing their paste ids, they are not listed on any index, +since it isn't technically possible (by design). + +All pastes are symetrically encrypted server-side with an HMAC derived key and +SHA-256 hashing, a server-side salt and a randomly generated password. Public +paste's passwords are derived from their ids. Private paste's passwords are +randomly generated and stored inside a symetrically encrypted personal +database, with the encryption key also being derived through the same HMAC +mechanism, where the HTTP basic authentication credentials act as the master +password. + +Paste ids, usernames, and any other identifiable attributes are only stored +inside storage backends as keyed and salted BLAKE2 hashes. + +The program supports output formatting for syntax highlighting (powered by `pygments`_), as well as MIME type output manipulation, and input encoding. -Therefore httpaste can server as an anonymous object storage for small data. +The program can therefore serve as a minimalist, anonymous object storage for +small data. -Minute-based and 'burn-after-read' paste expiration are supported. +Minute-based and 'burn-after-read' paste expiration are also supported. -httpaste focuses on security through cryptography, making it a computationally intensive application. - -.. include:: guide/get-started.rst +.. include:: guide/getting-started.rst Documentation ------------- @@ -69,4 +82,8 @@ This program uses licensed third-party software. .. _ix.io: http://ix.io/ .. _sprunge.us: http://sprunge.us .. _pygments: https://pygments.org/ -.. _icon: https://favpng.com/png_view/parrot-parrot-royalty-free-cartoon-png/gps7HM42 \ No newline at end of file +.. _icon: https://favpng.com/png_view/parrot-parrot-royalty-free-cartoon-png/gps7HM42 + +.. _Open Issues: https://victorykit.atlassian.net/issues/?jql=project%20%3D%20HTTPASTE%20AND%20fixVersion%20in%20(1.1.0-beta%2C%201.2.0-beta%2C%201.3.0) + +.. _httpaste.it: http://httpaste.it \ No newline at end of file diff --git a/docs/guide/get-started.rst b/docs/guide/get-started.rst deleted file mode 100644 index 0bbeedc..0000000 --- a/docs/guide/get-started.rst +++ /dev/null @@ -1,70 +0,0 @@ -Get Started -=========== - -Install -""""""" - -.. code-block:: shell - - $ python3 -m pip install httpaste-victorykit - $ httpaste --help - - -Create Configuration -"""""""""""""""""""" - -.. code-block:: shell - - $ httpaste default-config --dump myconfig.ini - -.. note:: - The default configuration creates an in-memory SQLite backend, which is not - suitable for WWW deployments. Visit `backend`, for more - information on configuring the backend. - - -Run a Local Evaluation Server -""""""""""""""""""""""""""""" - -.. code-block:: shell - - $ httpaste standalone --config myconfig.ini --port 8080 - - -Publish a Private Paste -""""""""""""""""""""""" - -.. code-block:: shell - - $ echo 'My first private paste' | curl -F 'data=<-' -u myusername:mypassword http://localhost:8080/paste/private - http://localhost:8080/paste/private/UALUA9 - -.. note:: - If the user does not exist, they will be created upon authentication. - - -Retrieve a Private Paste -"""""""""""""""""""""""" - -.. code-block:: shell - - $ curl -u myusername:mypassword http://localhost:8080/paste/private/UALUA9 - My first private paste - - -Publish a Public Paste -"""""""""""""""""""""" - -.. code-block:: shell - - $ echo 'My first public paste' | curl -F 'data=<-' http://localhost:8080/paste/public - http://localhost:8080/paste/public/X4L39J - - -Retrieve a Public Paste -"""""""""""""""""""""""" - -.. code-block:: shell - - $ curl http://localhost:8080/paste/public/X4L39J - My first public paste \ No newline at end of file diff --git a/docs/guide/getting-started.rst b/docs/guide/getting-started.rst index 431fa2b..048f06a 100644 --- a/docs/guide/getting-started.rst +++ b/docs/guide/getting-started.rst @@ -9,9 +9,6 @@ Install $ python3 -m pip install httpaste-victorykit $ httpaste --help -.. note:: - httpaste is publicly available at `https://httpaste.it`_, and can be accessed - over the TOR network via `https://pastefao6mwyafs3cznoe2u2a6iizw5laulrznla3dytcnvaizte73yd.onion`_ aswell. Both are hosted on different servers of different service providers. Create Configuration """""""""""""""""""" diff --git a/docs/index.rst b/docs/index.rst index bb9e5a0..6b0a60b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ :maxdepth: 1 :caption: Guides - guide/get-started + guide/getting-started guide/advanced-usage guide/backend guide/cli