From 23d5128ea765c8701b05e9a6c0e3145c4d3873b2 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 00:29:44 +0200 Subject: [PATCH 01/55] refactor(backend/sqlite): normalize functions --- src/httpaste/backend/sqlite/paste.py | 77 +++++++++++++++++----------- src/httpaste/backend/sqlite/user.py | 54 +++++++++++-------- 2 files changed, 82 insertions(+), 49 deletions(-) diff --git a/src/httpaste/backend/sqlite/paste.py b/src/httpaste/backend/sqlite/paste.py index bb0eb17..167ca83 100644 --- a/src/httpaste/backend/sqlite/paste.py +++ b/src/httpaste/backend/sqlite/paste.py @@ -9,73 +9,92 @@ def load(proto: object, connection: Connection, model_class: type): """load a paste """ - cur = connection.cursor() + cursor = connection.cursor() - cur.execute( - 'SELECT pid, data, data_hash, sub, expiration, encoding FROM pastes WHERE pid=?', - (proto.pid, - )) + statement = '''SELECT pid, data, data_hash, sub, expiration, encoding + FROM pastes + WHERE pid=?''' - result = cur.fetchone() + cursor.execute(statement, (proto.pid,)) - if result: + row = cursor.fetchone() + + if row is not None: return model_class( - result['pid'], - result['sub'], - result['data'], - result['data_hash'], - result['expiration'], - result['encoding']) + row['pid'], + row['sub'], + row['data'], + row['data_hash'], + row['expiration'], + row['encoding']) return None -def dump(model: object, connection: Connection): +def dump(model: object, connection: Connection) -> None: """dump a paste """ - cur = connection.cursor() + cursor = connection.cursor() - cur.execute( - '''INSERT INTO pastes (pid, data, data_hash, sub, expiration, encoding) - VALUES (?,?,?,?,?,?)''', - (model.pid, + statement = '''INSERT INTO pastes + (pid, data, data_hash, sub, expiration, encoding) + VALUES (?,?,?,?,?,?)''' + + values = (model.pid, model.data, model.data_hash, model.sub, model.expiration, - model.encoding)) + model.encoding) + + cursor.execute(statement, values) connection.commit() + return None -def delete(proto: object, connection: Connection) -> bool: - cur = connection.cursor() +def delete(proto: object, connection: Connection) -> None: - cur.execute('''DELETE FROM pastes WHERE pid=?''', (proto.pid,)) + cursor = connection.cursor() + + cursor.execute('''DELETE FROM pastes WHERE pid=?''', (proto.pid,)) connection.commit() + return None + def init(connection: Connection): - cur = connection.cursor() + cursor = connection.cursor() with open(path.join(path.dirname(__file__), 'paste.sql'), 'r') as fh: - cur.execute(fh.read()) + statement = fh.read() + + cursor.execute(statement) connection.commit() -def sanitize(connection: Connection, model_class: type) -> bool: +def sanitize(connection: Connection, model_class: type) -> int: - cur = connection.cursor() + cursor = connection.cursor() - cur.execute('''SELECT pid FROM pastes WHERE expiration < ? AND expiration > 0''', (int(time()),)) + statement = '''SELECT pid FROM pastes + WHERE expiration < ? AND expiration > 0''' + + cursor.execute(statement, (int(time()),)) + + srow_count = 0 for row in cur.fetchall(): - delete(model_class(row['pid'])) \ No newline at end of file + delete(model_class(row['pid'])) + + srow_count += 1 + + return srow_count \ No newline at end of file diff --git a/src/httpaste/backend/sqlite/user.py b/src/httpaste/backend/sqlite/user.py index 11fc53e..f6d82fa 100644 --- a/src/httpaste/backend/sqlite/user.py +++ b/src/httpaste/backend/sqlite/user.py @@ -2,59 +2,73 @@ """ from os import path from sqlite3 import Connection -from httpaste.model import User -def load(proto: User, connection: Connection): +def load(proto: object, connection: Connection, model_class: type): """load a user """ - cur = connection.cursor() + cursor = connection.cursor() - cur.execute( - 'SELECT sub, key_hash, paste_index FROM users WHERE sub=?', (proto.sub,)) + statement = '''SELECT sub, key_hash, paste_index + FROM users + WHERE sub=?''' - result = cur.fetchone() + cursor.execute(statement, (proto.sub,)) - if result: + row = cursor.fetchone() - return User(result['sub'], result['key_hash'], result['paste_index']) + if row is not None: + + return model_class(result['sub'], result['key_hash'], + result['paste_index']) return None -def dump(model: User, connection: Connection): +def dump(model: object, connection: Connection) -> None: """dump a user """ - cur = connection.cursor() + cursor = connection.cursor() - cur.execute('''INSERT OR REPLACE INTO users (sub, key_hash, paste_index) - VALUES (?,?,?)''', (model.sub, model.key_hash, model.index)) + statement = '''INSERT OR REPLACE INTO users + (sub, key_hash, paste_index) + VALUES (?,?,?)''' + + cursor.execute(statement, (model.sub, model.key_hash, model.index)) connection.commit() + return None -def delete(proto: object, connection: Connection) -> bool: - cur = connection.cursor() +def delete(proto: object, connection: Connection) -> None: - cur.execute('''DELETE FROM users WHERE sub=?''', (proto.sub,)) + cursor = connection.cursor() + + cursor.execute('''DELETE FROM users WHERE sub=?''', (proto.sub,)) connection.commit() + return None -def init(connection: Connection): - cur = connection.cursor() +def init(connection: Connection) -> None: + + cursor = connection.cursor() with open(path.join(path.dirname(__file__), 'user.sql'), 'r') as fh: - cur.execute(fh.read()) + statement = fh.read() + + cursor.execute(statement) connection.commit() + return None -def sanitize(connection: Connection, model_class) -> bool: - return None \ No newline at end of file +def sanitize(connection: Connection, model_class) -> int: + + return 0 \ No newline at end of file From 5e25606880faa2ed367b72ea373240d48ea2965c Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 21:32:15 +0200 Subject: [PATCH 02/55] feat(backend/mysql): initialize mysql backend created prototype for backend/mysql module feat(backend/mysql): implement interface tested with MariaDB docs(backend/sql): initialize docs --- docs/guide/backend.rst | 6 ++ src/httpaste/backend/__init__.py | 20 ++++ src/httpaste/backend/mysql/__init__.py | 120 ++++++++++++++++++++++++ src/httpaste/backend/mysql/paste.py | 122 +++++++++++++++++++++++++ src/httpaste/backend/mysql/paste.sql | 9 ++ src/httpaste/backend/mysql/user.py | 101 ++++++++++++++++++++ src/httpaste/backend/mysql/user.sql | 6 ++ 7 files changed, 384 insertions(+) create mode 100644 src/httpaste/backend/mysql/__init__.py create mode 100644 src/httpaste/backend/mysql/paste.py create mode 100644 src/httpaste/backend/mysql/paste.sql create mode 100644 src/httpaste/backend/mysql/user.py create mode 100644 src/httpaste/backend/mysql/user.sql diff --git a/docs/guide/backend.rst b/docs/guide/backend.rst index bc2f9c3..a09c3e9 100644 --- a/docs/guide/backend.rst +++ b/docs/guide/backend.rst @@ -13,4 +13,10 @@ Filesystem ---------- .. autoclass:: httpaste.backend.file.Parameters + :members: + +MySQL +----- + +.. autoclass:: httpaste.backend.mysql.Parameters :members: \ No newline at end of file diff --git a/src/httpaste/backend/__init__.py b/src/httpaste/backend/__init__.py index 48978a0..60800b1 100644 --- a/src/httpaste/backend/__init__.py +++ b/src/httpaste/backend/__init__.py @@ -11,9 +11,13 @@ from .sqlite import Parameters as SqliteParameters from .sqlite import User as SqliteUser from .sqlite import Paste as SqlitePaste from .sqlite import get_connection as get_sqlite_connection +from .mysql import get_connection as get_mysql_connection from .file import Parameters as FileParameters from .file import User as FileUser from .file import Paste as FilePaste +from .mysql import Parameters as MySQLParameters +from .mysql import User as MySQLUser +from .mysql import Paste as MySQLPaste class SQLite(Backend): @@ -46,6 +50,22 @@ class File(Backend): self.paste = FilePaste(parameters, Paste, PasteDataSchema) +class MySQL(Backend): + """MySQL backend interface + """ + + parameter_class = MySQLParameters + user: MySQLUser + paste: MySQLPaste + + def __init__(self, parameters: MySQLParameters): + + parameters = MySQLParameters(*parameters[1:], get_mysql_connection(parameters)) + + self.user = MySQLUser(parameters, User) + self.paste = MySQLPaste(parameters, Paste) + + def get_backend_map() -> Dict[str, Tuple[type, type]]: """get a map of backend ids and their classes """ diff --git a/src/httpaste/backend/mysql/__init__.py b/src/httpaste/backend/mysql/__init__.py new file mode 100644 index 0000000..2a21162 --- /dev/null +++ b/src/httpaste/backend/mysql/__init__.py @@ -0,0 +1,120 @@ +"""MySQL backend +""" +try: + from mysql.connector import connect + from mysql.connector.connection import MySQLConnection +except ImportError as e: + raise ImportError(' '.join(( + '\'mysql-connector-python\' is not installed.', + 'Install it by running', + '\'python3 -m pip install mysql-connector-python\'.' + ))) from e + +from typing import NamedTuple, Optional + +from . import user +from . import paste + + +class Parameters(NamedTuple): + """MySQL parameters + """ + + #: user name + user: str + #: user password + password: str + #: hostname or IP address + host: str + #: database identifier + database: str + #: a mysql.connection.MySQLConnection object (does not apply to config) + connection: Optional[MySQLConnection] = None + + +class User(object): + """MySQL user model backend + """ + + connection: MySQLConnection + + def __init__(self, parameters: Parameters, model_class: type) -> None: + + self.model_class = model_class + + self.connection = get_connection(parameters) + + def load(self, proto: object) -> object: + + return user.load(proto, self.connection, self.model_class) + + def dump(self, model: object) -> None: + + return user.dump(model, self.connection) + + def delete(self, proto: object) -> None: + + return user.delete(proto, self.connection) + + def init(self) -> None: + + return user.init(self.connection) + + def sanitize(self) -> None: + + return user.sanitize(self.connection, self.model_class) + + +class Paste(object): + """MySQL paste model backend + """ + + connection: MySQLConnection + + def __init__(self, parameters: Parameters, model_class: type) -> None: + + self.model_class = model_class + + self.connection = get_connection(parameters) + + def load(self, proto: object) -> object: + + return paste.load(proto, self.connection, self.model_class) + + def dump(self, model: object) -> None: + + return paste.dump(model, self.connection) + + def delete(self, proto: object) -> None: + + return paste.delete(proto, self.connection) + + def init(self) -> None: + + return paste.init(self.connection) + + def sanitize(self) -> None: + + return paste.sanitize(self.connection, self.model_class) + + +def get_connection(parameters: Parameters) -> MySQLConnection: + """get a mysql.connection.MySQLConnection object + """ + + if parameters.connection is not None: + + return parameters.connection + + connection = connect(user=parameters.user, password=parameters.password, + host=parameters.host, + database=parameters.database) + + return connection + + +__all__ = [ + Parameters, + User, + Paste +] \ No newline at end of file diff --git a/src/httpaste/backend/mysql/paste.py b/src/httpaste/backend/mysql/paste.py new file mode 100644 index 0000000..5e4057f --- /dev/null +++ b/src/httpaste/backend/mysql/paste.py @@ -0,0 +1,122 @@ +from os import path +from time import time + +try: + from mysql.connector.connection import MySQLConnection +except ImportError as e: + raise ImportError(' '.join(( + '\'mysql-connector-python\' is not installed.', + 'Install it by running', + '\'python3 -m pip install mysql-connector-python\'.' + ))) from e + + +def load(proto:object, connection: MySQLConnection, model_class: type): + """load a paste model + + :param model: model prototype + :param connection: mysql connector connection object + :param model_class: model class + """ + + cursor = connection.cursor(dictionary=True) + + statement = '''SELECT pid, data, data_hash, sub, expiration, encoding + FROM httpaste_pastes + WHERE pid=%s''' + + cursor.execute(statement, (proto.pid,)) + + row = cursor.fetchone() + + if row is not None: + + return model_class( + row['pid'], + row['sub'], + row['data'], + row['data_hash'], + row['expiration'], + row['encoding']) + + return None + + +def dump(model:object, connection: MySQLConnection): + """dump a paste model + + :param model: model object + :param connection: mysql connector connection object + :param model_class: model class + """ + + cursor = connection.cursor() + + statement = '''REPLACE INTO httpaste_pastes + (pid, data, data_hash, sub, expiration, encoding) + VALUES (%s, %s, %s, %s, %s, %s)''' + + cursor.execute(statement, (model.pid, model.data, model.data_hash, + model.sub, model.expiration, model.encoding)) + + connection.commit() + + return None + + +def delete(proto: object, connection: MySQLConnection): + """delete a paste model + + :param model: model prototype + :param connection: mysql connector connection object + :param model_class: model class + """ + + cursor = connection.cursor() + + statement = '''DELETE FROM httpaste_pastes + WHERE pid=%s''' + + cursor.execute(statement, (proto.pid,)) + + connection.commit() + + return None + + +def init(connection: MySQLConnection): + """initialize paste model table + + :param connection: mysql connector connection object + """ + + cursor = connection.cursor() + + with open(path.join(path.dirname(__file__), 'paste.sql'), 'r') as fh: + + cursor.execute(fh.read()) + + connection.commit() + + return None + + +def sanitize(connection: MySQLConnection, model_class:type): + """sanitize paste model table + + :param connection: mysql connector connection object + """ + + cursor = connection.cursor(dictionary=True) + + statement = '''SELECT pid + FROM httpaste_pastes + WHERE expiration < %s AND expiration > 0''' + + cursor.execute(statement, (time(),)) + + for row in cursor.fetchall(): + + delete(model_class(row['pid'])) + + return None \ No newline at end of file diff --git a/src/httpaste/backend/mysql/paste.sql b/src/httpaste/backend/mysql/paste.sql new file mode 100644 index 0000000..d4c366e --- /dev/null +++ b/src/httpaste/backend/mysql/paste.sql @@ -0,0 +1,9 @@ +CREATE TABLE `httpaste_pastes` ( + `pid` blob NOT NULL, + `data` longblob NOT NULL, + `data_hash` blob NOT NULL, + `sub` blob DEFAULT NULL, + `expiration` int(16) NOT NULL, + `encoding` tinytext DEFAULT NULL, + PRIMARY KEY (`pid`(128)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/src/httpaste/backend/mysql/user.py b/src/httpaste/backend/mysql/user.py new file mode 100644 index 0000000..7efebfc --- /dev/null +++ b/src/httpaste/backend/mysql/user.py @@ -0,0 +1,101 @@ +from os import path +try: + from mysql.connector.connection import MySQLConnection +except ImportError as e: + raise ImportError(' '.join(( + '\'mysql-connector-python\' is not installed.', + 'Install it by running', + '\'python3 -m pip install mysql-connector-python\'.' + ))) from e + + +def load(proto:object, connection: MySQLConnection, model_class: type): + """load a user model + + :param model: model prototype + :param connection: mysql connector connection object + :param model_class: model class + """ + + cursor = connection.cursor(dictionary=True) + + statement = '''SELECT sub, key_hash, paste_index + FROM httpaste_users + WHERE sub=%s''' + + cursor.execute(statement, (proto.sub,)) + + row = cursor.fetchone() + + if row is not None: + + return model_class(row['sub'], row['key_hash'], row['paste_index']) + + return None + + +def dump(model:object, connection: MySQLConnection): + """dump a user model + + :param model: model object + :param connection: mysql connector connection object + :param model_class: model class + """ + + cursor = connection.cursor() + + statement = '''REPLACE INTO httpaste_users + (sub, key_hash, paste_index) + VALUES (%s, %s, %s)''' + + cursor.execute(statement, (model.sub, model.key_hash, model.index)) + + connection.commit() + + return None + + +def delete(proto: object, connection: MySQLConnection): + """delete a user model + + :param model: model prototype + :param connection: mysql connector connection object + :param model_class: model class + """ + + cursor = connection.cursor() + + statement = '''DELETE FROM httpaste_users + WHERE sub=%s''' + + cursor.execute(statement, (proto.sub,)) + + connection.commit() + + return None + + +def init(connection: MySQLConnection): + """initialize user model table + + :param connection: mysql connector connection object + """ + + cursor = connection.cursor() + + with open(path.join(path.dirname(__file__), 'user.sql'), 'r') as fh: + + cursor.execute(fh.read()) + + connection.commit() + + return None + + +def sanitize(connection: MySQLConnection, model_class: type): + """sanitize user model table + + :param connection: mysql connector connection object + """ + + return None \ No newline at end of file diff --git a/src/httpaste/backend/mysql/user.sql b/src/httpaste/backend/mysql/user.sql new file mode 100644 index 0000000..62045a9 --- /dev/null +++ b/src/httpaste/backend/mysql/user.sql @@ -0,0 +1,6 @@ +CREATE TABLE `httpaste_users` ( + `sub` blob NOT NULL, + `key_hash` blob NOT NULL, + `paste_index` blob NOT NULL, + PRIMARY KEY (`sub`(128)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file From cefbcf9318649da45620f082e744c7c79fbd1543 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 01:09:12 +0200 Subject: [PATCH 03/55] fix(backend/sqlite): remove faulty var from load() resolves HTTPASTE-31 --- src/httpaste/backend/sqlite/user.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/httpaste/backend/sqlite/user.py b/src/httpaste/backend/sqlite/user.py index f6d82fa..3c9d8b9 100644 --- a/src/httpaste/backend/sqlite/user.py +++ b/src/httpaste/backend/sqlite/user.py @@ -20,8 +20,7 @@ def load(proto: object, connection: Connection, model_class: type): if row is not None: - return model_class(result['sub'], result['key_hash'], - result['paste_index']) + return model_class(row['sub'], row['key_hash'], row['paste_index']) return None From 9c31f044cef0a0b9feeebe9ffd6fd63a814bb3aa Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 02:45:01 +0200 Subject: [PATCH 04/55] fix(backend/mysql): isolate third-party module imports currently there is a problem with loading the backends through the httpaste.Config class, since they aren't being lazily loaded, will have to rework this sometime later in the future. --- src/httpaste/backend/mysql/__init__.py | 70 +++++++++++++++----------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/src/httpaste/backend/mysql/__init__.py b/src/httpaste/backend/mysql/__init__.py index 2a21162..abda80c 100644 --- a/src/httpaste/backend/mysql/__init__.py +++ b/src/httpaste/backend/mysql/__init__.py @@ -1,19 +1,6 @@ """MySQL backend """ -try: - from mysql.connector import connect - from mysql.connector.connection import MySQLConnection -except ImportError as e: - raise ImportError(' '.join(( - '\'mysql-connector-python\' is not installed.', - 'Install it by running', - '\'python3 -m pip install mysql-connector-python\'.' - ))) from e - -from typing import NamedTuple, Optional - -from . import user -from . import paste +from typing import NamedTuple, Optional, Callable class Parameters(NamedTuple): @@ -29,65 +16,77 @@ class Parameters(NamedTuple): #: database identifier database: str #: a mysql.connection.MySQLConnection object (does not apply to config) - connection: Optional[MySQLConnection] = None + connection: Optional[object] = None class User(object): """MySQL user model backend """ - connection: MySQLConnection + connection: object def __init__(self, parameters: Parameters, model_class: type) -> None: + from . import user + + connect = get_mysql_connect_callee() + + self.interface = user + self.model_class = model_class self.connection = get_connection(parameters) def load(self, proto: object) -> object: - return user.load(proto, self.connection, self.model_class) + return self.interface.load(proto, self.connection, self.model_class) def dump(self, model: object) -> None: - return user.dump(model, self.connection) + return self.interface.dump(model, self.connection) def delete(self, proto: object) -> None: - return user.delete(proto, self.connection) + return self.interface.delete(proto, self.connection) def init(self) -> None: - return user.init(self.connection) + return self.interface.init(self.connection) def sanitize(self) -> None: - return user.sanitize(self.connection, self.model_class) + return self.interface.sanitize(self.connection, self.model_class) class Paste(object): """MySQL paste model backend """ - connection: MySQLConnection + connection: object def __init__(self, parameters: Parameters, model_class: type) -> None: + from . import paste + + connect = get_mysql_connect_callee() + + self.interface = paste + self.model_class = model_class - self.connection = get_connection(parameters) + self.connection = get_connection(parameters, connect) def load(self, proto: object) -> object: - return paste.load(proto, self.connection, self.model_class) + return self.interface.load(proto, self.connection, self.model_class) def dump(self, model: object) -> None: - return paste.dump(model, self.connection) + return self.interface.dump(model, self.connection) def delete(self, proto: object) -> None: - return paste.delete(proto, self.connection) + return self.interface.delete(proto, self.connection) def init(self) -> None: @@ -95,10 +94,25 @@ class Paste(object): def sanitize(self) -> None: - return paste.sanitize(self.connection, self.model_class) + return self.interface.sanitize(self.connection, self.model_class) -def get_connection(parameters: Parameters) -> MySQLConnection: +def get_mysql_connect_callee() -> object: + + try: + from mysql.connector import connect + from mysql.connector.connection import MySQLConnection + except ImportError as e: + raise ImportError(' '.join(( + '\'mysql-connector-python\' is not installed.', + 'Install it by running', + '\'python3 -m pip install mysql-connector-python\'.' + ))) from e + + return connect + + +def get_connection(parameters: Parameters, connect_callee: Callable) -> object: """get a mysql.connection.MySQLConnection object """ From 809ce6522ba0937c82737195d69ac00c3b2d965b Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 04:02:08 +0200 Subject: [PATCH 05/55] fix(backend/mysql): load files through importlib is required since package will be distributed as python egg --- src/httpaste/backend/mysql/paste.py | 4 +++- src/httpaste/backend/mysql/user.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/httpaste/backend/mysql/paste.py b/src/httpaste/backend/mysql/paste.py index 5e4057f..efa5f0e 100644 --- a/src/httpaste/backend/mysql/paste.py +++ b/src/httpaste/backend/mysql/paste.py @@ -1,5 +1,7 @@ from os import path from time import time +from importlib.resources import open_text + try: from mysql.connector.connection import MySQLConnection @@ -92,7 +94,7 @@ def init(connection: MySQLConnection): cursor = connection.cursor() - with open(path.join(path.dirname(__file__), 'paste.sql'), 'r') as fh: + with open_text('httpaste.backend.mysql', 'paste.sql') as fh: cursor.execute(fh.read()) diff --git a/src/httpaste/backend/mysql/user.py b/src/httpaste/backend/mysql/user.py index 7efebfc..a7148ff 100644 --- a/src/httpaste/backend/mysql/user.py +++ b/src/httpaste/backend/mysql/user.py @@ -1,4 +1,5 @@ from os import path +from importlib.resources import open_text try: from mysql.connector.connection import MySQLConnection except ImportError as e: @@ -83,7 +84,7 @@ def init(connection: MySQLConnection): cursor = connection.cursor() - with open(path.join(path.dirname(__file__), 'user.sql'), 'r') as fh: + with open_text('httpaste.backend.mysql', 'user.sql') as fh: cursor.execute(fh.read()) From d8ac419c180230e55948769cee2dfc1b793e706a Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 04:04:56 +0200 Subject: [PATCH 06/55] fix(backend/mysql): add missing var for get_connection() --- src/httpaste/backend/__init__.py | 2 +- src/httpaste/backend/mysql/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/httpaste/backend/__init__.py b/src/httpaste/backend/__init__.py index 60800b1..67f1c8c 100644 --- a/src/httpaste/backend/__init__.py +++ b/src/httpaste/backend/__init__.py @@ -60,7 +60,7 @@ class MySQL(Backend): def __init__(self, parameters: MySQLParameters): - parameters = MySQLParameters(*parameters[1:], get_mysql_connection(parameters)) + #parameters = MySQLParameters(*parameters[1:], get_mysql_connection(parameters)) self.user = MySQLUser(parameters, User) self.paste = MySQLPaste(parameters, Paste) diff --git a/src/httpaste/backend/mysql/__init__.py b/src/httpaste/backend/mysql/__init__.py index abda80c..e4b0b5c 100644 --- a/src/httpaste/backend/mysql/__init__.py +++ b/src/httpaste/backend/mysql/__init__.py @@ -35,7 +35,7 @@ class User(object): self.model_class = model_class - self.connection = get_connection(parameters) + self.connection = get_connection(parameters, connect) def load(self, proto: object) -> object: @@ -112,7 +112,7 @@ def get_mysql_connect_callee() -> object: return connect -def get_connection(parameters: Parameters, connect_callee: Callable) -> object: +def get_connection(parameters: Parameters, connect: Callable) -> object: """get a mysql.connection.MySQLConnection object """ From 3940b4cec7ce64e5585c586a0f9f686443831cb6 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 02:36:38 +0200 Subject: [PATCH 07/55] feat(docker): init Dockerfile --- .dockerignore | 1 + .gitignore | 2 +- Dockerfile | 21 +++++++++++++++++++++ tox.ini | 9 +++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) create mode 120000 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 0000000..3e4e48b --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index e2bd56f..be245d1 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ .coverage /*.md /.eggs/ -/devel/ \ No newline at end of file +/devel/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c8fd5dd --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.10-slim + +LABEL org.label-schema.schema-version="1.0" +LABEL org.label-schema.vendor="Tiara Rodney (victoryk.it)" +LABEL org.label-schema.name="victorykit/httpaste" +LABEL org.label-schema.description="a versatile HTTP pastebin" +LABEL org.label-schema.vcs-url="https://bitbucket.org/victorykit/docker-selenium-grid" +LABEL org.label-schema.docker.cmd="docker run {image-id} {httpaste-args}" +LABEL org.label-schema.version=$BUILD_VERSION +LABEL org.label-schema.build-date=$BUILD_DATE + +WORKDIR /usr/local/src/httpaste + +COPY . . + +RUN apt-get update && \ + apt-get install -y libffi-dev gcc && \ + python3 setup.py install && \ + apt-get remove -y libffi-dev gcc && apt-get autoremove -y && apt-get clean -y + +ENTRYPOINT ["httpaste"] diff --git a/tox.ini b/tox.ini index 28b7f1d..c1f0e88 100644 --- a/tox.ini +++ b/tox.ini @@ -35,6 +35,15 @@ deps = commands = python3 -m build {posargs} +[testenv:build-docker] +description = build docker image +passenv = + DOCKER_* +allowlist_externals = + docker + sh +commands = + docker image build -t victorykit/httpaste . [testenv:docs] description = build documentation From 9541cee98ad4bcac0439d602ecf5814fce24d61d Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 03:30:26 +0200 Subject: [PATCH 08/55] refactor(docker): remove entrypoint --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c8fd5dd..ebcf3aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,6 @@ COPY . . RUN apt-get update && \ apt-get install -y libffi-dev gcc && \ python3 setup.py install && \ - apt-get remove -y libffi-dev gcc && apt-get autoremove -y && apt-get clean -y + apt-get remove -y libffi-dev gcc && apt-get autoremove -y && apt-get clean -y -ENTRYPOINT ["httpaste"] +CMD ["httpaste", "--help"] From a9472d321caaff55d135d9c298e5188f2488346a Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 02:36:38 +0200 Subject: [PATCH 09/55] feat(docker): init Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ebcf3aa..9767ba9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,6 @@ COPY . . RUN apt-get update && \ apt-get install -y libffi-dev gcc && \ python3 setup.py install && \ - apt-get remove -y libffi-dev gcc && apt-get autoremove -y && apt-get clean -y + apt-get remove -y libffi-dev gcc && apt-get autoremove -y && apt-get clean -y CMD ["httpaste", "--help"] From edf450613a170f91f72e8f8acd1c773cc34fc2ea Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 03:30:26 +0200 Subject: [PATCH 10/55] refactor(docker): remove entrypoint --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9767ba9..ebcf3aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,6 @@ COPY . . RUN apt-get update && \ apt-get install -y libffi-dev gcc && \ python3 setup.py install && \ - apt-get remove -y libffi-dev gcc && apt-get autoremove -y && apt-get clean -y + apt-get remove -y libffi-dev gcc && apt-get autoremove -y && apt-get clean -y CMD ["httpaste", "--help"] From e8ae877a48b37ef9a429caa3dd60a03fa0c69165 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 9 Apr 2022 02:27:01 +0200 Subject: [PATCH 11/55] fix(docker): set entrypoint to uwsgi for base image --- Dockerfile | 8 +++- Pipfile | 7 +++- Pipfile.lock | 104 +++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 92 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index ebcf3aa..27a6f0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,11 @@ COPY . . RUN apt-get update && \ apt-get install -y libffi-dev gcc && \ + python3 -m pip install pipenv && \ + python3 -m pipenv install --deploy --system --verbose && \ python3 setup.py install && \ - apt-get remove -y libffi-dev gcc && apt-get autoremove -y && apt-get clean -y + apt-get remove -y libffi-dev gcc && apt-get autoremove -y && apt-get clean -y -CMD ["httpaste", "--help"] +ENTRYPOINT ["uwsgi", "--master", "--enable-threads", "--manage-script-name", "-w", "httpaste.wsgi:application"] + +CMD ["-s", "/tmp/yourapplication.sock"] \ No newline at end of file diff --git a/Pipfile b/Pipfile index 01b679f..e683928 100644 --- a/Pipfile +++ b/Pipfile @@ -7,7 +7,10 @@ name = 'pypi' python_version = '3' [packages] -httpaste = {editable = true, path = "."} +httpaste-victorykit = {editable = true, path = "."} +flup = '==1.0.3' +mysql-connector-python = '==8.0.28' +uWSGI = '==2.0.20' [dev-packages] -tox = '==3.23.0' \ No newline at end of file +tox = '==3.23.0' \ No newline at end of file diff --git a/Pipfile.lock b/Pipfile.lock index 4223497..fc24f88 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6fc8f1480cab514207ed13c95c3533fd240e04aa466d8fe781b969aa42b6313d" + "sha256": "e8725ecbf33a0d4931d941bfa72dcb15bbcdbdcff1048ea65a4025146018a498" }, "pipfile-spec": 6, "requires": { @@ -96,11 +96,11 @@ }, "click": { "hashes": [ - "sha256:5e0d195c2067da3136efb897449ec1e9e6c98282fbf30d7f9e164af9be901a6b", - "sha256:7ab900e38149c9872376e8f9b5986ddcaf68c0f413cf73678a0bca5547e6f976" + "sha256:24e1a4a9ec5bf6299411369b208c1df2188d9eb8d916302fe6bf03faed227f1e", + "sha256:479707fe14d9ec9a0757618b7a100a0ae4c4e236fac5b7f80ca68028141a1a72" ], "markers": "python_version >= '3.7'", - "version": "==8.1.1" + "version": "==8.1.2" }, "clickclick": { "hashes": [ @@ -110,9 +110,6 @@ "version": "==20.10.2" }, "connexion": { - "extras": [ - "swagger-ui" - ], "hashes": [ "sha256:0ba5c163d34cb3cb3bf597d5b95fc14bad5d3596bf10ec86e32cdb63f68d0c8a", "sha256:26a570a0283bbe4cdaf5d90dfb3441aaf8e18cb9de10f3f96bbc128a8a3d8b47" @@ -154,9 +151,13 @@ "markers": "python_version >= '3.7'", "version": "==2.1.1" }, - "httpaste": { - "editable": true, - "path": "." + "flup": { + "hashes": [ + "sha256:5eb09f26eb0751f8380d8ac43d1dfb20e1d42eca0fa45ea9289fa532a79cd159", + "sha256:ca9fd78e1cc0431da1236f73fafd1c01db684675b4d369460d5f5c62e6f0b8d6" + ], + "index": "pypi", + "version": "==1.0.3" }, "httpaste-victorykit": { "editable": true, @@ -256,6 +257,33 @@ "markers": "python_version >= '3.7'", "version": "==2.1.1" }, + "mysql-connector-python": { + "hashes": [ + "sha256:04d75ec7c181e7907df3d40c2a573063f25ecfc5a95a7a90374861c02ce738a9", + "sha256:47f059bc2a7378acd56ac7a60b0526a2ba95d96b696a875b5b233a0feae30980", + "sha256:4d126ce5e03675d926a9e49ce1638d06af43ca7bac2b502d93373dc9425d386f", + "sha256:50c87ff50762f4a0cc0816365dde0e7de763949e125488b8e872de6471e0e427", + "sha256:687071dc9e51892d0861bbcbcbd48e0f3579e3155f2a0ec310198704137c775a", + "sha256:73c5149b33401610e28589d1fc669cba11d3b16215a8f6a75f63ece1f3af5f88", + "sha256:77ec293e265d01db1896a8e63a16b3d5c848a885cf76c77148adfed8453846e8", + "sha256:78bb1abb57bbb85263d65a240a901195e3de0e0992f25e42c48af0869079bb74", + "sha256:7d518491d6d51b186b3182b3698b1560d9bd80675c055163359d0aeea0001de1", + "sha256:8d8dd02e0e6bb7262156a836c3e83582d1a1a1ebb9d72e777a46813709404601", + "sha256:91be638d1b084835edf7aa426d85228174611a1cd6f016ca0f6d4339ac3d9d7b", + "sha256:aaec9d13fc0177e421a3c4392f0eaf86347b825949d5dfc202d535cdb1e07f04", + "sha256:b3a747c5efd6de7b76686ab93834186e2276a62684600dbede615537040436ca", + "sha256:b4c5ce835078555b6640921cae036daad46884dd21027f43c742fb505221e4e6", + "sha256:bb317b179bfbb3e86c771bb2b34794188a2d2b010cdaa1b4d1b5ea0961d0812c", + "sha256:bd89598b173aa0fc525b59fff6e3598ff3cabad4260a3bb49cf420eac10d3b3b", + "sha256:bdb4f187f737316d1c403085b2fb7c91717268d052ecbfc86066cef59f6d72a4", + "sha256:c76d771fdce1314b07619efff184ec03f56abef6b4ccdc686d3a995f5b225fec", + "sha256:d559f69e8b58ac248e37d30e5676718adf69eeff56ed8a7c03f064d74af68f99", + "sha256:e008127430c8dc66bb1b6d6c7a17498ec57ffa81188fc1f8c9f764363c01d12e", + "sha256:f5da43c77d409c8135132f5b5aee9ac91c2e97c3f87352e1b3017438a9cb9b82" + ], + "index": "pypi", + "version": "==8.0.28" + }, "packaging": { "hashes": [ "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", @@ -264,6 +292,36 @@ "markers": "python_version >= '3.6'", "version": "==21.3" }, + "protobuf": { + "hashes": [ + "sha256:001c2160c03b6349c04de39cf1a58e342750da3632f6978a1634a3dcca1ec10e", + "sha256:0b250c60256c8824219352dc2a228a6b49987e5bf94d3ffcf4c46585efcbd499", + "sha256:1d24c81c2310f0063b8fc1c20c8ed01f3331be9374b4b5c2de846f69e11e21fb", + "sha256:1eb13f5a5a59ca4973bcfa2fc8fff644bd39f2109c3f7a60bd5860cb6a49b679", + "sha256:25d2fcd6eef340082718ec9ad2c58d734429f2b1f7335d989523852f2bba220b", + "sha256:32bf4a90c207a0b4e70ca6dd09d43de3cb9898f7d5b69c2e9e3b966a7f342820", + "sha256:38fd9eb74b852e4ee14b16e9670cd401d147ee3f3ec0d4f7652e0c921d6227f8", + "sha256:47257d932de14a7b6c4ae1b7dbf592388153ee35ec7cae216b87ae6490ed39a3", + "sha256:4eda68bd9e2a4879385e6b1ea528c976f59cd9728382005cc54c28bcce8db983", + "sha256:52bae32a147c375522ce09bd6af4d2949aca32a0415bc62df1456b3ad17c6001", + "sha256:542f25a4adf3691a306dcc00bf9a73176554938ec9b98f20f929a044f80acf1b", + "sha256:5b5860b790498f233cdc8d635a17fc08de62e59d4dcd8cdb6c6c0d38a31edf2b", + "sha256:6efe066a7135233f97ce51a1aa007d4fb0be28ef093b4f88dac4ad1b3a2b7b6f", + "sha256:71b2c3d1cd26ed1ec7c8196834143258b2ad7f444efff26fdc366c6f5e752702", + "sha256:7a53d4035427b9dbfbb397f46642754d294f131e93c661d056366f2a31438263", + "sha256:7dcd84dc31ebb35ade755e06d1561d1bd3b85e85dbdbf6278011fc97b22810db", + "sha256:88c8be0558bdfc35e68c42ae5bf785eb9390d25915d4863bbc7583d23da77074", + "sha256:8be43a91ab66fe995e85ccdbdd1046d9f0443d59e060c0840319290de25b7d33", + "sha256:8d84453422312f8275455d1cb52d850d6a4d7d714b784e41b573c6f5bfc2a029", + "sha256:9d0f3aca8ca51c8b5e204ab92bd8afdb2a8e3df46bd0ce0bd39065d79aabcaa4", + "sha256:a1eebb6eb0653e594cb86cd8e536b9b083373fca9aba761ade6cd412d46fb2ab", + "sha256:bc14037281db66aa60856cd4ce4541a942040686d290e3f3224dd3978f88f554", + "sha256:fbcbb068ebe67c4ff6483d2e2aa87079c325f8470b24b098d6bf7d4d21d57a69", + "sha256:fd7133b885e356fa4920ead8289bb45dc6f185a164e99e10279f33732ed5ce15" + ], + "markers": "python_version >= '3.7'", + "version": "==3.20.0" + }, "pycparser": { "hashes": [ "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", @@ -361,13 +419,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==2.27.1" }, - "swagger-ui-bundle": { - "hashes": [ - "sha256:b462aa1460261796ab78fd4663961a7f6f347ce01760f1303bbbdf630f11f516", - "sha256:cea116ed81147c345001027325c1ddc9ca78c1ee7319935c3c75d3669279d575" - ], - "version": "==0.0.9" - }, "urllib3": { "hashes": [ "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", @@ -376,21 +427,28 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.26.9" }, + "uwsgi": { + "hashes": [ + "sha256:88ab9867d8973d8ae84719cf233b7dafc54326fcaec89683c3f9f77c002cdff9" + ], + "index": "pypi", + "version": "==2.0.20" + }, "werkzeug": { "hashes": [ - "sha256:094ecfc981948f228b30ee09dbfe250e474823b69b9b1292658301b5894bbf08", - "sha256:9b55466a3e99e13b1f0686a66117d39bda85a992166e0a79aedfcf3586328f7a" + "sha256:3c5493ece8268fecdcdc9c0b112211acd006354723b280d643ec732b6d4063d6", + "sha256:f8e89a20aeabbe8a893c24a461d3ee5dad2123b05cc6abd73ceed01d39c3ae74" ], "markers": "python_version >= '3.7'", - "version": "==2.1.0" + "version": "==2.1.1" }, "zipp": { "hashes": [ - "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d", - "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375" + "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad", + "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099" ], "markers": "python_version >= '3.7'", - "version": "==3.7.0" + "version": "==3.8.0" } }, "develop": { From f25e3f766c39e8f2a936121d25994b05fe0ac75a Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 9 Apr 2022 02:27:41 +0200 Subject: [PATCH 12/55] feat(samples): init httpaste.it sample --- samples/httpaste.it/docker-compose.yml | 33 +++++++++++++++ samples/httpaste.it/httpaste/config.ini | 16 +++++++ samples/httpaste.it/httpd/Dockerfile | 3 ++ samples/httpaste.it/httpd/httpd.conf | 55 +++++++++++++++++++++++++ 4 files changed, 107 insertions(+) create mode 100644 samples/httpaste.it/docker-compose.yml create mode 100644 samples/httpaste.it/httpaste/config.ini create mode 100644 samples/httpaste.it/httpd/Dockerfile create mode 100644 samples/httpaste.it/httpd/httpd.conf diff --git a/samples/httpaste.it/docker-compose.yml b/samples/httpaste.it/docker-compose.yml new file mode 100644 index 0000000..35fb3db --- /dev/null +++ b/samples/httpaste.it/docker-compose.yml @@ -0,0 +1,33 @@ +version: "3.3" +services: + httpaste: + build: + context: ../.. + dockerfile: Dockerfile + environment: + HTTPASTE_CONFIGPATH: /usr/local/httpaste/config.ini + volumes: + - + type: volume + source: system-shared + target: /shared + volume: + nocopy: true + - ./httpaste/config.ini:/usr/local/httpaste/config.ini + command: -s /shared/uwsgi.sock --chmod-socket=666 + httpd: + build: + context: ./httpd + dockerfile: Dockerfile + ports: + - "80:80" + volumes: + - + type: volume + source: system-shared + target: /shared + volume: + nocopy: true + - ./httpd/httpd.conf:/usr/local/apache2/conf/httpd.conf +volumes: + system-shared: \ No newline at end of file diff --git a/samples/httpaste.it/httpaste/config.ini b/samples/httpaste.it/httpaste/config.ini new file mode 100644 index 0000000..801d0d6 --- /dev/null +++ b/samples/httpaste.it/httpaste/config.ini @@ -0,0 +1,16 @@ +[general] +salt = '&)UxB-_$Lk$m=CB}dw[d85{-ZWR?uUNx' +paste_id_size = 8 +paste_key_size = 32 +paste_lifetime = 5 +paste_max_lifetime = 1440 +hmac_iterations = 20000 +paste_default_encoding = 'utf-8' + +[backend] +type = file +base_dirname = 'sample_data' + +[server] +swagger_ui = False +bind_address = 'sample.sock' \ No newline at end of file diff --git a/samples/httpaste.it/httpd/Dockerfile b/samples/httpaste.it/httpd/Dockerfile new file mode 100644 index 0000000..afcc50e --- /dev/null +++ b/samples/httpaste.it/httpd/Dockerfile @@ -0,0 +1,3 @@ +FROM httpd:2.4 + +RUN apt-get update -y && apt-get install -y libapache2-mod-proxy-uwsgi \ No newline at end of file diff --git a/samples/httpaste.it/httpd/httpd.conf b/samples/httpaste.it/httpd/httpd.conf new file mode 100644 index 0000000..0ac6021 --- /dev/null +++ b/samples/httpaste.it/httpd/httpd.conf @@ -0,0 +1,55 @@ + +ServerRoot "/usr/local/apache2" + +Listen 0.0.0.0:80 + +LoadModule mpm_event_module modules/mod_mpm_event.so +LoadModule authn_core_module modules/mod_authn_core.so +LoadModule authz_core_module modules/mod_authz_core.so +#LoadModule brotli_module modules/mod_brotli.so +LoadModule mime_module modules/mod_mime.so +LoadModule log_config_module modules/mod_log_config.so +#LoadModule log_debug_module modules/mod_log_debug.so +#LoadModule log_forensic_module modules/mod_log_forensic.so +LoadModule env_module modules/mod_env.so +LoadModule proxy_module modules/mod_proxy.so +LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so +LoadModule unixd_module modules/mod_unixd.so + + + User www-data + Group www-data + + +ServerAdmin you@example.com + + +ErrorLog /proc/self/fd/2 + +LogLevel warn + + + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + # You need to enable mod_logio.c to use %I and %O + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + CustomLog /proc/self/fd/1 common + + + + SSLRandomSeed startup builtin + SSLRandomSeed connect builtin + + +ServerName 127.0.0.1 + + + #ProxyPreserveHost On + SetEnv proxy-sendchunks + + ProxyPass "/" "unix:/shared/uwsgi.sock|uwsgi://localhost/" + \ No newline at end of file From d558a656093f84e9b1087f76f56f9dd955f30f06 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 10 Apr 2022 13:41:46 +0200 Subject: [PATCH 13/55] fix(samples/httpaste.it): restrict httpd host disallow any host not httpaste.it --- samples/httpaste.it/httpd/httpd.conf | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/samples/httpaste.it/httpd/httpd.conf b/samples/httpaste.it/httpd/httpd.conf index 0ac6021..04b90e0 100644 --- a/samples/httpaste.it/httpd/httpd.conf +++ b/samples/httpaste.it/httpd/httpd.conf @@ -15,6 +15,7 @@ LoadModule env_module modules/mod_env.so LoadModule proxy_module modules/mod_proxy.so LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so LoadModule unixd_module modules/mod_unixd.so +LoadModule access_compat_module modules/mod_access_compat.so User www-data @@ -47,9 +48,16 @@ LogLevel warn ServerName 127.0.0.1 + + + Deny from all + Allow from none + + + #ProxyPreserveHost On + ServerName httpaste.it SetEnv proxy-sendchunks - - ProxyPass "/" "unix:/shared/uwsgi.sock|uwsgi://localhost/" - \ No newline at end of file + ProxyPass "/" "unix:/shared/uwsgi.sock|uwsgi://localhost/" + From 60a01ea511b3432beb674fc14c3703bc724e2313 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 13 Apr 2022 12:54:59 +0200 Subject: [PATCH 14/55] refactor(samples/httpaste.it): finalize initial sample - add tor daemon - clean directory structure --- samples/httpaste.it/docker-compose.yml | 11 +++++++++-- samples/httpaste.it/httpaste.service | 17 +++++++++++++++++ .../{ => usr/local/httpaste}/config.ini | 0 .../{ => usr/local/apache2/conf}/httpd.conf | 7 +++++++ samples/httpaste.it/tor/Dockerfile | 10 ++++++++++ samples/httpaste.it/tor/etc/tor/torrc | 3 +++ .../httpaste.it/tor/usr/local/sbin/hostname.sh | 3 +++ 7 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 samples/httpaste.it/httpaste.service rename samples/httpaste.it/httpaste/{ => usr/local/httpaste}/config.ini (100%) rename samples/httpaste.it/httpd/{ => usr/local/apache2/conf}/httpd.conf (90%) create mode 100644 samples/httpaste.it/tor/Dockerfile create mode 100644 samples/httpaste.it/tor/etc/tor/torrc create mode 100755 samples/httpaste.it/tor/usr/local/sbin/hostname.sh diff --git a/samples/httpaste.it/docker-compose.yml b/samples/httpaste.it/docker-compose.yml index 35fb3db..a334189 100644 --- a/samples/httpaste.it/docker-compose.yml +++ b/samples/httpaste.it/docker-compose.yml @@ -4,6 +4,7 @@ services: build: context: ../.. dockerfile: Dockerfile + target: uwsgi environment: HTTPASTE_CONFIGPATH: /usr/local/httpaste/config.ini volumes: @@ -13,7 +14,7 @@ services: target: /shared volume: nocopy: true - - ./httpaste/config.ini:/usr/local/httpaste/config.ini + - ./httpaste/usr/local/httpaste/config.ini:/usr/local/httpaste/config.ini command: -s /shared/uwsgi.sock --chmod-socket=666 httpd: build: @@ -28,6 +29,12 @@ services: target: /shared volume: nocopy: true - - ./httpd/httpd.conf:/usr/local/apache2/conf/httpd.conf + - ./httpd/usr/local/apache2/conf/httpd.conf:/usr/local/apache2/conf/httpd.conf + tor: + build: + context: ./tor + dockerfile: Dockerfile + volumes: + - ./tor/etc/tor/torrc:/etc/tor/torrc volumes: system-shared: \ No newline at end of file diff --git a/samples/httpaste.it/httpaste.service b/samples/httpaste.it/httpaste.service new file mode 100644 index 0000000..37a85cb --- /dev/null +++ b/samples/httpaste.it/httpaste.service @@ -0,0 +1,17 @@ + +[Unit] +Description=httpaste (via Docker Compose) +Requires=docker.service +After=docker.service + +[Service] +WorkingDirectory=/usr/local/src/httpaste/samples/httpaste.it +ExecStart=docker-compose up +ExecStop=docker-compose down +TimeoutStartSec=0 +Restart=on-failure +StartLimitIntervalSec=60 +StartLimitBurst=3 + +[Install] +WantedBy=multi-user.target diff --git a/samples/httpaste.it/httpaste/config.ini b/samples/httpaste.it/httpaste/usr/local/httpaste/config.ini similarity index 100% rename from samples/httpaste.it/httpaste/config.ini rename to samples/httpaste.it/httpaste/usr/local/httpaste/config.ini diff --git a/samples/httpaste.it/httpd/httpd.conf b/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf similarity index 90% rename from samples/httpaste.it/httpd/httpd.conf rename to samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf index 04b90e0..4c5ead0 100644 --- a/samples/httpaste.it/httpd/httpd.conf +++ b/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf @@ -61,3 +61,10 @@ ServerName 127.0.0.1 SetEnv proxy-sendchunks ProxyPass "/" "unix:/shared/uwsgi.sock|uwsgi://localhost/" + + + #ProxyPreserveHost On + ServerAlias *.onion + SetEnv proxy-sendchunks + ProxyPass "/" "unix:/shared/uwsgi.sock|uwsgi://localhost/" + diff --git a/samples/httpaste.it/tor/Dockerfile b/samples/httpaste.it/tor/Dockerfile new file mode 100644 index 0000000..cca2e04 --- /dev/null +++ b/samples/httpaste.it/tor/Dockerfile @@ -0,0 +1,10 @@ +FROM debian:bullseye-slim + +RUN apt-get update -y && apt-get install -y tor + +COPY ./usr/local/sbin/hostname.sh /usr/local/sbin/hostname +RUN chmod +x /usr/local/sbin/hostname + +USER debian-tor + +ENTRYPOINT ["tor"] \ No newline at end of file diff --git a/samples/httpaste.it/tor/etc/tor/torrc b/samples/httpaste.it/tor/etc/tor/torrc new file mode 100644 index 0000000..677f5b2 --- /dev/null +++ b/samples/httpaste.it/tor/etc/tor/torrc @@ -0,0 +1,3 @@ +DataDirectory /var/lib/tor +HiddenServiceDir /var/lib/tor/hidden_service/ +HiddenServicePort 80 httpd:80 diff --git a/samples/httpaste.it/tor/usr/local/sbin/hostname.sh b/samples/httpaste.it/tor/usr/local/sbin/hostname.sh new file mode 100755 index 0000000..ee0ff27 --- /dev/null +++ b/samples/httpaste.it/tor/usr/local/sbin/hostname.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +prop=HiddenServiceDir +cat $(grep $prop /etc/tor/torrc | sed "s/$prop //g")/hostname \ No newline at end of file From 6ec39a9303291b5936364a1dc239a031b5881938 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Wed, 13 Apr 2022 12:56:30 +0200 Subject: [PATCH 15/55] refactor(Dockerfile): add build stages --- Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 27a6f0c..d64fa0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ -FROM python:3.10-slim +FROM python:3.10-slim as base LABEL org.label-schema.schema-version="1.0" LABEL org.label-schema.vendor="Tiara Rodney (victoryk.it)" LABEL org.label-schema.name="victorykit/httpaste" LABEL org.label-schema.description="a versatile HTTP pastebin" -LABEL org.label-schema.vcs-url="https://bitbucket.org/victorykit/docker-selenium-grid" +LABEL org.label-schema.vcs-url="https://bitbucket.org/victorykit/httpaste" LABEL org.label-schema.docker.cmd="docker run {image-id} {httpaste-args}" LABEL org.label-schema.version=$BUILD_VERSION LABEL org.label-schema.build-date=$BUILD_DATE @@ -20,6 +20,11 @@ RUN apt-get update && \ python3 setup.py install && \ apt-get remove -y libffi-dev gcc && apt-get autoremove -y && apt-get clean -y +ENTRYPOINT ["httpaste"] + + +FROM base as uwsgi + ENTRYPOINT ["uwsgi", "--master", "--enable-threads", "--manage-script-name", "-w", "httpaste.wsgi:application"] CMD ["-s", "/tmp/yourapplication.sock"] \ No newline at end of file From fdf45fd114a860aa27792ba8ff07187101ecded1 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 01:52:45 +0200 Subject: [PATCH 16/55] refactor: cleanup - lazy load backend - proper config evaluation - object-orientation of configuration --- src/httpaste/__init__.py | 165 +++++----------------- src/httpaste/__main__.py | 6 +- src/httpaste/backend/__init__.py | 141 ++++++++++-------- src/httpaste/backend/file/__init__.py | 107 +++++++------- src/httpaste/backend/mysql/__init__.py | 116 +++++++-------- src/httpaste/backend/mysql/paste.py | 24 ++-- src/httpaste/backend/mysql/paste.sql | 9 -- src/httpaste/backend/mysql/user.py | 19 ++- src/httpaste/backend/mysql/user.sql | 6 - src/httpaste/backend/sqlite/__init__.py | 143 ++++++++++--------- src/httpaste/backend/sqlite/paste.py | 34 +++-- src/httpaste/backend/sqlite/paste.sql | 9 -- src/httpaste/backend/sqlite/user.py | 29 ++-- src/httpaste/backend/sqlite/user.sql | 6 - src/httpaste/context.py | 18 +++ src/httpaste/controller/__init__.py | 10 +- src/httpaste/controller/paste/__init__.py | 56 +++++--- src/httpaste/controller/paste/private.py | 2 - src/httpaste/controller/user/session.py | 6 +- src/httpaste/helper/config.py | 111 +++++++++++++++ src/httpaste/helper/crypto.py | 4 +- src/httpaste/model/__init__.py | 151 ++------------------ src/httpaste/model/paste.py | 99 +++++++------ src/httpaste/model/user.py | 35 +++-- src/httpaste/schema/__init__.py | 137 ++++++++++++++++++ src/httpaste/server.py | 17 +++ 26 files changed, 783 insertions(+), 677 deletions(-) delete mode 100644 src/httpaste/backend/mysql/paste.sql delete mode 100644 src/httpaste/backend/mysql/user.sql delete mode 100644 src/httpaste/backend/sqlite/paste.sql delete mode 100644 src/httpaste/backend/sqlite/user.sql create mode 100755 src/httpaste/context.py create mode 100755 src/httpaste/helper/config.py create mode 100755 src/httpaste/server.py diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 14f2385..a4aafd2 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -137,21 +137,23 @@ NOTES SUCH DAMAGES. """ -from typing import NamedTuple, Tuple, Any -from string import ascii_uppercase, digits, ascii_letters, punctuation -from inspect import isclass +from typing import NamedTuple from configparser import ConfigParser -from ast import literal_eval -from io import StringIO -from os import environ from importlib.resources import path as resource_path +from pathlib import Path from connexion import FlaskApp from connexion.resolver import RestyResolver -from httpaste.model import Backend -from httpaste.backend import get_backend_map -from httpaste.helper.common import generate_random_string +from httpaste.server import get_server_config +from httpaste.server import Config as ServerConfig +from httpaste.context import get_context_config +from httpaste.context import Config as ContextConfig +from httpaste.model import get_model_config +from httpaste.model import Config as ModelConfig +from httpaste.backend import get_backend_config +from httpaste.backend import Config as BackendConfig +from httpaste.helper.config import get_configparser, CONFIGPATH_ENVIRON from httpaste.helper.http import ( BadRequestError, ForbiddenError, @@ -160,147 +162,48 @@ from httpaste.helper.http import ( UnauthorizedError) -CONFIGPATH_ENVIRON = 'HTTPASTE_CONFIGPATH' - - -def get_sanitized_config_charset(charset: str): - - for x in ["$", "%"]: - - charset = charset.replace(x, f'{x}{x}') - - return charset - - -class ConfigError(Exception): - """Config Exception +class Config(NamedTuple): """ - - -class Config: - """httpaste global config """ - salt: bytes = get_sanitized_config_charset(generate_random_string( - 32, ascii_letters + digits + punctuation)).encode('utf-8') - paste_id_size: int = 8 - paste_id_charset: str = ascii_letters + digits - paste_key_size: int = 32 - paste_key_charset: str = get_sanitized_config_charset( - ascii_letters + digits + punctuation) - paste_lifetime: int = 5 - backend: Backend = None - hmac_iterations: int = 20000 - paste_default_encoding: str = 'utf-8' + context: ContextConfig + server: ServerConfig + model: ModelConfig + backend: BackendConfig -class ServerConfig: - """connexion config - """ - swagger_ui: bool = True - bind_address = None - - -def get_config_path(var_name: str = CONFIGPATH_ENVIRON): +def get_config(configIni: ConfigParser, path: Path): """ """ - try: + from httpaste.model import Config as ModelConfig - return environ[var_name] - except KeyError as e: + context_config = get_context_config(configIni) + server_config = get_server_config(configIni) + model_config = get_model_config(configIni, path) + backend_config = get_backend_config(configIni, path) - raise ConfigError( - f'environment variable \'{var_name}\' not set.') from e + return Config( + context=context_config, + server=server_config, + model=model_config, + backend=backend_config + ) -def load_config(path: str) -> Tuple[Config, ServerConfig]: - """get config objects from file - """ - - _config = ConfigParser() - _config.read(path) - - backends = get_backend_map() - bconf = dict(_config.items('backend')) - btype = bconf.pop('type') - - try: - bcl, bparamcl = backends[btype] - except KeyError as e: - bids = ', '.join(backends.keys()) - raise ConfigError(' '.join(( - f'invalid backend \'{btype}\' in \'{path}\'. ', - f'must be any of [{bids}]' - ))) from e - - config = dict(_config.items('general')) - server_config = dict(_config.items('server')) - - c = Config() - sc = ServerConfig() - - # typecast model_backend section items - bconf = {k: literal_eval(v) for k, v in bconf.items()} - # initialize model backend - c.backend = bcl(bparamcl(**bconf)) - - # typecast general section items - for k, v in config.items(): - setattr(c, k, literal_eval(v)) - # typecast server section items - for k, v in server_config.items(): - setattr(sc, k, literal_eval(v)) - - c.salt = c.salt.encode('utf-8') - - return c, sc - - -def default_config() -> str: +def load_config(path: str = None, var_name: str = CONFIGPATH_ENVIRON): """ """ - config = ConfigParser() + configIni, _ = get_configparser(path, var_name) - config['general'] = { - 'salt': Config.salt.decode('utf-8'), - 'paste_key_charset': Config.paste_key_charset, - 'paste_id_charset': Config.paste_id_charset - } - - for literal in [ - 'paste_id_size', - 'paste_key_size', - 'paste_lifetime' - ]: - config['general'][literal] = str(getattr(Config, literal)) - - config['backend'] = { - 'type': 'sqlite', - 'path': 'file::memory:?cache=shared' - } - - config['server'] = {} - for literal in [ - 'swagger_ui', - 'bind_address' - ]: - config['server'][literal] = str(getattr(ServerConfig, literal)) - - stream = StringIO() - config.write(stream) - stream.seek(0) - - return stream.read() + return get_config(configIni, Path(path).resolve().parent) -def get_flask_app( - config: Config, - server_config: ServerConfig = ServerConfig) -> FlaskApp: +def get_flask_app(config: Config) -> FlaskApp: """get a flask app object """ - options = {"swagger_ui": server_config.swagger_ui} + options = {"swagger_ui": config.server.swagger_ui} #context manager returns a pathlib.Path object with resource_path('httpaste.schema', 'httpaste.openapi.json') as path: @@ -340,8 +243,6 @@ def get_flask_app( __all__ = [ Config, - ServerConfig, load_config, - default_config, get_flask_app ] diff --git a/src/httpaste/__main__.py b/src/httpaste/__main__.py index ec35404..a625849 100644 --- a/src/httpaste/__main__.py +++ b/src/httpaste/__main__.py @@ -40,9 +40,9 @@ def command_standalone(**kwargs): 'Please install it by running \'python3 -m pip install gevent\'.' ))) from e - config, server_config = load_config(kwargs.get('config_path')) + config = load_config(kwargs.get('config_path')) - application = get_flask_app(config, server_config) + application = get_flask_app(config) http_server = WSGIServer(('', kwargs.get('port')), application) http_server.serve_forever() @@ -122,7 +122,7 @@ def parser(): p_standalone = sp.add_parser('standalone', help=command_standalone.__doc__) p_standalone.add_argument('--config-path', '-c', required=True) - p_standalone.add_argument('--port', '-p', default=8080) + p_standalone.add_argument('--port', '-p', default=8082) p_wsgi = sp.add_parser('wsgi', help=command_wsgi.__doc__) p_wsgi.add_argument('--echo', '-e', action='store_true') diff --git a/src/httpaste/backend/__init__.py b/src/httpaste/backend/__init__.py index 67f1c8c..ef510a2 100644 --- a/src/httpaste/backend/__init__.py +++ b/src/httpaste/backend/__init__.py @@ -2,83 +2,114 @@ implements backend of model """ -import sys -from inspect import isclass -from typing import Dict, Tuple +from abc import ABC, abstractmethod +from importlib import import_module +from configparser import ConfigParser +from typing import NamedTuple +from pathlib import Path -from httpaste.model import Backend, UserDataSchema, PasteDataSchema, User, Paste -from .sqlite import Parameters as SqliteParameters -from .sqlite import User as SqliteUser -from .sqlite import Paste as SqlitePaste -from .sqlite import get_connection as get_sqlite_connection -from .mysql import get_connection as get_mysql_connection -from .file import Parameters as FileParameters -from .file import User as FileUser -from .file import Paste as FilePaste -from .mysql import Parameters as MySQLParameters -from .mysql import User as MySQLUser -from .mysql import Paste as MySQLPaste +from httpaste.schema import User, Paste, UserDataSchema, PasteDataSchema +from httpaste.helper.config import get_config, ConfigError -class SQLite(Backend): - """SQLite backend interface +class BackendError(Exception): + """ """ - parameter_class = SqliteParameters - user: SqliteUser - paste: SqlitePaste - def __init__(self, parameters: SqliteParameters): - - parameters = SqliteParameters(parameters.path, get_sqlite_connection(parameters)) - - self.user = SqliteUser(parameters, User) - self.paste = SqlitePaste(parameters, Paste) - - -class File(Backend): - """File backend interface +class ObjectBackend(ABC): + """ """ - parameter_class = FileParameters - user: FileUser - paste: FilePaste + @abstractmethod + def load(self, proto: object) -> object: + pass - def __init__(self, parameters: FileParameters): + @abstractmethod + def dump(self, model: object) -> None: + pass - self.user = FileUser(parameters, User, UserDataSchema) - self.paste = FilePaste(parameters, Paste, PasteDataSchema) + @abstractmethod + def delete(self, proto: object) -> None: + pass + + @abstractmethod + def init(self) -> object: + pass + + @abstractmethod + def sanitize(self) -> None: + pass -class MySQL(Backend): - """MySQL backend interface +class BackendInterface(ABC): + """ """ - parameter_class = MySQLParameters - user: MySQLUser - paste: MySQLPaste + @abstractmethod + def __init__(self, params: object, + user_model_class: type, + paste_model_class: type, + user_schema: type, + paste_schema: type) -> None: + pass - def __init__(self, parameters: MySQLParameters): + @property + @abstractmethod + def user(self) -> ObjectBackend: + pass - #parameters = MySQLParameters(*parameters[1:], get_mysql_connection(parameters)) - - self.user = MySQLUser(parameters, User) - self.paste = MySQLPaste(parameters, Paste) + @property + @abstractmethod + def paste(self) -> ObjectBackend: + pass -def get_backend_map() -> Dict[str, Tuple[type, type]]: - """get a map of backend ids and their classes +class Config(NamedTuple): + """Backend Configuration + """ + interface: type + config: dict + + +def load_backend(config: Config) -> BackendInterface: + """load a backend """ - mod = sys.modules[__name__] - out = {} + backend = config.interface(config.config, Paste, User, PasteDataSchema, + UserDataSchema) - for i in dir(mod): + return backend - obj = getattr(mod, i) - if isclass(obj) and obj.__module__ == __name__: +def get_backend_config(configIni: ConfigParser, path:Path) -> Config: + """retrieve a cascaded backend configuration from an INI config object + """ - out[i.lower()] = (obj, obj.parameter_class) + if 'backend' not in configIni: - return out + raise ConfigError('missing [backend] section.') + + if 'type' not in configIni['backend']: + + raise ConfigError('missing [backend] \'type\'.') + + mod_name = configIni['backend']['type'] + + section = f'backend.{mod_name}' + + try: + mod = import_module(f'.{mod_name}', 'httpaste.backend') + except ImportError as e: + raise BackendError(f'backend \'{mod_name}\' does not exist: {e}') from e + else: + interface = mod.Backend + config = get_config(configIni, section, mod.Config, path) + + return Config(interface=interface,config=config) + + +__all__ = [ + load_backend, + get_backend_config +] diff --git a/src/httpaste/backend/file/__init__.py b/src/httpaste/backend/file/__init__.py index 9352f8f..1a239d2 100644 --- a/src/httpaste/backend/file/__init__.py +++ b/src/httpaste/backend/file/__init__.py @@ -3,110 +3,117 @@ from os import path from pathlib import Path from typing import NamedTuple, Optional - -from . import user -from . import paste +from httpaste.backend import BackendInterface as BackendAbc +from httpaste.backend import ObjectBackend as ObjectBackendAbc -class Parameters(NamedTuple): - """Filesystem backend parameters +class Config(NamedTuple): + """Filesystem backend config """ #: path of base directory - base_dirname: str + base_dirname: Path #: basename of users table directory - user_dirname: Optional[str] = 'users' + user_dirname: str = 'users' #: basename of pastes table directory - paste_dirname: Optional[str] = 'pastes' + paste_dirname: str = 'pastes' -class User(object): - """Filesystem user model backend - """ +class ObjectBackendBc(ObjectBackendAbc): dirname: Path path: Path def __init__( self, - parameters: Parameters, + interface: object, + basename_attr: str, + config: Config, model_class: type, model_schema: type): + self.interface = interface self.model_class = model_class - self.model_schema = model_schema - - self.dirname = path.join(parameters.base_dirname, - parameters.user_dirname) - + self.dirname = path.join(config.base_dirname, + getattr(config, basename_attr)) self.path = Path(self.dirname) def load(self, proto: object): - return user.load(proto, self.path, self.model_class, self.model_schema) + return self.interface.load(proto, self.path, self.model_class, self.model_schema) def dump(self, model: object): - return user.dump(model, self.path, self.model_schema) + return self.interface.dump(model, self.path, self.model_schema) def delete(self, proto: object): - return user.delete(proto, self.path) + return self.interface.delete(proto, self.path) def init(self): - return user.init(self.path) + return self.interface.init(self.path) def sanitize(self): if self.path.exists(): - return user.sanitize(self.path, self.model_class, self.model_schema) + return self.interface.sanitize(self.path, self.model_class, self.model_schema) return None -class Paste(object): +class UserBackend(ObjectBackendBc): + """Filesystem user model backend + """ + + def __init__(self, *args): + + from . import user + + super().__init__(user, 'user_dirname', *args) + + +class PasteBackend(ObjectBackendBc): """Filesystem paste model backend """ - dirname: str - path: Path + def __init__(self, *args): - def __init__( - self, - parameters: Parameters, - model_class: type, - model_schema: type): + from . import paste - self.model_class = model_class + super().__init__(paste, 'paste_dirname', *args) - self.model_schema = model_schema - self.dirname = path.join(parameters.base_dirname, - parameters.paste_dirname) +class Backend(BackendAbc): + """File backend interface + """ - self.path = Path(self.dirname) + _user: UserBackend + _paste: PasteBackend - def load(self, proto: object): + def __init__(self, + config: Config, + paste_model_class: type, + user_model_class: type, + paste_schema: type, + user_schema: type): - return paste.load(proto, self.path, self.model_class, self.model_schema) + self._user = UserBackend(config, user_model_class, user_schema) + self._paste = PasteBackend(config, paste_model_class, paste_schema) - def dump(self, model: object): + @property + def user(self) -> UserBackend: - return paste.dump(model, self.path, self.model_schema) + return self._user - def delete(self, proto: object): + @property + def paste(self) -> PasteBackend: - return paste.delete(proto, self.path) + return self._paste - def init(self): - return paste.init(self.path) - - def sanitize(self): - - if self.path.exists(): - return paste.sanitize(self.path, self.model_class, self.model_schema) - - return None \ No newline at end of file +__all__ = [ + Config, + Backend +] diff --git a/src/httpaste/backend/mysql/__init__.py b/src/httpaste/backend/mysql/__init__.py index e4b0b5c..bbe84c8 100644 --- a/src/httpaste/backend/mysql/__init__.py +++ b/src/httpaste/backend/mysql/__init__.py @@ -1,10 +1,12 @@ """MySQL backend """ -from typing import NamedTuple, Optional, Callable +from typing import NamedTuple, Optional +from httpaste.backend import BackendInterface as BackendAbc +from httpaste.backend import ObjectBackend as ObjectBackendAbc -class Parameters(NamedTuple): - """MySQL parameters +class Config(NamedTuple): + """MySQL config """ #: user name @@ -16,26 +18,18 @@ class Parameters(NamedTuple): #: database identifier database: str #: a mysql.connection.MySQLConnection object (does not apply to config) - connection: Optional[object] = None + connection: object = None -class User(object): - """MySQL user model backend - """ +class ObjectBackendBc(ObjectBackendAbc): connection: object - def __init__(self, parameters: Parameters, model_class: type) -> None: - - from . import user - - connect = get_mysql_connect_callee() - - self.interface = user + def __init__(self, interface: object, config: Config, model_class: type) -> None: + self.interface = interface self.model_class = model_class - - self.connection = get_connection(parameters, connect) + self.connection = get_connection(config) def load(self, proto: object) -> object: @@ -58,50 +52,54 @@ class User(object): return self.interface.sanitize(self.connection, self.model_class) -class Paste(object): +class UserBackend(ObjectBackendBc): + """MySQL user model backend + """ + + def __init__(self, *args) -> None: + + from . import user + + super().__init__(paste, *args) + + +class PasteBackend(ObjectBackendBc): """MySQL paste model backend """ connection: object - def __init__(self, parameters: Parameters, model_class: type) -> None: + def __init__(self, *args) -> None: from . import paste - connect = get_mysql_connect_callee() - - self.interface = paste - - self.model_class = model_class - - self.connection = get_connection(parameters, connect) - - def load(self, proto: object) -> object: - - return self.interface.load(proto, self.connection, self.model_class) - - def dump(self, model: object) -> None: - - return self.interface.dump(model, self.connection) - - def delete(self, proto: object) -> None: - - return self.interface.delete(proto, self.connection) - - def init(self) -> None: - - return paste.init(self.connection) - - def sanitize(self) -> None: - - return self.interface.sanitize(self.connection, self.model_class) + super().__init__(paste, *args) -def get_mysql_connect_callee() -> object: +class Backend(BackendAbc): + """MySQL backend interface + """ + + user: UserBackend + paste: PasteBackend + + def __init__(self, + config: Config, + paste_model_class: type, + user_model_class: type, + paste_schema: type, + user_schema: type): + + self.user = UserBackend(config, user_model_class, user_schema) + self.paste = PasteBackend(config, paste_model_class, paste_schema) + + +def get_connection(config: Config) -> object: + """get a mysql.connection.MySQLConnection object + """ try: from mysql.connector import connect - from mysql.connector.connection import MySQLConnection except ImportError as e: raise ImportError(' '.join(( '\'mysql-connector-python\' is not installed.', @@ -109,26 +107,18 @@ def get_mysql_connect_callee() -> object: '\'python3 -m pip install mysql-connector-python\'.' ))) from e - return connect + if config.connection is not None: + return config.connection -def get_connection(parameters: Parameters, connect: Callable) -> object: - """get a mysql.connection.MySQLConnection object - """ - - if parameters.connection is not None: - - return parameters.connection - - connection = connect(user=parameters.user, password=parameters.password, - host=parameters.host, - database=parameters.database) + connection = connect(user=config.user, password=config.password, + host=config.host, + database=config.database) return connection __all__ = [ - Parameters, - User, - Paste -] \ No newline at end of file + Config, + Backend +] diff --git a/src/httpaste/backend/mysql/paste.py b/src/httpaste/backend/mysql/paste.py index efa5f0e..4465971 100644 --- a/src/httpaste/backend/mysql/paste.py +++ b/src/httpaste/backend/mysql/paste.py @@ -1,16 +1,6 @@ from os import path from time import time -from importlib.resources import open_text - - -try: - from mysql.connector.connection import MySQLConnection -except ImportError as e: - raise ImportError(' '.join(( - '\'mysql-connector-python\' is not installed.', - 'Install it by running', - '\'python3 -m pip install mysql-connector-python\'.' - ))) from e +from mysql.connector.connection import MySQLConnection def load(proto:object, connection: MySQLConnection, model_class: type): @@ -94,9 +84,17 @@ def init(connection: MySQLConnection): cursor = connection.cursor() - with open_text('httpaste.backend.mysql', 'paste.sql') as fh: + statement = '''CREATE TABLE `httpaste_pastes` ( + `pid` blob NOT NULL, + `data` longblob NOT NULL, + `data_hash` blob NOT NULL, + `sub` blob DEFAULT NULL, + `expiration` int(16) NOT NULL, + `encoding` tinytext DEFAULT NULL, + PRIMARY KEY (`pid`(128)) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;''' - cursor.execute(fh.read()) + cursor.execute(statement) connection.commit() diff --git a/src/httpaste/backend/mysql/paste.sql b/src/httpaste/backend/mysql/paste.sql deleted file mode 100644 index d4c366e..0000000 --- a/src/httpaste/backend/mysql/paste.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE `httpaste_pastes` ( - `pid` blob NOT NULL, - `data` longblob NOT NULL, - `data_hash` blob NOT NULL, - `sub` blob DEFAULT NULL, - `expiration` int(16) NOT NULL, - `encoding` tinytext DEFAULT NULL, - PRIMARY KEY (`pid`(128)) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/src/httpaste/backend/mysql/user.py b/src/httpaste/backend/mysql/user.py index a7148ff..47dbf4b 100644 --- a/src/httpaste/backend/mysql/user.py +++ b/src/httpaste/backend/mysql/user.py @@ -1,13 +1,5 @@ from os import path -from importlib.resources import open_text -try: - from mysql.connector.connection import MySQLConnection -except ImportError as e: - raise ImportError(' '.join(( - '\'mysql-connector-python\' is not installed.', - 'Install it by running', - '\'python3 -m pip install mysql-connector-python\'.' - ))) from e +from mysql.connector.connection import MySQLConnection def load(proto:object, connection: MySQLConnection, model_class: type): @@ -84,9 +76,14 @@ def init(connection: MySQLConnection): cursor = connection.cursor() - with open_text('httpaste.backend.mysql', 'user.sql') as fh: + statement = '''CREATE TABLE `httpaste_users` ( + `sub` blob NOT NULL, + `key_hash` blob NOT NULL, + `paste_index` blob NOT NULL, + PRIMARY KEY (`sub`(128)) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;''' - cursor.execute(fh.read()) + cursor.execute(statement) connection.commit() diff --git a/src/httpaste/backend/mysql/user.sql b/src/httpaste/backend/mysql/user.sql deleted file mode 100644 index 62045a9..0000000 --- a/src/httpaste/backend/mysql/user.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE `httpaste_users` ( - `sub` blob NOT NULL, - `key_hash` blob NOT NULL, - `paste_index` blob NOT NULL, - PRIMARY KEY (`sub`(128)) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file diff --git a/src/httpaste/backend/sqlite/__init__.py b/src/httpaste/backend/sqlite/__init__.py index 5680f5c..bb1dbb5 100644 --- a/src/httpaste/backend/sqlite/__init__.py +++ b/src/httpaste/backend/sqlite/__init__.py @@ -2,96 +2,113 @@ """ from sqlite3 import Connection, Row, connect from typing import NamedTuple, Optional +from pathlib import Path -from . import user -from . import paste +from httpaste.backend import BackendInterface as BackendAbc +from httpaste.backend import ObjectBackend as ObjectBackendAbc -class Parameters(NamedTuple): - """SQLite backend parameters +class Config(NamedTuple): + """SQLite backend config """ #: local path or URI - path: str + uri: Path + user_table_name: str = 'httpaste_users' + paste_table_name: str = 'httpaste_pastes' #: a sqlite3.Connection object (does not apply to config) - connection: Optional[object] = None + connection: Connection = None -class User(object): - """SQLite user model backend +class ObjectBackendBc(ObjectBackendAbc): + + connection: object + + def __init__(self, interface: object, table_name_attr: str, config: Config, model_class: type, schema: type) -> None: + + self.interface = interface + self.model_class = model_class + self.connection = get_connection(config) + self.table = getattr(config, table_name_attr) + + def load(self, proto: object) -> object: + + return self.interface.load(proto, self.connection, self.table, self.model_class) + + def dump(self, model: object) -> None: + + return self.interface.dump(model, self.connection, self.table) + + def delete(self, proto: object) -> None: + + return self.interface.delete(proto, self.connection, self.table) + + def init(self) -> None: + + return self.interface.init(self.connection, self.table) + + def sanitize(self) -> None: + + return self.interface.sanitize(self.connection, self.table, self.model_class) + + +class UserBackend(ObjectBackendBc): + """sqlite user model backend """ - connection: Connection + def __init__(self, *args) -> None: - def __init__(self, parameters: Parameters, model_class: type): + from . import user - self.model_class = model_class - - self.connection = get_connection(parameters) - - def load(self, proto: object): - - return user.load(proto, self.connection, self.model_class) - - def dump(self, model: object): - - return user.dump(model, self.connection) - - def delete(self, proto: object): - - return user.delete(proto, self.connection) - - def init(self): - - return user.init(self.connection) - - def sanitize(self): - - return user.sanitize(self.connection, self.model_class) + super().__init__(paste, 'user_table_name', *args) -class Paste(object): - """SQLite paste model backend +class PasteBackend(ObjectBackendBc): + """sqlite paste model backend """ - connection: Connection + connection: object - def __init__(self, parameters: Parameters, model_class: type): + def __init__(self, *args) -> None: - self.model_class = model_class + from . import paste - self.connection = get_connection(parameters) - - def load(self, proto: object): - - return paste.load(proto, self.connection, self.model_class) - - def dump(self, model: object): - - return paste.dump(model, self.connection) - - def delete(self, proto: object): - - return paste.delete(proto, self.connection) - - def init(self): - - return paste.init(self.connection) - - def sanitize(self): - - return paste.sanitize(self.connection, self.model_class) + super().__init__(paste, 'paste_table_name', *args) -def get_connection(parameters: Parameters): +class Backend(BackendAbc): + """sqlite backend interface + """ + + user: UserBackend + paste: PasteBackend + + def __init__(self, + config: Config, + paste_model_class: type, + user_model_class: type, + paste_schema: type, + user_schema: type): + + self.user = UserBackend(config, user_model_class, user_schema) + self.paste = PasteBackend(config, paste_model_class, paste_schema) + + +def get_connection(config: Config): """get an sqlite connection object """ - if parameters.connection: + if config.connection: - return parameters.connection + return config.connection - connection = connect(parameters.path, check_same_thread=False) + connection = connect(config.uri, check_same_thread=False) connection.row_factory = Row return connection + + +__all__ = [ + Config, + Backend +] diff --git a/src/httpaste/backend/sqlite/paste.py b/src/httpaste/backend/sqlite/paste.py index dd8e049..4b7f513 100644 --- a/src/httpaste/backend/sqlite/paste.py +++ b/src/httpaste/backend/sqlite/paste.py @@ -6,14 +6,14 @@ from time import time from importlib.resources import open_text -def load(proto: object, connection: Connection, model_class: type): +def load(proto: object, connection: Connection, table: str, model_class: type): """load a paste """ cursor = connection.cursor() - statement = '''SELECT pid, data, data_hash, sub, expiration, encoding - FROM pastes + statement = f'''SELECT pid, data, data_hash, sub, expiration, encoding + FROM {table} WHERE pid=?''' cursor.execute(statement, (proto.pid,)) @@ -33,13 +33,13 @@ def load(proto: object, connection: Connection, model_class: type): return None -def dump(model: object, connection: Connection) -> None: +def dump(model: object, connection: Connection, table: str) -> None: """dump a paste """ cursor = connection.cursor() - statement = '''INSERT INTO pastes + statement = f'''INSERT INTO "{table}" (pid, data, data_hash, sub, expiration, encoding) VALUES (?,?,?,?,?,?)''' @@ -57,35 +57,41 @@ def dump(model: object, connection: Connection) -> None: return None -def delete(proto: object, connection: Connection) -> None: +def delete(proto: object, connection: Connection, table: str) -> None: cursor = connection.cursor() - cursor.execute('''DELETE FROM pastes WHERE pid=?''', (proto.pid,)) + cursor.execute(f'''DELETE FROM {table} WHERE pid=?''', (proto.pid,)) connection.commit() return None -def init(connection: Connection): +def init(connection: Connection, table: str): cursor = connection.cursor() - with open_text('httpaste.backend.sqlite', 'paste.sql') as fh: + statement = f'''CREATE TABLE IF NOT EXISTS "{table}" ( + "pid" BLOB NOT NULL UNIQUE, + "data" BLOB NOT NULL, + "data_hash" BLOB NOT NULL, + "sub" BLOB UNIQUE, + "expiration" INTEGER NOT NULL, + "encoding" TEXT, + PRIMARY KEY("pid") + );''' - statement = fh.read() - - cursor.execute(statement) + cursor.execute(statement) connection.commit() -def sanitize(connection: Connection, model_class: type) -> int: +def sanitize(connection: Connection, table: str, model_class: type) -> int: cursor = connection.cursor() - statement = '''SELECT pid FROM pastes + statement = f'''SELECT pid FROM {table} WHERE expiration < ? AND expiration > 0''' cursor.execute(statement, (int(time()),)) diff --git a/src/httpaste/backend/sqlite/paste.sql b/src/httpaste/backend/sqlite/paste.sql deleted file mode 100644 index 7f9bb46..0000000 --- a/src/httpaste/backend/sqlite/paste.sql +++ /dev/null @@ -1,9 +0,0 @@ -CREATE TABLE IF NOT EXISTS "pastes" ( - "pid" BLOB NOT NULL UNIQUE, - "data" BLOB NOT NULL, - "data_hash" BLOB NOT NULL, - "sub" BLOB UNIQUE, - "expiration" INTEGER NOT NULL, - "encoding" TEXT, - PRIMARY KEY("pid") -); \ No newline at end of file diff --git a/src/httpaste/backend/sqlite/user.py b/src/httpaste/backend/sqlite/user.py index 207e69d..97e4e69 100644 --- a/src/httpaste/backend/sqlite/user.py +++ b/src/httpaste/backend/sqlite/user.py @@ -6,14 +6,14 @@ from httpaste.model import User from importlib.resources import open_text -def load(proto: object, connection: Connection, model_class: type): +def load(proto: object, connection: Connection, table: str, model_class: type): """load a user """ cursor = connection.cursor() - statement = '''SELECT sub, key_hash, paste_index - FROM users + statement = f'''SELECT sub, key_hash, paste_index + FROM {table} WHERE sub=?''' cursor.execute(statement, (proto.sub,)) @@ -27,13 +27,13 @@ def load(proto: object, connection: Connection, model_class: type): return None -def dump(model: object, connection: Connection) -> None: +def dump(model: object, connection: Connection, table: str) -> None: """dump a user """ cursor = connection.cursor() - statement = '''INSERT OR REPLACE INTO users + statement = f'''INSERT OR REPLACE INTO {table} (sub, key_hash, paste_index) VALUES (?,?,?)''' @@ -44,32 +44,35 @@ def dump(model: object, connection: Connection) -> None: return None -def delete(proto: object, connection: Connection) -> None: +def delete(proto: object, connection: Connection, table: str) -> None: cursor = connection.cursor() - cursor.execute('''DELETE FROM users WHERE sub=?''', (proto.sub,)) + cursor.execute(f'''DELETE FROM {table} WHERE sub=?''', (proto.sub,)) connection.commit() return None -def init(connection: Connection) -> None: +def init(connection: Connection, table: str) -> None: cursor = connection.cursor() - with open_text('httpaste.backend.sqlite', 'user.sql') as fh: + statement = f'''CREATE TABLE IF NOT EXISTS "{table}" ( + "sub" BLOB NOT NULL UNIQUE, + "key_hash" BLOB NOT NULL, + "paste_index" BLOB, + PRIMARY KEY("sub") + );''' - statement = fh.read() - - cursor.execute(statement) + cursor.execute(statement) connection.commit() return None -def sanitize(connection: Connection, model_class) -> int: +def sanitize(connection: Connection, table: str, model_class) -> int: return 0 diff --git a/src/httpaste/backend/sqlite/user.sql b/src/httpaste/backend/sqlite/user.sql deleted file mode 100644 index 1d5b947..0000000 --- a/src/httpaste/backend/sqlite/user.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE IF NOT EXISTS "users" ( - "sub" BLOB NOT NULL UNIQUE, - "key_hash" BLOB NOT NULL, - "paste_index" BLOB, - PRIMARY KEY("sub") -); \ No newline at end of file diff --git a/src/httpaste/context.py b/src/httpaste/context.py new file mode 100755 index 0000000..351460c --- /dev/null +++ b/src/httpaste/context.py @@ -0,0 +1,18 @@ +from typing import NamedTuple +from string import ascii_uppercase, digits, ascii_letters, punctuation +from configparser import ConfigParser + +from httpaste.helper.common import generate_random_string +from httpaste.helper.config import get_sanitized_config_charset, get_config + +class Config(NamedTuple): + """httpaste global config + """ + salt: bytes = get_sanitized_config_charset(generate_random_string( + 32, ascii_letters + digits + punctuation)).encode('utf-8') + hmac_iter: int = 20000 + + +def get_context_config(configIni: ConfigParser) -> Config: + + return get_config(configIni, 'context', Config) \ No newline at end of file diff --git a/src/httpaste/controller/__init__.py b/src/httpaste/controller/__init__.py index 2849f5b..f3ca075 100644 --- a/src/httpaste/controller/__init__.py +++ b/src/httpaste/controller/__init__.py @@ -5,12 +5,14 @@ import httpaste def get(**kwargs): config = current_app.httpaste + context = config.context + model = config.model return httpaste.__doc__.format( url=connexion.request.url, - hmac_iterations=config.hmac_iterations, - paste_lifetime=config.paste_lifetime, - paste_max_lifetime=str(round(config.paste_max_lifetime / 60)), - paste_default_encoding=config.paste_default_encoding + hmac_iterations=context.hmac_iter, + paste_lifetime=model.paste.default_lifetime, + paste_max_lifetime=str(round(model.paste.default_max_lifetime / 60)), + paste_default_encoding=model.paste.default_encoding ), 200 diff --git a/src/httpaste/controller/paste/__init__.py b/src/httpaste/controller/paste/__init__.py index c8f4b87..bd0a076 100644 --- a/src/httpaste/controller/paste/__init__.py +++ b/src/httpaste/controller/paste/__init__.py @@ -5,8 +5,10 @@ from flask import current_app from httpaste.helper.common import decode, DecodeError, join_url import httpaste.model.paste as paste_model import httpaste.model.user as user_model +from httpaste.backend import load_backend from httpaste.helper.http import BadRequestError, GoneError, NotFoundError -from httpaste.model import ( +from httpaste.helper.syntax import highlight +from httpaste.schema import ( PasteKey, PasteData, PasteLifetime, @@ -15,6 +17,13 @@ from httpaste.model import ( Sub) +class Config: + default_mime_type: str = 'text/plain' + default_linenos: bool = False + default_syntax: bool = False + default_formatter: str = 'terminal256' + + def delete(**kwargs): """ """ @@ -45,12 +54,15 @@ def get(**kwargs): """ config = current_app.httpaste + backend = load_backend(config.backend) + context = config.context + + syntax = kwargs.get('syntax') + formatter = kwargs.get('format', Config.default_formatter) + linenos = kwargs.get('linenos', Config.default_linenos) + mime = kwargs.get('mime', Config.default_mime_type) pid = PasteKey(kwargs['id'].encode('utf-8')) - syntax = kwargs.get('syntax') - formatter = kwargs.get('format', 'terminal256') - linenos = kwargs.get('linenos', False) - mime = kwargs.get('mime', 'text/plain') if kwargs.get('user') is not None: # authenticated @@ -58,26 +70,23 @@ def get(**kwargs): key = MasterKey(kwargs['token_info'].get('master_key')) sub = Sub(kwargs['token_info'].get('sub')) - pkey = user_model.load_paste_key(pid, sub, key, config.backend.user, - config.salt, config.hmac_iterations) + pkey = user_model.load_paste_key(pid, sub, key, backend.user, context) def call(): return paste_model.get_safe(pid, pkey, sub, - config.backend.paste, - config.salt, config.hmac_iterations) + config.model.paste, + backend.paste, context) else: # unauthenticated - def call(): return paste_model.get(pid, config.backend.paste, - config.salt, config.hmac_iterations) + def call(): return paste_model.get(pid, backend.paste, context) try: data, expiration, encoding = call() except paste_model.LifetimeError as e: if kwargs.get('user') is not None: - paste_model.remove_safe(pid, sub, pkey, config.backend.paste, - config.salt, config.hmac_iterations) + paste_model.remove_safe(pid, sub, pkey, backend.paste, context) else: - paste_model.remove(pid, config.backend.paste) + paste_model.remove(pid, backend.paste) raise GoneError(str(e)) from e except paste_model.NotFoundError as e: raise NotFoundError(str(e)) @@ -87,10 +96,9 @@ def get(**kwargs): # burn after read if expiration < 0: if kwargs.get('user') is not None: - paste_model.remove_safe(pid, sub, pkey, config.backend.paste, - config.salt, config.hmac_iterations) + paste_model.remove_safe(pid, sub, pkey, backend.paste, context) else: - paste_model.remove(pid, config.backend.paste) + paste_model.remove(pid, backend.paste) if syntax is not None: data = highlight(data, str(syntax), formatter, linenos) @@ -110,12 +118,14 @@ def post(**kwargs): """ config = current_app.httpaste + backend = load_backend(config.backend) + context = config.context if kwargs['body'].get('data') is None: raise BadRequestError('form field \'data\' missing.') encoding = PasteEncoding(kwargs.get('encoding', 'utf-8')) - lifetime = PasteLifetime(kwargs.get('lifetime', config.paste_lifetime)) + lifetime = PasteLifetime(kwargs.get('lifetime', config.model.paste.default_lifetime)) if encoding not in ['utf-8', 'utf-16', 'ascii']: try: @@ -135,15 +145,15 @@ def post(**kwargs): sub = Sub(kwargs['token_info'].get('sub')) pid, pkey = paste_model.create_safe(pdata, lifetime, sub, encoding, - config.backend.paste, config.salt, config.hmac_iterations) + config.model.paste, backend.paste, + context) - user_model.dump_paste_key(pid, pkey, sub, key, config.backend.user, - config.salt, config.hmac_iterations) + user_model.dump_paste_key(pid, pkey, sub, key, backend.user, context) else: # unauthenticated - pid = paste_model.create(pdata, lifetime, encoding, config.backend.paste, - config.salt, config.hmac_iterations) + pid = paste_model.create(pdata, lifetime, encoding, config.model.paste, + backend.paste, context) base_url = join_url(request.root_url, request.path) diff --git a/src/httpaste/controller/paste/private.py b/src/httpaste/controller/paste/private.py index c0274c1..9fcf05c 100644 --- a/src/httpaste/controller/paste/private.py +++ b/src/httpaste/controller/paste/private.py @@ -5,8 +5,6 @@ def search(**kwargs): """ """ - print(args) - return 'Hallo', 200 diff --git a/src/httpaste/controller/user/session.py b/src/httpaste/controller/user/session.py index f3ba67c..3639b3b 100644 --- a/src/httpaste/controller/user/session.py +++ b/src/httpaste/controller/user/session.py @@ -4,20 +4,22 @@ from flask import current_app from httpaste.helper.http import ForbiddenError from httpaste.model.user import authenticate, AuthenticationError - +from httpaste.backend import load_backend def post(*args, **kwargs): """ """ config = current_app.httpaste + backend = load_backend(config.backend) + context = config.context user_id = args[0].encode('utf-8') password = args[1].encode('utf-8') try: - return authenticate(user_id, password, config.backend.user, config.salt, config.hmac_iterations) + return authenticate(user_id, password, backend.user, context) except AuthenticationError as e: raise ForbiddenError('You shall not pass!') from e diff --git a/src/httpaste/helper/config.py b/src/httpaste/helper/config.py new file mode 100755 index 0000000..f053cf3 --- /dev/null +++ b/src/httpaste/helper/config.py @@ -0,0 +1,111 @@ +import os +from pathlib import Path +from configparser import ConfigParser, NoSectionError +from typing import Optional, NamedTuple +from os import environ + + +CONFIGPATH_ENVIRON = 'HTTPASTE_CONFIGPATH' + + +class ConfigError(Exception): + """ + """ + + +def get_sanitized_config_charset(charset: str): + + for x in ["$", "%"]: + + charset = charset.replace(x, f'{x}{x}') + + return charset + + +def typecast(obj: dict, aclass: type, dirname:Optional[Path] = None) -> dict: + """typecast a dictionary according to class annotations + + :param obj: dictionary to typecast + :param aclass: class containing typehint annotations + :param basepath: basepath for filesystem path typecasting + + :returns: typecasted dictionary + """ + + casted = {} + + for k,v in obj.items(): + + v = v.strip('\'"') + + try: + bclass = aclass.__annotations__[k] + except KeyError as e: + raise KeyError(f'{k}: not allowed') from e + + if issubclass(bclass, Path) and not str(v[0]).startswith(os.path.sep): + if not isinstance(dirname, Path): + raise TypeError('no dirname for Path type specified.') + + casted[k] = casted_val = dirname / v + elif issubclass(bclass, bytes): + + casted[k] = bclass(v.encode('utf-8')) + else: + try: + casted_val = bclass(v) + except ValueError as e: + raise ValueError(f'{k}: {e}') from e + else: + casted[k] = casted_val + + return casted + + +def get_config(configIni: ConfigParser, section: str, bclass: type, dirname: Path = None) -> object: + """get an object-oriented configuration from an INI file + + :param configIni: configparser.Configparser object with initialized stream + :param section: name of section to get configuration for + :param bclass: configuration base class + :param dirname: directory name of INI file stream + + :returns: initialized configuration instance + """ + + try: + raw_config = dict(configIni.items(section)) + except NoSectionError as e: + raw_config = {} + + try: + casted_config = typecast(raw_config, bclass, dirname) + except KeyError as e: + raise ConfigError(f'[{section}] {e}') from e + except ValueError as e: + raise ConfigError(f'[{section}] {e}') from e + + try: + config = bclass(**casted_config) + except TypeError as e: + raise ConfigError(f'[{section}] {e}') from e + + return config + + + +def get_configparser(path: Path = None, var_name: str = CONFIGPATH_ENVIRON): + """ + """ + + if path is None: + try: + path = environ[var_name] + except KeyError as e: + raise ConfigError( + f'environment variable \'{var_name}\' not set.') from e + + configIni = ConfigParser() + configIni.read(path) + + return configIni, path \ No newline at end of file diff --git a/src/httpaste/helper/crypto.py b/src/httpaste/helper/crypto.py index 0a8d525..e4febea 100755 --- a/src/httpaste/helper/crypto.py +++ b/src/httpaste/helper/crypto.py @@ -8,7 +8,7 @@ from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC from cryptography.fernet import Fernet, InvalidToken -from httpaste import Config +from httpaste.context import Config DEFAULT_HMAC_ITERATIONS = 20000 @@ -38,7 +38,7 @@ def dhash(data: bytes): return hashlib.sha512(data).digest() -def derive_key(main_key: str, salt: bytes = Config.salt, iterations:int=DEFAULT_HMAC_ITERATIONS) -> bytes: +def derive_key(main_key: str, salt: bytes, iterations:int=DEFAULT_HMAC_ITERATIONS) -> bytes: """derive a key from a main key :param main_key: main key to derive from diff --git a/src/httpaste/model/__init__.py b/src/httpaste/model/__init__.py index 7d1280f..8383a63 100644 --- a/src/httpaste/model/__init__.py +++ b/src/httpaste/model/__init__.py @@ -1,144 +1,19 @@ """Model """ -from typing import NamedTuple, Optional, Dict, Union, Any, TypedDict +from typing import NamedTuple +from configparser import ConfigParser +from pathlib import Path + +from httpaste.model.paste import Config as PasteConfig +from httpaste.model.paste import get_paste_model_config + +class Config(NamedTuple): + """Model Configuration""" + paste: PasteConfig -class PasteDataSchema: - """Paste Interface schema between Model and Backend - """ - pid = bytes - data = bytes - data_hash = bytes - sub = bytes - timestamp = int - lifetime = int - expiration = int - encoding = str +def get_model_config(configIni: ConfigParser, path:Path) -> Config: + paste_config = get_paste_model_config(configIni) -class UserDataSchema: - """User Interface Schema between Model and Backend - """ - sub = bytes - key_hash = bytes - index = bytes - - -class Backend(object): - """Backend - """ - parameter_class: str - - -class Salt(bytes): - """Salt - """ - - -class PasteData(PasteDataSchema.data): - """Paste Data - """ - - -class PasteHash(PasteDataSchema.data_hash): - """Paste Data Hash - """ - - -class PasteTimestamp(PasteDataSchema.timestamp): - """Paste Timestamp - """ - - -class PasteEncoding(PasteDataSchema.encoding): - """ - """ - - -class PasteExpiration(PasteDataSchema.expiration): - """Paste Expiration - - < 0: after first acccess - 0: never - """ - - -class PasteLifetime(PasteDataSchema.lifetime): - """Paste Lifetime - """ - - -class PasteSub(PasteDataSchema.sub): - """Hashed user id - """ - - -class KeyHash(UserDataSchema.key_hash): - """User Master Key Hash - """ - - -class PasteKey(bytes): - """Paste encryption key - """ - - -class PasteId(PasteDataSchema.pid): - """Paste unique identifier - """ - - -class MasterKey(bytes): - """User's master encryption key - """ - - -class Sub(UserDataSchema.sub): - """User id - """ - - -class Index(TypedDict): - """User Paste Index - """ - auth_expires: int - pastes: Dict[str, Dict[str, Any]] - - -class SerializedIndex(UserDataSchema.index): - """User Paste Index (serialized) - """ - - -class User(NamedTuple): - """Global User Model (and Prototype) - - non-optional values are prototype values - """ - - #: user id - sub: Sub - #: user's master key hash - key_hash: Optional[KeyHash] = None - #: user's paste index - index: Optional[Union[Index, SerializedIndex]] = None - - -class Paste(NamedTuple): - """Global Paste Model (and Prototype) - - non-optional values are prototype values - """ - - #: paste id - pid: PasteId - #: paste owner - sub: Optional[PasteSub] = None - #: paste data - data: Optional[PasteData] = None - #: paste data hash - data_hash: Optional[PasteHash] = None - #: paste timestamp - expiration: Optional[PasteExpiration] = None - #: paste encoding - encoding: Optional[PasteEncoding] = None + return Config(paste=paste_config) \ No newline at end of file diff --git a/src/httpaste/model/paste.py b/src/httpaste/model/paste.py index f7f856d..3e030ef 100755 --- a/src/httpaste/model/paste.py +++ b/src/httpaste/model/paste.py @@ -2,15 +2,30 @@ """paste model interface """ import json -from typing import Optional, Tuple +from typing import Optional, Tuple, NamedTuple import time +from configparser import ConfigParser +from string import ascii_uppercase, digits, ascii_letters, punctuation -from httpaste import Config + +from httpaste.context import Config as ContextConfig from httpaste.helper.crypto import dhash, shash, encrypt, decrypt +from httpaste.helper.config import get_sanitized_config_charset, get_config from httpaste.helper.common import generate_random_string -from httpaste.model import (Paste, PasteId, Sub, MasterKey, PasteKey, Salt, - PasteData, PasteHash, PasteTimestamp, PasteSub, - PasteLifetime, PasteEncoding, PasteExpiration) +from httpaste.schema import (Paste, PasteId, Sub, MasterKey, PasteKey, Salt, + PasteData, PasteHash, PasteTimestamp, PasteSub, + PasteLifetime, PasteEncoding, PasteExpiration) + + +class Config(NamedTuple): + id_size: int = 8 + id_charset: str = ascii_letters + digits + key_size: int = 32 + key_charset: str = get_sanitized_config_charset(ascii_letters + digits + punctuation) + default_lifetime: int = 5 + default_max_lifetime: int = 1440 + default_min_lifetime: int = 1 + default_encoding: str = 'utf-8' class NotFoundError(Exception): @@ -38,9 +53,7 @@ class BackendError(Exception): """ -def generate_paste_id( - length: int = Config.paste_id_size, - charset: str = Config.paste_id_charset) -> bytes: +def generate_paste_id(length: int, charset: str) -> bytes: """generate a paste id :param length: length of id @@ -50,9 +63,7 @@ def generate_paste_id( return generate_random_string(length, charset).encode('utf-8') -def generate_paste_key( - length: int = Config.paste_key_size, - charset: str = Config.paste_key_charset) -> bytes: +def generate_paste_key(length: int, charset: str) -> bytes: """generate a paste encryption key :param length: length of key @@ -98,8 +109,7 @@ def load_safe( proto: Paste, key: PasteKey, backend: object, - salt: Salt = Config.salt, - hmac_iter: int = Config.hmac_iterations): + context: ContextConfig): """load an encrypted paste model :param proto: paste model prototype @@ -109,7 +119,7 @@ def load_safe( model = load(proto, backend) - data = decrypt(model.data, key, salt, hmac_iter) + data = decrypt(model.data, key, context.salt, context.hmac_iter) if model.data_hash and dhash(data) != model.data_hash: @@ -131,10 +141,7 @@ def dump(model: Paste, backend: object) -> None: :param backend: model backend object """ - try: - backend.dump(model) - except Exception as e: - raise BackendError(str(e)) from e + backend.dump(model) def delete(proto: Paste, backend: object) -> None: @@ -158,13 +165,12 @@ def delete_safe( proto: Paste, key: PasteKey, backend: object, - salt: Salt = Config.salt, - hmac_iter: int = Config.hmac_iterations) -> None: + context: ContextConfig) -> None: """ """ try: - model = load_safe(proto, key, backend, salt, hmac_iter) + model = load_safe(proto, key, backend, context) except LifetimeError: pass @@ -177,9 +183,9 @@ def create( data: PasteData, lifetime: PasteLifetime, encoding: PasteEncoding, + config: Config, backend: object, - salt: Salt = Config.salt, - hmac_iter: int = Config.hmac_iterations) -> PasteId: + context: ContextConfig) -> PasteId: """create an unencrypted paste :param data: paste data @@ -187,18 +193,20 @@ def create( :param backend: model backend object """ - pid = PasteId(generate_paste_id()) + pid = PasteId(generate_paste_id(config.id_size, config.id_charset)) safe_pid = PasteId(dhash(pid)) data_hash = PasteHash(dhash(data)) sub = None timestamp = PasteTimestamp(int(time.time())) - if lifetime < 0: + if lifetime is None: + lifetime = config.default_lifetime + elif lifetime < 0: expiration = -1 else: expiration = PasteExpiration(timestamp + (lifetime * 60)) - safe_data = PasteData(encrypt(data, pid, salt, hmac_iter)) + safe_data = PasteData(encrypt(data, pid, context.salt, context.hmac_iter)) model = Paste( safe_pid, @@ -217,9 +225,9 @@ def create_safe(data: PasteData, lifetime: PasteLifetime, sub: Sub, encoding: PasteEncoding, + config: Config, backend: object, - salt: Salt = Config.salt, - hmac_iter: int = Config.hmac_iterations) -> Tuple[PasteId,PasteKey]: + context: ContextConfig) -> Tuple[PasteId,PasteKey]: """create an encrypted paste :param data: paste data @@ -229,19 +237,21 @@ def create_safe(data: PasteData, :param salt: randomization salt """ - pid = PasteId(generate_paste_id()) + pid = PasteId(generate_paste_id(config.id_size, config.id_charset)) safe_pid = PasteId(dhash(pid)) - pkey = PasteKey(generate_paste_key()) + pkey = PasteKey(generate_paste_key(config.key_size, config.key_charset)) data_hash = PasteHash(dhash(data)) safe_sub = PasteSub(shash(sub, data_hash, pid)) timestamp = PasteTimestamp(int(time.time())) - if lifetime < 0: + if lifetime is None: + lifetime = config.default_lifetime + elif lifetime < 0: expiration = -1 else: expiration = PasteExpiration(timestamp + (lifetime * 60)) - safe_data = PasteData(encrypt(data, pkey, salt, hmac_iter)) + safe_data = PasteData(encrypt(data, pkey, context.salt, context.hmac_iter)) dump(Paste( safe_pid, @@ -269,21 +279,20 @@ def remove_safe( sub: Sub, key: PasteKey, backend: object, - salt: Salt = Config.salt, - hmac_iter: int = Config.hmac_iterations): + context: ContextConfig): proto = Paste(pid, sub) - delete_safe(proto, key, backend, salt, hmac_iter) + delete_safe(proto, key, backend, context) -def get(pid: PasteId, backend: object, salt: Salt = Config.salt, hmac_iter: int = Config.hmac_iterations) -> PasteData: +def get(pid: PasteId, backend: object, context: ContextConfig) -> PasteData: """conveniently load an unencrypted paste """ model = load(Paste(pid), backend) - data = decrypt(model.data, pid, salt, hmac_iter) + data = decrypt(model.data, pid, context.salt, context.hmac_iter) return PasteData(data), model.expiration, model.encoding @@ -292,12 +301,22 @@ def get_safe( pid: PasteId, pkey: PasteKey, sub: Sub, + config: Config, backend: object, - salt: Salt = Config.salt, - hmac_iter: int = Config.hmac_iterations) -> PasteData: + context: ContextConfig) -> PasteData: """conveniently load an encrypted paste """ - model = load_safe(Paste(pid, sub), pkey, backend, salt, hmac_iter) + model = load_safe(Paste(pid, sub), pkey, backend, context) return PasteData(model.data), model.expiration, model.encoding + + +def get_paste_model_config(configIni: ConfigParser) -> Config: + + return get_config(configIni, 'model.paste', Config) + + +__all__ = [ + get_paste_model_config +] \ No newline at end of file diff --git a/src/httpaste/model/user.py b/src/httpaste/model/user.py index b925063..c55724f 100755 --- a/src/httpaste/model/user.py +++ b/src/httpaste/model/user.py @@ -5,7 +5,7 @@ import json from time import time from typing import Optional -from httpaste import Config +from httpaste.context import Config as ContextConfig from httpaste.helper.crypto import ( dhash, shash, @@ -13,7 +13,7 @@ from httpaste.helper.crypto import ( decrypt, derive_key, DecryptionError) -from httpaste.model import ( +from httpaste.schema import ( User, KeyHash, Index, @@ -39,8 +39,7 @@ def _load( proto: User, master_key: str, backend: object, - salt: Salt = Config.salt, - hmac_iter: int = Config.hmac_iterations) -> Optional[User]: + context: ContextConfig) -> Optional[User]: """load user model :param model: user model prototype @@ -55,7 +54,7 @@ def _load( return None try: - serialized_data = decrypt(model.index, master_key, salt, hmac_iter) + serialized_data = decrypt(model.index, master_key, context.salt, context.hmac_iter) except DecryptionError as e: raise IndexError('unable to decrypt user index') from e else: @@ -71,8 +70,7 @@ def _dump( model: User, key: MasterKey, backend: object, - salt: Salt = Config.salt, - hmac_iter: int = Config.hmac_iterations) -> None: + context: ContextConfig) -> None: """dump a user model :param model: user model @@ -87,7 +85,7 @@ def _dump( serialized_index = json.dumps(model.index).encode('utf-8') - safe_index = SerializedIndex(encrypt(serialized_index, key, salt, hmac_iter)) + safe_index = SerializedIndex(encrypt(serialized_index, key, context.salt, context.hmac_iter)) backend.dump(User(*model[:-1], safe_index)) @@ -96,7 +94,8 @@ def load_paste_key( pid: PasteId, sub: Sub, key: MasterKey, - backend: object, salt: Salt = Config.salt, hmac_iter: int = Config.hmac_iterations) -> Optional[PasteKey]: + backend: object, + context: ContextConfig) -> Optional[PasteKey]: """load a user paste key :param pid: paste id @@ -106,7 +105,7 @@ def load_paste_key( :param salt: randomization salt """ - model = _load(User(sub), key, backend, salt, hmac_iter) + model = _load(User(sub), key, backend, context) for k, v in model.index.get('pastes').items(): @@ -123,8 +122,7 @@ def dump_paste_key( sub: Sub, key: MasterKey, backend: object, - salt: str = Config.salt, - hmac_iter: int = Config.hmac_iterations) -> None: + context: ContextConfig) -> None: """dump a user paste key :param pid: paste id @@ -134,21 +132,20 @@ def dump_paste_key( :param backend: user model backend """ - model = _load(User(sub), key, backend, salt, hmac_iter) + model = _load(User(sub), key, backend, context) model.index.setdefault('pastes', {})[pid.hex()] = { 'key': pkey.hex() } - _dump(model, key, backend, salt, hmac_iter) + _dump(model, key, backend, context) def authenticate( user_id: bytes, password: bytes, backend: object, - salt: Salt = Config.salt, - hmac_iter: int = Config.hmac_iterations): + context: ContextConfig): """authenticate a user :param user_id: human-readable user id @@ -156,7 +153,7 @@ def authenticate( """ sub = Sub(dhash(user_id)) - key = MasterKey(derive_key(password, salt, hmac_iter)) + key = MasterKey(derive_key(password, context.salt, context.hmac_iter)) key_hash = KeyHash(dhash(key)) proto = User(sub) @@ -164,7 +161,7 @@ def authenticate( bogus_decline_msg = 'unable to authenticate' try: - model = _load(proto, key, backend, salt, hmac_iter) + model = _load(proto, key, backend, context) except IndexError as e: raise AuthenticationError(bogus_decline_msg) from e @@ -175,7 +172,7 @@ def authenticate( } model = User(sub, key_hash, Index(data)) - _dump(model, key, backend, salt, hmac_iter) + _dump(model, key, backend, context) else: if model.key_hash != key_hash: diff --git a/src/httpaste/schema/__init__.py b/src/httpaste/schema/__init__.py index e69de29..6d9e310 100644 --- a/src/httpaste/schema/__init__.py +++ b/src/httpaste/schema/__init__.py @@ -0,0 +1,137 @@ +from typing import NamedTuple, Optional, Dict, Union, Any, TypedDict + + +class PasteDataSchema: + """Paste Interface schema between Model and Backend + """ + pid = bytes + data = bytes + data_hash = bytes + sub = bytes + timestamp = int + lifetime = int + expiration = int + encoding = str + + +class UserDataSchema: + """User Interface Schema between Model and Backend + """ + sub = bytes + key_hash = bytes + index = bytes + + + +class Salt(bytes): + """Salt + """ + + +class PasteData(PasteDataSchema.data): + """Paste Data + """ + + +class PasteHash(PasteDataSchema.data_hash): + """Paste Data Hash + """ + + +class PasteTimestamp(PasteDataSchema.timestamp): + """Paste Timestamp + """ + + +class PasteEncoding(PasteDataSchema.encoding): + """ + """ + + +class PasteExpiration(PasteDataSchema.expiration): + """Paste Expiration + + < 0: after first acccess + 0: never + """ + + +class PasteLifetime(PasteDataSchema.lifetime): + """Paste Lifetime + """ + + +class PasteSub(PasteDataSchema.sub): + """Hashed user id + """ + + +class KeyHash(UserDataSchema.key_hash): + """User Master Key Hash + """ + + +class PasteKey(bytes): + """Paste encryption key + """ + + +class PasteId(PasteDataSchema.pid): + """Paste unique identifier + """ + + +class MasterKey(bytes): + """User's master encryption key + """ + + +class Sub(UserDataSchema.sub): + """User id + """ + + +class Index(TypedDict): + """User Paste Index + """ + auth_expires: int + pastes: Dict[str, Dict[str, Any]] + + +class SerializedIndex(UserDataSchema.index): + """User Paste Index (serialized) + """ + + +class User(NamedTuple): + """Global User Model (and Prototype) + + non-optional values are prototype values + """ + + #: user id + sub: Sub + #: user's master key hash + key_hash: Optional[KeyHash] = None + #: user's paste index + index: Optional[Union[Index, SerializedIndex]] = None + + +class Paste(NamedTuple): + """Global Paste Model (and Prototype) + + non-optional values are prototype values + """ + + #: paste id + pid: PasteId + #: paste owner + sub: Optional[PasteSub] = None + #: paste data + data: Optional[PasteData] = None + #: paste data hash + data_hash: Optional[PasteHash] = None + #: paste timestamp + expiration: Optional[PasteExpiration] = None + #: paste encoding + encoding: Optional[PasteEncoding] = None \ No newline at end of file diff --git a/src/httpaste/server.py b/src/httpaste/server.py new file mode 100755 index 0000000..55c0542 --- /dev/null +++ b/src/httpaste/server.py @@ -0,0 +1,17 @@ +from typing import NamedTuple +from configparser import ConfigParser + + +from httpaste.helper.config import get_config + + +class Config(NamedTuple): + """connexion config + """ + swagger_ui: bool = True + bind_address: str = None + + +def get_server_config(configIni: ConfigParser) -> Config: + + return get_config(configIni, 'server', Config) \ No newline at end of file From dd187a10691792fb427be2247bfadea9eddbbc60 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 01:53:53 +0200 Subject: [PATCH 17/55] test: init --- tests/httpaste/backend/__init__.py | 0 tests/httpaste/backend/test__init__.py | 134 ++++++++++++++++++++++ tests/httpaste/helper/__init__.py | 0 tests/httpaste/helper/test_config.py | 149 +++++++++++++++++++++++++ tests/httpaste/model/test_paste.py | 79 +++++++++++++ 5 files changed, 362 insertions(+) create mode 100644 tests/httpaste/backend/__init__.py create mode 100755 tests/httpaste/backend/test__init__.py create mode 100644 tests/httpaste/helper/__init__.py create mode 100644 tests/httpaste/helper/test_config.py create mode 100755 tests/httpaste/model/test_paste.py diff --git a/tests/httpaste/backend/__init__.py b/tests/httpaste/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/httpaste/backend/test__init__.py b/tests/httpaste/backend/test__init__.py new file mode 100755 index 0000000..b10af79 --- /dev/null +++ b/tests/httpaste/backend/test__init__.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +import pytest +from textwrap import dedent +from unittest.mock import mock_open, patch +from configparser import ConfigParser +from pathlib import Path + +@pytest.fixture +def module(): + + from httpaste import backend + + return backend + + +class Test_get_backend_config(): + + @pytest.fixture(autouse=True) + def setup(self, module): + + self.func = module.get_backend_config + + def test_default_file(self, module): + + data = dedent(""" + [backend] + type = file + + [backend.file] + base_dirname = 'sample_data' + """) + + path = Path('/foo') + + configIni = ConfigParser() + + with patch('builtins.open', mock_open(read_data=data)): + + configIni.read(str(path)) + + config = self.func(configIni, path) + + assert isinstance(config, module.Config) + assert issubclass(config.interface, module.BackendInterface) + assert str(config.config.base_dirname) == '/foo/sample_data' + + def test_sqlite(self, module): + + data = dedent(""" + [backend] + type = sqlite + + [backend.sqlite] + uri = 'foobar.db' + """) + + configIni = ConfigParser() + + with patch('builtins.open', mock_open(read_data=data)): + + configIni.read('void') + + config = self.func(configIni, Path('/foo')) + + assert str(config.config.uri) == '/foo/foobar.db' + + def test_mysql(self, module): + + data = dedent(""" + [backend] + type = mysql + + [backend.mysql] + user = 'foo' + password = bar + host = manana + database = test + """) + + configIni = ConfigParser() + + with patch('builtins.open', mock_open(read_data=data)): + + configIni.read('void') + + config = self.func(configIni, Path('/foo')) + + assert config.config.user == 'foo' + assert config.config.password == 'bar' + assert config.config.host == 'manana' + assert config.config.database == 'test' + + + +#class Test_load(): +# +# @pytest.fixture(autouse=True) +# def setup(self, module): +# +# self.func = module.load +# +# def test_missing_parameter(self, module): +# +# config = module.Config() +# config.name = 'file' +# config.parameters = {} +# +# with pytest.raises(module.BackendError): +# self.func(config) +# +# def test_unknown_parameter(self, module): +# +# config = module.Config() +# config.name = 'file' +# config.parameters = { +# 'base_dirname': 'foofoo', +# 'foo': 'bar' +# } +# +# with pytest.raises(module.BackendError): +# self.func(config) +# +# def test_file(self, module): +# +# config = module.Config() +# config.name = 'file' +# config.parameters = { +# 'base_dirname': 'foofoo', +# 'user_dirnamea': 'test' +# } +# +# backend = self.func(config) +# +# assert isinstance(backend, module.BackendInterface) \ No newline at end of file diff --git a/tests/httpaste/helper/__init__.py b/tests/httpaste/helper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/httpaste/helper/test_config.py b/tests/httpaste/helper/test_config.py new file mode 100644 index 0000000..d36355e --- /dev/null +++ b/tests/httpaste/helper/test_config.py @@ -0,0 +1,149 @@ +import pytest +from typing import NamedTuple +from unittest.mock import mock_open, patch +from textwrap import dedent +from pathlib import Path +from configparser import ConfigParser + + + +@pytest.fixture +def module(): + + from httpaste.helper import config + + return config + + +@pytest.fixture +def mock_aclass(): + + class Foobar(NamedTuple): + foo: int + bar: str = 'test' + + return Foobar + + +@pytest.fixture +def mock_aclass_special(): + + class Foobar(NamedTuple): + foobar: Path + + return Foobar + + +class Test_typecast(): + + @pytest.fixture(autouse=True) + def setup(self, module, mock_aclass): + + self.func = module.typecast + + self.mock_aclass = mock_aclass + + def test_default(self, module): + + foobar = { + 'foo': '45' + } + + result = self.func(foobar, self.mock_aclass) + + assert isinstance(result, dict) + + assert result['foo'] == 45 + assert result.get('bar') is None + + + def test_type_mismatch(self, module): + + foobar = { + 'foo': 'foobar' + } + + with pytest.raises(ValueError): + + self.func(foobar, self.mock_aclass) + + + def test_unknown_key(self, module): + + foobar = { + 'foo': '45', + 'foobar': 'foobar' + } + + with pytest.raises(KeyError): + + self.func(foobar, self.mock_aclass) + + +class Test_get_config(): + + @pytest.fixture(autouse=True) + def setup(self, module, mock_aclass): + + self.func = module.get_config + + self.mock_aclass = mock_aclass + + def test_default(self): + + data = dedent(""" + [foobar] + foo = 45 + """) + + configIni = ConfigParser() + + with patch('builtins.open', mock_open(read_data=data)): + + configIni.read(str('void')) + + result = self.func(configIni, 'foobar', self.mock_aclass) + + assert isinstance(result, self.mock_aclass) + + assert result.foo == 45 + + def test_relative_path(self, mock_aclass_special): + + data = dedent(""" + [foobar] + foobar = 'bar/foo' + """) + + dirname = Path('/foo/bar') + + configIni = ConfigParser() + + with patch('builtins.open', mock_open(read_data=data)): + + configIni.read(str('void')) + + result = self.func(configIni, 'foobar', mock_aclass_special, dirname) + + assert isinstance(result, mock_aclass_special) + assert isinstance(result.foobar, Path) + assert str(result.foobar) == '/foo/bar/bar/foo' + + def test_absolute_path(self, mock_aclass_special): + + data = dedent(""" + [foobar] + foobar = '/bar/foo' + """) + + configIni = ConfigParser() + + with patch('builtins.open', mock_open(read_data=data)): + + configIni.read(str('void')) + + result = self.func(configIni, 'foobar', mock_aclass_special) + + assert isinstance(result, mock_aclass_special) + assert isinstance(result.foobar, Path) + assert str(result.foobar) == '/bar/foo' \ No newline at end of file diff --git a/tests/httpaste/model/test_paste.py b/tests/httpaste/model/test_paste.py new file mode 100755 index 0000000..aafd415 --- /dev/null +++ b/tests/httpaste/model/test_paste.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +import pytest +from textwrap import dedent +from unittest.mock import mock_open, patch +from configparser import ConfigParser +from pathlib import Path + +@pytest.fixture +def module(): + + from httpaste.model import paste + + return paste + + +class Test_get_paste_model_config(): + + @pytest.fixture(autouse=True) + def setup(self, module): + + self.func = module.get_paste_model_config + + def test_default(self, module): + + data = '' + + configIni = ConfigParser() + + with patch('builtins.open', mock_open(read_data=data)): + + configIni.read('void') + + result = self.func(configIni) + + assert isinstance(result, module._Config) + assert isinstance(result.id_size, int), result.id_size + assert isinstance(result.key_size, int), result.key_size + + +#class Test_load(): +# +# @pytest.fixture(autouse=True) +# def setup(self, module): +# +# self.func = module.load +# +# def test_missing_parameter(self, module): +# +# config = module.Config() +# config.name = 'file' +# config.parameters = {} +# +# with pytest.raises(module.BackendError): +# self.func(config) +# +# def test_unknown_parameter(self, module): +# +# config = module.Config() +# config.name = 'file' +# config.parameters = { +# 'base_dirname': 'foofoo', +# 'foo': 'bar' +# } +# +# with pytest.raises(module.BackendError): +# self.func(config) +# +# def test_file(self, module): +# +# config = module.Config() +# config.name = 'file' +# config.parameters = { +# 'base_dirname': 'foofoo', +# 'user_dirnamea': 'test' +# } +# +# backend = self.func(config) +# +# assert isinstance(backend, module.BackendInterface) \ No newline at end of file From 65f3102a7ff681f464cb3cbc47e0afa8a2d21d9d Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 01:56:12 +0200 Subject: [PATCH 18/55] refactor(samples/httpaste.it): update config file --- samples/httpaste.it/httpaste/config.ini | 36 ++++++++++++++++++++----- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/samples/httpaste.it/httpaste/config.ini b/samples/httpaste.it/httpaste/config.ini index 801d0d6..1e6b9b5 100644 --- a/samples/httpaste.it/httpaste/config.ini +++ b/samples/httpaste.it/httpaste/config.ini @@ -1,16 +1,38 @@ -[general] +[context] salt = '&)UxB-_$Lk$m=CB}dw[d85{-ZWR?uUNx' -paste_id_size = 8 -paste_key_size = 32 -paste_lifetime = 5 -paste_max_lifetime = 1440 -hmac_iterations = 20000 -paste_default_encoding = 'utf-8' +hmac_iter = 20000 + +[model.paste] +default_encoding = 'utf-8' +id_size = 8 +key_size = 32 +default_lifetime = 5 +default_max_lifetime = 1440 + + +[controller.paste] +default_mime_type = 'text/plain' +default_linenos = False +default_syntax = False +default_formatter = 'terminal256' + [backend] type = file + +[backend.file] base_dirname = 'sample_data' +[backend.sqlite] +path = 'devel/sample.db' + +[backend.mysql] +user = 'example-user' +password = 'my_cool_secret' +database = 'httpaste' +host = '127.0.0.1' + + [server] swagger_ui = False bind_address = 'sample.sock' \ No newline at end of file From 843354e18dc6ed7f7273866a81a3b37e296b6647 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 02:12:35 +0200 Subject: [PATCH 19/55] fix(cgi): remove redundant function --- src/httpaste/cgi.py | 4 ++-- src/httpaste/fcgi.py | 4 ++-- src/httpaste/wsgi.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/httpaste/cgi.py b/src/httpaste/cgi.py index c28d730..417e8a1 100755 --- a/src/httpaste/cgi.py +++ b/src/httpaste/cgi.py @@ -2,9 +2,9 @@ """httpaste CGI entrypoint """ from wsgiref.handlers import CGIHandler -from httpaste import load_config, get_flask_app, get_config_path +from httpaste import load_config, get_flask_app -config, server_config = load_config(get_config_path()) +config, server_config = load_config() application = get_flask_app(config, server_config) diff --git a/src/httpaste/fcgi.py b/src/httpaste/fcgi.py index e193e16..256fbc6 100755 --- a/src/httpaste/fcgi.py +++ b/src/httpaste/fcgi.py @@ -2,9 +2,9 @@ """httpaste FastCGI entrypoint """ from flup.server.fcgi import WSGIServer -from httpaste import load_config, get_flask_app, get_config_path +from httpaste import load_config, get_flask_app -config, server_config = load_config(get_config_path()) +config, server_config = load_config() application = get_flask_app(config, server_config) diff --git a/src/httpaste/wsgi.py b/src/httpaste/wsgi.py index 9e43ff0..30e513c 100755 --- a/src/httpaste/wsgi.py +++ b/src/httpaste/wsgi.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """httpaste WSGI entrypoint """ -from httpaste import load_config, get_flask_app, get_config_path +from httpaste import load_config, get_flask_app -config, server_config = load_config(get_config_path()) +config, server_config = load_config() application = get_flask_app(config, server_config) From 0fb50c5a575d2f9993cf10934c7ceadeffdeefc2 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 02:28:02 +0200 Subject: [PATCH 20/55] fix(helper/config::typecast): add boolean evaluation --- src/httpaste/helper/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/httpaste/helper/config.py b/src/httpaste/helper/config.py index f053cf3..29795e3 100755 --- a/src/httpaste/helper/config.py +++ b/src/httpaste/helper/config.py @@ -3,6 +3,7 @@ from pathlib import Path from configparser import ConfigParser, NoSectionError from typing import Optional, NamedTuple from os import environ +from ast import literal_eval CONFIGPATH_ENVIRON = 'HTTPASTE_CONFIGPATH' @@ -49,8 +50,9 @@ def typecast(obj: dict, aclass: type, dirname:Optional[Path] = None) -> dict: casted[k] = casted_val = dirname / v elif issubclass(bclass, bytes): - casted[k] = bclass(v.encode('utf-8')) + elif issubclass(bclass, bool): + casted[k] = literal_eval(v) else: try: casted_val = bclass(v) From 49604c1e37eeb1c72554042082c1ca4db12faf5a Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 02:33:28 +0200 Subject: [PATCH 21/55] fix(__init__::load_config): override path --- src/httpaste/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index a4aafd2..ab8c190 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -194,7 +194,7 @@ def load_config(path: str = None, var_name: str = CONFIGPATH_ENVIRON): """ """ - configIni, _ = get_configparser(path, var_name) + configIni, path = get_configparser(path, var_name) return get_config(configIni, Path(path).resolve().parent) @@ -203,6 +203,8 @@ def get_flask_app(config: Config) -> FlaskApp: """get a flask app object """ + print(config.server.swagger_ui) + options = {"swagger_ui": config.server.swagger_ui} #context manager returns a pathlib.Path object From 9845c85510f61eb6db90df2a5bf49dc76c16e253 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 02:34:24 +0200 Subject: [PATCH 22/55] fix(cgi): adapt load_config() signature --- src/httpaste/cgi.py | 4 ++-- src/httpaste/fcgi.py | 4 ++-- src/httpaste/wsgi.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/httpaste/cgi.py b/src/httpaste/cgi.py index 417e8a1..3c380c5 100755 --- a/src/httpaste/cgi.py +++ b/src/httpaste/cgi.py @@ -4,8 +4,8 @@ from wsgiref.handlers import CGIHandler from httpaste import load_config, get_flask_app -config, server_config = load_config() +config = load_config() -application = get_flask_app(config, server_config) +application = get_flask_app(config) CGIHandler().run(application) diff --git a/src/httpaste/fcgi.py b/src/httpaste/fcgi.py index 256fbc6..5ea3eab 100755 --- a/src/httpaste/fcgi.py +++ b/src/httpaste/fcgi.py @@ -4,9 +4,9 @@ from flup.server.fcgi import WSGIServer from httpaste import load_config, get_flask_app -config, server_config = load_config() +config = load_config() -application = get_flask_app(config, server_config) +application = get_flask_app(config) if __name__ == '__main__': diff --git a/src/httpaste/wsgi.py b/src/httpaste/wsgi.py index 30e513c..94887c1 100755 --- a/src/httpaste/wsgi.py +++ b/src/httpaste/wsgi.py @@ -3,6 +3,6 @@ """ from httpaste import load_config, get_flask_app -config, server_config = load_config() +config = load_config() -application = get_flask_app(config, server_config) +application = get_flask_app(config) From 42ccaaccc6418cf7d4432281f2d232c826947ff5 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 03:04:15 +0200 Subject: [PATCH 23/55] fix(samples/httpasteit): upgrade docker compose version --- samples/httpaste.it/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/httpaste.it/docker-compose.yml b/samples/httpaste.it/docker-compose.yml index a334189..2150c79 100644 --- a/samples/httpaste.it/docker-compose.yml +++ b/samples/httpaste.it/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.3" +version: "3.4" services: httpaste: build: @@ -37,4 +37,4 @@ services: volumes: - ./tor/etc/tor/torrc:/etc/tor/torrc volumes: - system-shared: \ No newline at end of file + system-shared: From b83d0a3614996f3845d02874242b19268b11cb11 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 03:33:33 +0200 Subject: [PATCH 24/55] docs: update README --- README.md | 2 +- docs/README.rst | 4 +++- docs/guide/backend.rst | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8be40cf..636d05d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![](docs/_assets/images/favpng_parrot-royalty-free-cartoon.png) -**NOTE**: httpaste is publicly hosted at [httpaste.it](http://httpaste.it) and as a hidden Tor service ([https://paste77ubkwxy4fqezffsmthxdh3xerwi72tlsw2mch7ecjhw2xn7iyd.onion](https://paste77ubkwxy4fqezffsmthxdh3xerwi72tlsw2mch7ecjhw2xn7iyd.onion)). +**NOTE**: httpaste is publicly hosted at [httpaste.it](http://httpaste.it) and as a [Tor Onion Service](https://community.torproject.org/onion-services/overview/) ([http://paste77ubkwxy4fqezffsmthxdh3xerwi72tlsw2mch7ecjhw2xn7iyd.onion](http://paste77ubkwxy4fqezffsmthxdh3xerwi72tlsw2mch7ecjhw2xn7iyd.onion)). Both services are to be considered evaluatory, as long as the source code is in pre-release. Regarding voidance of pre-release status, see [Open Issues](https://victorykit.atlassian.net/issues/?jql=project%20%3D%20HTTPASTE%20AND%20fixVersion%20in%20(1.1.0-beta%2C%201.2.0-beta%2C%201.3.0)), for more information. diff --git a/docs/README.rst b/docs/README.rst index 96f45f3..c7099d1 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -10,7 +10,7 @@ httpaste - versatile HTTP pastebin .. image:: _assets/images/favpng_parrot-royalty-free-cartoon.png .. note:: - httpaste is publicly hosted at `httpaste.it`_ and as a hidden Tor service (``_). + httpaste is publicly hosted at `httpaste.it`_ and as a `Tor Onion Service`_ (``_). Both services are to be considered evaluatory, as long as the source code is in pre-release. Regarding voidance of pre-release status, see `Open Issues`_, for more information. @@ -79,6 +79,8 @@ This program uses licensed third-party software. ARCHITECTURE CONTRIBUTING + +.. _Tor Onion Service: https://community.torproject.org/onion-services/overview/ .. _ix.io: http://ix.io/ .. _sprunge.us: http://sprunge.us .. _pygments: https://pygments.org/ diff --git a/docs/guide/backend.rst b/docs/guide/backend.rst index a09c3e9..a30917a 100644 --- a/docs/guide/backend.rst +++ b/docs/guide/backend.rst @@ -6,17 +6,17 @@ The backend can be configured within the `[backend]` section of the configuratio SQLite ------ -.. autoclass:: httpaste.backend.sqlite.Parameters +.. autoclass:: httpaste.backend.sqlite.Config :members: Filesystem ---------- -.. autoclass:: httpaste.backend.file.Parameters +.. autoclass:: httpaste.backend.file.Config :members: MySQL ----- -.. autoclass:: httpaste.backend.mysql.Parameters +.. autoclass:: httpaste.backend.mysql.Config :members: \ No newline at end of file From 5288c64cbbf0b68998bcf23070290d6829c57546 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 03:38:18 +0200 Subject: [PATCH 25/55] docs(init): update references --- src/httpaste/__init__.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index ab8c190..9401ac3 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -9,8 +9,6 @@ SYNOPSIS HTTP [POST|PUT|DELETE|GET] {url}paste/[public|private] - {url}ui - DESCRIPTION This program offers an HTTP interface for storing public and private data @@ -21,7 +19,7 @@ DESCRIPTION listed on any index, since it isn't technically possible (by design). All pastes are symetrically encrypted with an HMAC derived key using - {hmac_iterations} iterations and SHA-512 hashing, a server-side salt and a + {hmac_iterations} iterations and SHA-256 hashing, a server-side salt and a randomly generated password. Public paste's passwords are derived from their ids. Private paste's passwords are randomly generated and stored inside a symetrically encrypted personal database, with the encryption key @@ -115,12 +113,12 @@ EXAMPLES SEE ALSO - Documentation + Documentation Sources - Host (HTTPS) - (HTTP) + Host (HTTP) + (Onion) NOTES From e4de8e285e84b94404ec13ccb45463d19fab3858 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 03:53:18 +0200 Subject: [PATCH 26/55] fix(controller/paste): prioritize encoding over highlighting --- src/httpaste/controller/paste/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/httpaste/controller/paste/__init__.py b/src/httpaste/controller/paste/__init__.py index bd0a076..08a74a1 100644 --- a/src/httpaste/controller/paste/__init__.py +++ b/src/httpaste/controller/paste/__init__.py @@ -100,11 +100,13 @@ def get(**kwargs): else: paste_model.remove(pid, backend.paste) + if encoding is not None: + data = data.decode(encoding) + if syntax is not None: data = highlight(data, str(syntax), formatter, linenos) - if encoding is not None: - data = data.decode(encoding) + return ConnexionResponse( status_code=200, From b081f4a5b698c0164754e9be2b9ffbd3e4288394 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 03:54:27 +0200 Subject: [PATCH 27/55] docs: fix faulty reference --- src/httpaste/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 9401ac3..9a062cf 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -118,7 +118,7 @@ SEE ALSO Sources Host (HTTP) - (Onion) + (Onion) NOTES From e79714e1f6387f430085d07fdcdb6f22a6cdd13f Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 04:50:10 +0200 Subject: [PATCH 28/55] feat(samples/httpasteit): add security to httpd - configure mod_security - configure mode_evasive --- samples/httpaste.it/httpd/Dockerfile | 13 ++++++++++++- .../httpd/usr/local/apache2/conf/httpd.conf | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/samples/httpaste.it/httpd/Dockerfile b/samples/httpaste.it/httpd/Dockerfile index afcc50e..0dc9490 100644 --- a/samples/httpaste.it/httpd/Dockerfile +++ b/samples/httpaste.it/httpd/Dockerfile @@ -1,3 +1,14 @@ FROM httpd:2.4 -RUN apt-get update -y && apt-get install -y libapache2-mod-proxy-uwsgi \ No newline at end of file +RUN apt-get update -y && apt-get install -y \ + libapache2-mod-proxy-uwsgi \ + libapache2-mod-evasive \ + libapache2-mod-security2 + +RUN mkdir -p /usr/local/apache2/crs-tecmint + +ADD https://github.com/SpiderLabs/owasp-modsecurity-crs/archive/refs/tags/v3.2.0.tar.gz /usr/local/apache2/crs/master + +RUN cd /usr/local/apache2/crs && \ + tar -xzf master && \ + cp owasp-modsecurity-crs-3.2.0/crs-setup.conf.example owasp-modsecurity-crs-3.2.0/crs-setup.conf \ No newline at end of file diff --git a/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf b/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf index 4c5ead0..d14da77 100644 --- a/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf +++ b/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf @@ -16,6 +16,9 @@ LoadModule proxy_module modules/mod_proxy.so LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so LoadModule unixd_module modules/mod_unixd.so LoadModule access_compat_module modules/mod_access_compat.so +LoadModule security2_module /usr/lib/apache2/modules/mod_security2.so +LoadModule evasive20_module /usr/lib/apache2/modules/mod_evasive20.so + User www-data @@ -24,6 +27,20 @@ LoadModule access_compat_module modules/mod_access_compat.so ServerAdmin you@example.com + + Include crs/owasp-modsecurity-crs-3.2.0/crs-setup.conf + Include crs/owasp-modsecurity-crs-3.2.0/rules/*.conf + + + + DOSHashTableSize 3097 + DOSPageCount 3 + DOSSiteCount 10 + DOSPageInterval 1 + DOSSiteInterval 1 + DOSBlockingPeriod 10 + DOSCloseSocket On + ErrorLog /proc/self/fd/2 @@ -58,6 +75,7 @@ ServerName 127.0.0.1 #ProxyPreserveHost On ServerName httpaste.it + ServerAlias localhost SetEnv proxy-sendchunks ProxyPass "/" "unix:/shared/uwsgi.sock|uwsgi://localhost/" From 4c5f0798bce49fee33a7560ed7e5a6fc6b9e310a Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 15 Apr 2022 04:55:01 +0200 Subject: [PATCH 29/55] feat(samples/httpaste.it): remove httpd header signatures --- samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf b/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf index d14da77..07c9156 100644 --- a/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf +++ b/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf @@ -26,6 +26,8 @@ LoadModule evasive20_module /usr/lib/apache2/modules/mod_evasive20.so ServerAdmin you@example.com +ServerSignature Off +ServerTokens Prod Include crs/owasp-modsecurity-crs-3.2.0/crs-setup.conf From 8016ee7f29e8644e2bda9a07a1aa362352b384fa Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 00:59:10 +0200 Subject: [PATCH 30/55] HTTPASTE-12 feature(router): catch authentication error --- src/httpaste/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 9a062cf..3224f1a 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -241,6 +241,9 @@ def get_flask_app(config: Config) -> FlaskApp: return application + + + __all__ = [ Config, load_config, From 096921ab076862427d9d3af77cf0976337aedd67 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 01:26:24 +0200 Subject: [PATCH 31/55] fix(router): handle only 401 request errors --- src/httpaste/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 3224f1a..9a062cf 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -241,9 +241,6 @@ def get_flask_app(config: Config) -> FlaskApp: return application - - - __all__ = [ Config, load_config, From 6b46159fd0c426fc3b1db12700805c2ff7889048 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 00:59:10 +0200 Subject: [PATCH 32/55] HTTPASTE-12 feature(router): catch authentication error --- src/httpaste/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 9a062cf..3224f1a 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -241,6 +241,9 @@ def get_flask_app(config: Config) -> FlaskApp: return application + + + __all__ = [ Config, load_config, From c518f281e89f05969ef96acb46bc3300ccab7e9e Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 3 Apr 2022 01:26:24 +0200 Subject: [PATCH 33/55] fix(router): handle only 401 request errors --- src/httpaste/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 3224f1a..9a062cf 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -241,9 +241,6 @@ def get_flask_app(config: Config) -> FlaskApp: return application - - - __all__ = [ Config, load_config, From 315f07c5ae6cc4cc1c866683292e2848401a0c1e Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 06:24:21 +0200 Subject: [PATCH 34/55] feat(helper/template): init jinja2 template helper --- src/httpaste/helper/template.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 src/httpaste/helper/template.py diff --git a/src/httpaste/helper/template.py b/src/httpaste/helper/template.py new file mode 100644 index 0000000..446418a --- /dev/null +++ b/src/httpaste/helper/template.py @@ -0,0 +1,6 @@ +from jinja2 import Environment, PackageLoader, select_autoescape + +views = Environment( + loader=PackageLoader("httpaste", "views"), + autoescape=select_autoescape() +) \ No newline at end of file From 9c5c9d743d619a057024051ab7fea66f4e4cfc88 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 06:24:59 +0200 Subject: [PATCH 35/55] feat(helper/url): init url helper --- src/httpaste/helper/url.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 src/httpaste/helper/url.py diff --git a/src/httpaste/helper/url.py b/src/httpaste/helper/url.py new file mode 100644 index 0000000..38528a1 --- /dev/null +++ b/src/httpaste/helper/url.py @@ -0,0 +1,20 @@ +from urllib.parse import urlparse, parse_qs + + +def url_query_string(fields:dict): + + return '&'.join([f'{k}={v}' for k,v in fields.items()]) + + +def url_append_query_param(url:str, name: str, value:str): + + urlcomps = urlparse(url) + + q = parse_qs(urlcomps.query) + + q[name] = value + + qs = url_query_string(q) + + return urlcomps._replace(query=qs).geturl() + From db3701c3d24105d1ec71715ac61080c1bfa8f9cd Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 06:25:49 +0200 Subject: [PATCH 36/55] feat(controller/ui): init ui controller --- src/httpaste/controller/ui/__init__.py | 13 + src/httpaste/controller/ui/paste/__init__.py | 90 ++++++ src/httpaste/controller/ui/paste/private.py | 24 ++ src/httpaste/controller/ui/paste/public.py | 23 ++ src/httpaste/controller/ui/user/__init__.py | 0 .../controller/ui/user/session/__init__.py | 19 ++ .../controller/ui/user/session/delete.py | 6 + src/httpaste/schema/httpaste.openapi.json | 269 +++++++++++++++++- .../views/container/get_paste_form.html | 26 ++ .../views/container/post_paste_form.html | 21 ++ src/httpaste/views/frame/base.html | 58 ++++ src/httpaste/views/frame/decorated.html | 31 ++ src/httpaste/views/viewport/ui/paste/get.html | 27 ++ .../viewport/ui/paste/private/search.html | 5 + .../viewport/ui/paste/public/search.html | 5 + .../views/viewport/ui/paste/search.html | 29 ++ src/httpaste/views/viewport/ui/search.html | 11 + .../viewport/ui/user/session/search.html | 6 + 18 files changed, 659 insertions(+), 4 deletions(-) create mode 100644 src/httpaste/controller/ui/__init__.py create mode 100644 src/httpaste/controller/ui/paste/__init__.py create mode 100644 src/httpaste/controller/ui/paste/private.py create mode 100644 src/httpaste/controller/ui/paste/public.py create mode 100644 src/httpaste/controller/ui/user/__init__.py create mode 100644 src/httpaste/controller/ui/user/session/__init__.py create mode 100644 src/httpaste/controller/ui/user/session/delete.py create mode 100644 src/httpaste/views/container/get_paste_form.html create mode 100644 src/httpaste/views/container/post_paste_form.html create mode 100644 src/httpaste/views/frame/base.html create mode 100644 src/httpaste/views/frame/decorated.html create mode 100644 src/httpaste/views/viewport/ui/paste/get.html create mode 100644 src/httpaste/views/viewport/ui/paste/private/search.html create mode 100644 src/httpaste/views/viewport/ui/paste/public/search.html create mode 100644 src/httpaste/views/viewport/ui/paste/search.html create mode 100644 src/httpaste/views/viewport/ui/search.html create mode 100644 src/httpaste/views/viewport/ui/user/session/search.html diff --git a/src/httpaste/controller/ui/__init__.py b/src/httpaste/controller/ui/__init__.py new file mode 100644 index 0000000..c1b52c2 --- /dev/null +++ b/src/httpaste/controller/ui/__init__.py @@ -0,0 +1,13 @@ +from httpaste.helper.template import views +from httpaste import __doc__ as man_page + +def search(**kwargs): + + template = views.get_template("viewport/ui/search.html") + + variables = { + 'paste_index_url': '/ui/paste', + 'man_page': man_page + } + + return template.render(**variables), 200 \ No newline at end of file diff --git a/src/httpaste/controller/ui/paste/__init__.py b/src/httpaste/controller/ui/paste/__init__.py new file mode 100644 index 0000000..771ed1a --- /dev/null +++ b/src/httpaste/controller/ui/paste/__init__.py @@ -0,0 +1,90 @@ +from io import BytesIO +from base64 import b64encode + +from connexion import request + +from httpaste.helper.template import views +from httpaste.helper.url import url_query_string, url_append_query_param +from httpaste.controller.paste import post as post_raw +from httpaste.controller.paste import get as get_raw + + +def search(**kwargs): + + template = views.get_template("viewport/ui/paste/search.html") + + variables = { + 'create_public_paste_url': '/ui/paste/public', + 'create_private_paste_url': '/ui/paste/private', + 'user': kwargs.get('user'), + 'delete_session_url': '/ui/user/session/delete' + } + + return template.render(**variables), 200 + + +def post(**kwargs): + + #rewriting strict form to mixed (as expected by cascaded controller) + data = kwargs['body'].pop('data') + kwargs = {**kwargs, **kwargs['body']} + kwargs.pop('body') + kwargs['body'] = {'data': data} + + #prepare octet stream data for cascaded controller + if kwargs.get('data').filename: + bfr = BytesIO() + kwargs.get('data').save(bfr) + bfr.seek(0) + kwargs['body']['data'] = b64encode(bfr.read()).decode('utf-8') + kwargs['encoding'] = 'base64' + + output, status_code = post_raw(**kwargs) + + #TODO: lifetime=-1 no preview handler + + url = output.strip('\n') + if kwargs.get('lifetime') and int(kwargs['lifetime']) < 0: + url = url_append_query_param(url, 'preview', 'False') + + return output, 302, {'Location': url} + + +def get(**kwargs): + + template = views.get_template("viewport/ui/paste/get.html") + + base_path = f'paste/public/{kwargs["id"]}' + + raw_paste_url = f'{request.host_url}{base_path}' + if kwargs.get('user'): + raw_paste_url = f'{request.host_url}{base_path}' + + paste_url = raw_paste_url + + paste_url_query = {} + for field in ['format', 'mime', 'syntax']: + if kwargs.get(field): + paste_url_query[field] = kwargs[field] + + if paste_url_query: + paste_url = '?'.join((paste_url, url_query_string(paste_url_query))) + + preview_url = f'/ui/{base_path}' + if kwargs.get('preview'): + paste_url_query['preview'] = kwargs['preview'] + preview_url = '?'.join((preview_url, url_query_string(paste_url_query))) + + variables = { + 'raw_paste_url': raw_paste_url, + 'paste_url': paste_url, + 'preview_url': preview_url, + 'query': { + 'format': kwargs.get('format', ''), + 'syntax': kwargs.get('syntax', ''), + 'mime': kwargs.get('mime', ''), + 'preview': kwargs.get('preview', True) + } + } + + return template.render(**variables) \ No newline at end of file diff --git a/src/httpaste/controller/ui/paste/private.py b/src/httpaste/controller/ui/paste/private.py new file mode 100644 index 0000000..4236f67 --- /dev/null +++ b/src/httpaste/controller/ui/paste/private.py @@ -0,0 +1,24 @@ +from httpaste.helper.template import views +from httpaste.controller.ui.paste import post as post_proxy +from httpaste.controller.ui.paste import get as get_proxy + +def search(**kwargs): + + template = views.get_template("viewport/ui/paste/private/search.html") + + variables = { + 'paste_form_url': '/ui/paste/private', + 'user': kwargs.get('user') + } + + return template.render(**variables), 200 + + +def post(**kwargs): + + return post_proxy(**kwargs) + + +def get(**kwargs): + + return get_proxy(**kwargs) \ No newline at end of file diff --git a/src/httpaste/controller/ui/paste/public.py b/src/httpaste/controller/ui/paste/public.py new file mode 100644 index 0000000..0659d5c --- /dev/null +++ b/src/httpaste/controller/ui/paste/public.py @@ -0,0 +1,23 @@ +from httpaste.helper.template import views +from httpaste.controller.ui.paste import post as post_proxy +from httpaste.controller.ui.paste import get as get_proxy + +def search(**kwargs): + + template = views.get_template("viewport/ui/paste/public/search.html") + + variables = { + 'paste_form_url': '/ui/paste/public' + } + + return template.render(**variables), 200 + + +def post(**kwargs): + + return post_proxy(**kwargs) + + +def get(**kwargs): + + return get_proxy(**kwargs) \ No newline at end of file diff --git a/src/httpaste/controller/ui/user/__init__.py b/src/httpaste/controller/ui/user/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/controller/ui/user/session/__init__.py b/src/httpaste/controller/ui/user/session/__init__.py new file mode 100644 index 0000000..54376bc --- /dev/null +++ b/src/httpaste/controller/ui/user/session/__init__.py @@ -0,0 +1,19 @@ +from httpaste.helper.template import views +from httpaste.controller.user.session import delete as raw_delete + +from connexion import request + +def search(**kwargs): + + template = views.get_template("viewport/ui/user/session/search.html") + + print(request.path) + + variables = {'session_delete_url': request.path + '/delete'} + + return template.render(**variables), 200 + + +def delete(**kwargs): + + return raw_delete(**kwargs) \ No newline at end of file diff --git a/src/httpaste/controller/ui/user/session/delete.py b/src/httpaste/controller/ui/user/session/delete.py new file mode 100644 index 0000000..24fd99c --- /dev/null +++ b/src/httpaste/controller/ui/user/session/delete.py @@ -0,0 +1,6 @@ +from httpaste.helper.template import views +from httpaste.controller.ui.user.session import delete as proxy_delete + +def search(**kwargs): + + return proxy_delete(**kwargs) \ No newline at end of file diff --git a/src/httpaste/schema/httpaste.openapi.json b/src/httpaste/schema/httpaste.openapi.json index 8f55586..508fa5e 100644 --- a/src/httpaste/schema/httpaste.openapi.json +++ b/src/httpaste/schema/httpaste.openapi.json @@ -18,7 +18,7 @@ "get": { "description": "get description", "responses": { - "200": { + "303": { "description": "", "content": { "text/plain": { @@ -185,6 +185,258 @@ } } } + }, + "/ui": { + "get": { + "description": "create a new public paste", + "security": [ + {} + ], + "responses": { + "200": { + "description": "paste location", + "content": { + "text/html": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/ui/paste": { + "get": { + "description": "create a new public paste", + "security": [ + {} + ], + "responses": { + "200": { + "description": "paste location", + "content": { + "text/html": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/ui/paste/public": { + "get": { + "description": "create a new public paste UI-driven", + "security": [ + {} + ], + "responses": { + "200": { + "description": "paste location", + "content": { + "text/html": { + "schema": { + "type": "string" + } + } + } + } + } + }, + "post": { + "description": "create a new public paste", + "requestBody": { + "$ref": "#/components/requestBodies/pastePost" + }, + "security": [ + {} + ], + "parameters": [ + { + "$ref": "#/components/parameters/lifetime" + } + ], + "responses": { + "200": { + "description": "paste location", + "content": { + "text/plain": { + "schema": { + "$ref": "#/components/schemas/PasteURL" + } + } + } + } + } + } + }, + "/ui/paste/private": { + "get": { + "description": "create a new public paste UI-driven", + "security": [ + { + "basicAuth": [] + } + ], + "responses": { + "200": { + "description": "paste location", + "content": { + "text/html": { + "schema": { + "type": "string" + } + } + } + } + } + }, + "post": { + "description": "create a new public paste", + "requestBody": { + "$ref": "#/components/requestBodies/pastePost" + }, + "security": [ + { + "basicAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/lifetime" + }, + { + "$ref": "#/components/parameters/encoding" + } + ], + "responses": { + "200": { + "description": "paste location", + "content": { + "text/plain": { + "schema": { + "$ref": "#/components/schemas/PasteURL" + } + } + } + } + } + } + }, + "/ui/paste/public/{id}": { + "get": { + "description": "get a public paste", + "security": [ + {} + ], + "parameters": [ + { + "$ref": "#/components/parameters/id" + }, + { + "$ref": "#/components/parameters/syntax" + }, + { + "$ref": "#/components/parameters/format" + }, + { + "$ref": "#/components/parameters/linenos" + }, + { + "$ref": "#/components/parameters/mime" + }, + { + "$ref": "#/components/parameters/ui_preview" + } + ], + "responses": { + "200": { + "description": "paste data. content type may vary.", + "content": { + "text/html": { + "schema": { + "$ref": "#/components/schemas/PasteData" + } + } + } + } + } + } + }, + "/ui/paste/private/{id}": { + "get": { + "description": "get a public paste", + "security": [ + { + "basicAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/id" + }, + { + "$ref": "#/components/parameters/syntax" + }, + { + "$ref": "#/components/parameters/format" + }, + { + "$ref": "#/components/parameters/linenos" + }, + { + "$ref": "#/components/parameters/mime" + } + ], + "responses": { + "200": { + "description": "paste data. content type may vary.", + "content": { + "text/html": { + "schema": { + "$ref": "#/components/schemas/PasteData" + } + } + } + } + } + } + }, + "/ui/user/session": { + "get": { + "description": "get a public paste", + "security": [ + { + "basicAuth": [] + } + ], + "responses": { + "200": { + "description": "paste data. content type may vary.", + "content": { + "text/html": {} + } + } + } + } + }, + "/ui/user/session/delete": { + "get": { + "description": "get a public paste", + "security": [ + {} + ], + "responses": { + "200": { + "description": "paste data. content type may vary.", + "content": { + "text/html": {} + } + } + } + } } }, "components": { @@ -215,9 +467,9 @@ "type": "string", "format": "binary" }, - "rsa_public_key": { - "description": "RSA public key", - "type": "string" + "fileName": { + "type": "string", + "format": "binary" } }, "required": [ @@ -294,6 +546,15 @@ "schema": { "type": "string" } + }, + "ui_preview": { + "description": "enable preview in UI", + "name": "preview", + "in": "query", + "required": false, + "schema": { + "type": "boolean" + } } }, "securitySchemes": { diff --git a/src/httpaste/views/container/get_paste_form.html b/src/httpaste/views/container/get_paste_form.html new file mode 100644 index 0000000..d3f658a --- /dev/null +++ b/src/httpaste/views/container/get_paste_form.html @@ -0,0 +1,26 @@ +
+
+ + + + + + Pygments lexer short name (e.g. 'terraform', 'python') +
+
+ + + + + Pygments formatter short name (e.g. 'html', 'terminal256') +
+
+ + + + + Content-Type Header the server should return +
+ + +
\ No newline at end of file diff --git a/src/httpaste/views/container/post_paste_form.html b/src/httpaste/views/container/post_paste_form.html new file mode 100644 index 0000000..31ed83c --- /dev/null +++ b/src/httpaste/views/container/post_paste_form.html @@ -0,0 +1,21 @@ +
+
+ + +

+ +
+ + Either supply a past text, or upload a file. +
+
+ + + + + Set a paste’s lifetime to make it expire after a specified amount of time.
+ The lifetime must be provided in minutes and cannot be less than 1
(, unless lesser than 0).
+ A lifetime of 0 will evaluate to a lifetime 1. +
+ +
\ No newline at end of file diff --git a/src/httpaste/views/frame/base.html b/src/httpaste/views/frame/base.html new file mode 100644 index 0000000..8d6a422 --- /dev/null +++ b/src/httpaste/views/frame/base.html @@ -0,0 +1,58 @@ + + + + + + + +
+ {% block content %}{% endblock %} +
+ + \ No newline at end of file diff --git a/src/httpaste/views/frame/decorated.html b/src/httpaste/views/frame/decorated.html new file mode 100644 index 0000000..7e13b26 --- /dev/null +++ b/src/httpaste/views/frame/decorated.html @@ -0,0 +1,31 @@ + + + + + + + +
+ {% block content %}{% endblock %} +
+ + \ No newline at end of file diff --git a/src/httpaste/views/viewport/ui/paste/get.html b/src/httpaste/views/viewport/ui/paste/get.html new file mode 100644 index 0000000..03a1538 --- /dev/null +++ b/src/httpaste/views/viewport/ui/paste/get.html @@ -0,0 +1,27 @@ +{% extends 'frame/base.html' %} + +{% block content %} + + Return +

Paste Conditioner

+ {% if query['preview'] %} + Preview + +
+ {% else %} +

Preview is disabled. +
+ This probably happened because the paste is set to expire after read. +
+ You can still proceed to condition the paste URL. +

+ {% endif %} + {% include 'container/get_paste_form.html' %} +
+
+

Paste URLs

+ Formatted: {{paste_url}} +
+ Raw: {{raw_paste_url}} +
+{% endblock %} \ No newline at end of file diff --git a/src/httpaste/views/viewport/ui/paste/private/search.html b/src/httpaste/views/viewport/ui/paste/private/search.html new file mode 100644 index 0000000..08bce72 --- /dev/null +++ b/src/httpaste/views/viewport/ui/paste/private/search.html @@ -0,0 +1,5 @@ +{% extends 'frame/base.html' %} + +{% block content %} + {% include 'container/post_paste_form.html' %} +{% endblock %} \ No newline at end of file diff --git a/src/httpaste/views/viewport/ui/paste/public/search.html b/src/httpaste/views/viewport/ui/paste/public/search.html new file mode 100644 index 0000000..08bce72 --- /dev/null +++ b/src/httpaste/views/viewport/ui/paste/public/search.html @@ -0,0 +1,5 @@ +{% extends 'frame/base.html' %} + +{% block content %} + {% include 'container/post_paste_form.html' %} +{% endblock %} \ No newline at end of file diff --git a/src/httpaste/views/viewport/ui/paste/search.html b/src/httpaste/views/viewport/ui/paste/search.html new file mode 100644 index 0000000..17cb192 --- /dev/null +++ b/src/httpaste/views/viewport/ui/paste/search.html @@ -0,0 +1,29 @@ + + + + + + + +

+ Create a Private Paste +

+

+ Create a Public Paste +

+

+ + Flush Local HTTP Authentication Cache + +

+ + \ No newline at end of file diff --git a/src/httpaste/views/viewport/ui/search.html b/src/httpaste/views/viewport/ui/search.html new file mode 100644 index 0000000..ddfe158 --- /dev/null +++ b/src/httpaste/views/viewport/ui/search.html @@ -0,0 +1,11 @@ +{% extends 'frame/decorated.html' %} + +{% block content %} +
+
+ httpaste - versatile HTTP pastebin (User Interface) + +
+
+ +{% endblock %} \ No newline at end of file diff --git a/src/httpaste/views/viewport/ui/user/session/search.html b/src/httpaste/views/viewport/ui/user/session/search.html new file mode 100644 index 0000000..f47ea6a --- /dev/null +++ b/src/httpaste/views/viewport/ui/user/session/search.html @@ -0,0 +1,6 @@ +{% extends 'frame/base.html' %} + +{% block content %} +Clear Local HTTP Authentication Cache + +{% endblock %} \ No newline at end of file From 75ce33e89837c1915edb7915d716981df4e3afdd Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 06:26:28 +0200 Subject: [PATCH 37/55] refactor(helper/http): remove typo --- src/httpaste/helper/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpaste/helper/http.py b/src/httpaste/helper/http.py index 8f8fd48..050cd00 100644 --- a/src/httpaste/helper/http.py +++ b/src/httpaste/helper/http.py @@ -21,7 +21,7 @@ class UnauthorizedError(RuntimeError): return { "detail": str(error), "status": 401, - "title": "Unauthorized s", + "title": "Unauthorized", }, 401 From b69158241a71eef5103bff06e332dafa2601926a Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 06:27:26 +0200 Subject: [PATCH 38/55] feat(controller/root): redirect web browsers to ui --- src/httpaste/controller/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpaste/controller/__init__.py b/src/httpaste/controller/__init__.py index f3ca075..3e923cd 100644 --- a/src/httpaste/controller/__init__.py +++ b/src/httpaste/controller/__init__.py @@ -15,4 +15,4 @@ def get(**kwargs): paste_lifetime=model.paste.default_lifetime, paste_max_lifetime=str(round(model.paste.default_max_lifetime / 60)), paste_default_encoding=model.paste.default_encoding - ), 200 + ), 302, {'Location': '/ui'} From a5e61f9c5cc29c094239f1028ccb60dcce172d83 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 06:28:10 +0200 Subject: [PATCH 39/55] fix(controller/user/session): return 401 upon authentication error --- src/httpaste/controller/user/session.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/httpaste/controller/user/session.py b/src/httpaste/controller/user/session.py index 3639b3b..1d75894 100644 --- a/src/httpaste/controller/user/session.py +++ b/src/httpaste/controller/user/session.py @@ -2,7 +2,7 @@ """ from flask import current_app -from httpaste.helper.http import ForbiddenError +from httpaste.helper.http import UnauthorizedError from httpaste.model.user import authenticate, AuthenticationError from httpaste.backend import load_backend @@ -22,4 +22,11 @@ def post(*args, **kwargs): return authenticate(user_id, password, backend.user, context) except AuthenticationError as e: - raise ForbiddenError('You shall not pass!') from e + raise UnauthorizedError('You shall not pass!') from e + + +def delete(**kwargs): + """ + """ + + raise UnauthorizedError('Authentication Rejection requested by client') \ No newline at end of file From 153ee43b18a63e88e4542f66ff69bf24b5735a9e Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 06:29:21 +0200 Subject: [PATCH 40/55] refactor(toolchain): include views --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 465a26b..d8b5b0e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,4 +40,5 @@ where = src [options.package_data] * = *.json - *.sql \ No newline at end of file + *.sql + *.html \ No newline at end of file From 05d1bc216d2d5db056ebcafd6868d87eb38c8879 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 06:53:23 +0200 Subject: [PATCH 41/55] fix(views): init as packages for bdist to pick up assets --- src/httpaste/views/__init__.py | 0 src/httpaste/views/container/__init__.py | 0 src/httpaste/views/frame/__init__.py | 0 src/httpaste/views/viewport/__init__.py | 0 src/httpaste/views/viewport/ui/__init__.py | 0 src/httpaste/views/viewport/ui/paste/__init__.py | 0 src/httpaste/views/viewport/ui/paste/private/__init__.py | 0 src/httpaste/views/viewport/ui/paste/public/__init__.py | 0 src/httpaste/views/viewport/ui/user/__init__.py | 0 src/httpaste/views/viewport/ui/user/session/__init__.py | 0 10 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/httpaste/views/__init__.py create mode 100644 src/httpaste/views/container/__init__.py create mode 100644 src/httpaste/views/frame/__init__.py create mode 100644 src/httpaste/views/viewport/__init__.py create mode 100644 src/httpaste/views/viewport/ui/__init__.py create mode 100644 src/httpaste/views/viewport/ui/paste/__init__.py create mode 100644 src/httpaste/views/viewport/ui/paste/private/__init__.py create mode 100644 src/httpaste/views/viewport/ui/paste/public/__init__.py create mode 100644 src/httpaste/views/viewport/ui/user/__init__.py create mode 100644 src/httpaste/views/viewport/ui/user/session/__init__.py diff --git a/src/httpaste/views/__init__.py b/src/httpaste/views/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/views/container/__init__.py b/src/httpaste/views/container/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/views/frame/__init__.py b/src/httpaste/views/frame/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/views/viewport/__init__.py b/src/httpaste/views/viewport/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/views/viewport/ui/__init__.py b/src/httpaste/views/viewport/ui/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/views/viewport/ui/paste/__init__.py b/src/httpaste/views/viewport/ui/paste/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/views/viewport/ui/paste/private/__init__.py b/src/httpaste/views/viewport/ui/paste/private/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/views/viewport/ui/paste/public/__init__.py b/src/httpaste/views/viewport/ui/paste/public/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/views/viewport/ui/user/__init__.py b/src/httpaste/views/viewport/ui/user/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/views/viewport/ui/user/session/__init__.py b/src/httpaste/views/viewport/ui/user/session/__init__.py new file mode 100644 index 0000000..e69de29 From 93ed72d5dc17edfc0794a91946d7c3e9c449dd3c Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 07:22:11 +0200 Subject: [PATCH 42/55] fix(model/paste): fix faulty condition --- src/httpaste/model/paste.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpaste/model/paste.py b/src/httpaste/model/paste.py index 3e030ef..534f9a8 100755 --- a/src/httpaste/model/paste.py +++ b/src/httpaste/model/paste.py @@ -94,7 +94,7 @@ def load(proto: Paste, backend: object) -> Optional[Paste]: if proto.sub and model.sub != shash( proto.sub, model.data_hash, - proto.pid) or not proto.sub and model.sub: + proto.pid) or (not proto.sub and model.sub): raise SubError('Paste not owned by user') From bf8e2c19cfe3036cc0902037c29f01088d6e3aee Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 07:22:54 +0200 Subject: [PATCH 43/55] fix(controller/paste): add missing type --- src/httpaste/controller/paste/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpaste/controller/paste/__init__.py b/src/httpaste/controller/paste/__init__.py index 08a74a1..e6efa93 100644 --- a/src/httpaste/controller/paste/__init__.py +++ b/src/httpaste/controller/paste/__init__.py @@ -6,7 +6,7 @@ from httpaste.helper.common import decode, DecodeError, join_url import httpaste.model.paste as paste_model import httpaste.model.user as user_model from httpaste.backend import load_backend -from httpaste.helper.http import BadRequestError, GoneError, NotFoundError +from httpaste.helper.http import BadRequestError, GoneError, NotFoundError, ForbiddenError from httpaste.helper.syntax import highlight from httpaste.schema import ( PasteKey, From 04661720de8bd402bc11899b48bf37fcd8d3f779 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 07:23:16 +0200 Subject: [PATCH 44/55] fix(httpaste/controller/ui/paste): fix url baking --- src/httpaste/controller/ui/paste/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/httpaste/controller/ui/paste/__init__.py b/src/httpaste/controller/ui/paste/__init__.py index 771ed1a..4d63bfd 100644 --- a/src/httpaste/controller/ui/paste/__init__.py +++ b/src/httpaste/controller/ui/paste/__init__.py @@ -54,12 +54,14 @@ def get(**kwargs): template = views.get_template("viewport/ui/paste/get.html") + + base_path = f'paste/public/{kwargs["id"]}' - raw_paste_url = f'{request.host_url}{base_path}' if kwargs.get('user'): - raw_paste_url = f'{request.host_url}{base_path}' + base_path = f'paste/private/{kwargs["id"]}' + raw_paste_url = f'{request.host_url}{base_path}' paste_url = raw_paste_url paste_url_query = {} From 5910cbcc9ad09d8e49da51cef2e7fef8047def22 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 22:31:43 +0200 Subject: [PATCH 45/55] refactor(view): make things a little more pretty --- src/httpaste/controller/ui/__init__.py | 5 +- src/httpaste/controller/ui/paste/__init__.py | 10 +++- src/httpaste/controller/ui/user/__init__.py | 12 ++++ src/httpaste/helper/http.py | 9 +++ src/httpaste/helper/syntax.py | 14 ++++- src/httpaste/helper/template.py | 2 +- src/httpaste/schema/httpaste.openapi.json | 19 ++++++ .../view/container/get_paste_form.html | 39 +++++++++++++ .../view/container/post_paste_form.html | 26 +++++++++ src/httpaste/view/frame/base.html | 16 +++++ src/httpaste/view/frame/decorated.html | 29 ++++++++++ src/httpaste/view/viewport/ui/paste/get.html | 36 ++++++++++++ .../viewport/ui/paste/private/search.html | 14 +++++ .../view/viewport/ui/paste/public/search.html | 14 +++++ .../view/viewport/ui/paste/search.html | 36 ++++++++++++ src/httpaste/view/viewport/ui/search.html | 41 +++++++++++++ .../view/viewport/ui/user/search.html | 16 +++++ .../viewport/ui/user/session/search.html | 0 .../views/container/get_paste_form.html | 26 --------- .../views/container/post_paste_form.html | 21 ------- src/httpaste/views/frame/base.html | 58 ------------------- src/httpaste/views/frame/decorated.html | 31 ---------- src/httpaste/views/viewport/ui/paste/get.html | 27 --------- .../viewport/ui/paste/private/search.html | 5 -- .../viewport/ui/paste/public/search.html | 5 -- .../views/viewport/ui/paste/search.html | 29 ---------- src/httpaste/views/viewport/ui/search.html | 11 ---- 27 files changed, 331 insertions(+), 220 deletions(-) create mode 100644 src/httpaste/view/container/get_paste_form.html create mode 100644 src/httpaste/view/container/post_paste_form.html create mode 100644 src/httpaste/view/frame/base.html create mode 100644 src/httpaste/view/frame/decorated.html create mode 100644 src/httpaste/view/viewport/ui/paste/get.html create mode 100644 src/httpaste/view/viewport/ui/paste/private/search.html create mode 100644 src/httpaste/view/viewport/ui/paste/public/search.html create mode 100644 src/httpaste/view/viewport/ui/paste/search.html create mode 100644 src/httpaste/view/viewport/ui/search.html create mode 100644 src/httpaste/view/viewport/ui/user/search.html rename src/httpaste/{views => view}/viewport/ui/user/session/search.html (100%) delete mode 100644 src/httpaste/views/container/get_paste_form.html delete mode 100644 src/httpaste/views/container/post_paste_form.html delete mode 100644 src/httpaste/views/frame/base.html delete mode 100644 src/httpaste/views/frame/decorated.html delete mode 100644 src/httpaste/views/viewport/ui/paste/get.html delete mode 100644 src/httpaste/views/viewport/ui/paste/private/search.html delete mode 100644 src/httpaste/views/viewport/ui/paste/public/search.html delete mode 100644 src/httpaste/views/viewport/ui/paste/search.html delete mode 100644 src/httpaste/views/viewport/ui/search.html diff --git a/src/httpaste/controller/ui/__init__.py b/src/httpaste/controller/ui/__init__.py index c1b52c2..23890e6 100644 --- a/src/httpaste/controller/ui/__init__.py +++ b/src/httpaste/controller/ui/__init__.py @@ -7,7 +7,10 @@ def search(**kwargs): variables = { 'paste_index_url': '/ui/paste', - 'man_page': man_page + 'user_index_url': '/ui/user', + 'man_page': man_page, + 'user': kwargs.get('user'), + 'delete_session_url': '/ui/user/session/delete' } return template.render(**variables), 200 \ No newline at end of file diff --git a/src/httpaste/controller/ui/paste/__init__.py b/src/httpaste/controller/ui/paste/__init__.py index 4d63bfd..e743a4c 100644 --- a/src/httpaste/controller/ui/paste/__init__.py +++ b/src/httpaste/controller/ui/paste/__init__.py @@ -5,6 +5,9 @@ from connexion import request from httpaste.helper.template import views from httpaste.helper.url import url_query_string, url_append_query_param +from httpaste.helper.syntax import syntax_shortnames, format_shortnames +from httpaste.helper.http import mime_types + from httpaste.controller.paste import post as post_raw from httpaste.controller.paste import get as get_raw @@ -54,8 +57,6 @@ def get(**kwargs): template = views.get_template("viewport/ui/paste/get.html") - - base_path = f'paste/public/{kwargs["id"]}' if kwargs.get('user'): @@ -86,7 +87,10 @@ def get(**kwargs): 'syntax': kwargs.get('syntax', ''), 'mime': kwargs.get('mime', ''), 'preview': kwargs.get('preview', True) - } + }, + 'syntax_shortnames': syntax_shortnames(), + 'format_shortnames': format_shortnames(), + 'mime_types': mime_types() } return template.render(**variables) \ No newline at end of file diff --git a/src/httpaste/controller/ui/user/__init__.py b/src/httpaste/controller/ui/user/__init__.py index e69de29..6c6d86a 100644 --- a/src/httpaste/controller/ui/user/__init__.py +++ b/src/httpaste/controller/ui/user/__init__.py @@ -0,0 +1,12 @@ +from httpaste.helper.template import views +from httpaste import __doc__ as man_page + +def search(**kwargs): + + template = views.get_template("viewport/ui/user/search.html") + + variables = { + 'delete_session_url': '/ui/user/session/delete' + } + + return template.render(**variables), 200 \ No newline at end of file diff --git a/src/httpaste/helper/http.py b/src/httpaste/helper/http.py index 050cd00..2e00d6f 100644 --- a/src/httpaste/helper/http.py +++ b/src/httpaste/helper/http.py @@ -1,3 +1,4 @@ +from mimetypes import types_map as mime_types_map class BadRequestError(RuntimeError): def __init__(self, msg=None): @@ -62,3 +63,11 @@ class NotFoundError(RuntimeError): "status": 404, "title": "Not Found", }, 404 + + +def mime_types(): + + types = list(set(mime_types_map.values())) + types.sort() + + return types \ No newline at end of file diff --git a/src/httpaste/helper/syntax.py b/src/httpaste/helper/syntax.py index d1ed048..c939267 100644 --- a/src/httpaste/helper/syntax.py +++ b/src/httpaste/helper/syntax.py @@ -1,5 +1,5 @@ -from pygments.lexers import get_lexer_by_name, find_lexer_class_by_name -from pygments.formatters import find_formatter_class, HtmlFormatter +from pygments.lexers import (get_lexer_by_name, find_lexer_class_by_name, get_all_lexers) +from pygments.formatters import (find_formatter_class, HtmlFormatter, get_all_formatters) def highlight( @@ -18,3 +18,13 @@ def highlight( formatter = find_formatter_class(format_alias)(linenos=linenos) return highlight(data, get_lexer_by_name(lexer_alias), formatter) + + +def syntax_shortnames(): + + return {l[0]:l[1][0] for l in get_all_lexers() if len(l[1]) > 0} + + +def format_shortnames(): + + return [f.aliases[0] for f in get_all_formatters()] \ No newline at end of file diff --git a/src/httpaste/helper/template.py b/src/httpaste/helper/template.py index 446418a..1ae6d0e 100644 --- a/src/httpaste/helper/template.py +++ b/src/httpaste/helper/template.py @@ -1,6 +1,6 @@ from jinja2 import Environment, PackageLoader, select_autoescape views = Environment( - loader=PackageLoader("httpaste", "views"), + loader=PackageLoader("httpaste", "view"), autoescape=select_autoescape() ) \ No newline at end of file diff --git a/src/httpaste/schema/httpaste.openapi.json b/src/httpaste/schema/httpaste.openapi.json index 508fa5e..35ef3ec 100644 --- a/src/httpaste/schema/httpaste.openapi.json +++ b/src/httpaste/schema/httpaste.openapi.json @@ -388,6 +388,9 @@ }, { "$ref": "#/components/parameters/mime" + }, + { + "$ref": "#/components/parameters/ui_preview" } ], "responses": { @@ -422,6 +425,22 @@ } } }, + "/ui/user": { + "get": { + "description": "get a public paste", + "security": [ + {} + ], + "responses": { + "200": { + "description": "paste data. content type may vary.", + "content": { + "text/html": {} + } + } + } + } + }, "/ui/user/session/delete": { "get": { "description": "get a public paste", diff --git a/src/httpaste/view/container/get_paste_form.html b/src/httpaste/view/container/get_paste_form.html new file mode 100644 index 0000000..9d3d8b4 --- /dev/null +++ b/src/httpaste/view/container/get_paste_form.html @@ -0,0 +1,39 @@ +
+
+ + + + + + + language to highlight syntax for +
+
+ + + + + output format of highlighted syntax +
+
+ + + + + content-type Header the server should return +
+ + +
\ No newline at end of file diff --git a/src/httpaste/view/container/post_paste_form.html b/src/httpaste/view/container/post_paste_form.html new file mode 100644 index 0000000..153a06a --- /dev/null +++ b/src/httpaste/view/container/post_paste_form.html @@ -0,0 +1,26 @@ +
+
+ + + + + Either supply a past text, or upload a file. + +
+
+ + + + + Set a paste’s lifetime to make it expire after a specified amount of time. + The lifetime must be provided in minutes and cannot be less than 1 (, unless lesser than 0). + A lifetime of 0 will evaluate to a lifetime 1. A lifetime of less than 0 will make the paste expire after first read. + +
+ +
\ No newline at end of file diff --git a/src/httpaste/view/frame/base.html b/src/httpaste/view/frame/base.html new file mode 100644 index 0000000..4a2060a --- /dev/null +++ b/src/httpaste/view/frame/base.html @@ -0,0 +1,16 @@ + + + + + + + + + +
+ {% block content %}{% endblock %} +
+ + \ No newline at end of file diff --git a/src/httpaste/view/frame/decorated.html b/src/httpaste/view/frame/decorated.html new file mode 100644 index 0000000..d7c6cf7 --- /dev/null +++ b/src/httpaste/view/frame/decorated.html @@ -0,0 +1,29 @@ + + + + + + + + +
+ {% block header %}{% endblock %} +
+
+
+ {% block content %}{% endblock %} +
+ + \ No newline at end of file diff --git a/src/httpaste/view/viewport/ui/paste/get.html b/src/httpaste/view/viewport/ui/paste/get.html new file mode 100644 index 0000000..b9cac69 --- /dev/null +++ b/src/httpaste/view/viewport/ui/paste/get.html @@ -0,0 +1,36 @@ +{% extends 'frame/decorated.html' %} +{% block header %} +

httpaste - Paste - Conditioner

+

+ Preview and conditon an existing paste +

+{% endblock %} +{% block content %} + + {% if query['preview'] %} + Preview + + {% else %} +

Preview is disabled. +
+ This probably happened because the paste is set to expire after read. +
+ You can still proceed to condition the paste URL. +

+ {% endif %} + {% include 'container/get_paste_form.html' %} +
+
+
URLs
+
+
Formatted
+
+ {{paste_url}} +
+
Raw
+
+ {{raw_paste_url}} +
+
+
+{% endblock %} \ No newline at end of file diff --git a/src/httpaste/view/viewport/ui/paste/private/search.html b/src/httpaste/view/viewport/ui/paste/private/search.html new file mode 100644 index 0000000..6d43869 --- /dev/null +++ b/src/httpaste/view/viewport/ui/paste/private/search.html @@ -0,0 +1,14 @@ +{% extends 'frame/decorated.html' %} + +{% block header %} +

httpaste - Paste - Private

+ +

+ Private pastes are authenticated +

+{% endblock %} +{% block content %} +{% include 'container/post_paste_form.html' %} + + +{% endblock %} \ No newline at end of file diff --git a/src/httpaste/view/viewport/ui/paste/public/search.html b/src/httpaste/view/viewport/ui/paste/public/search.html new file mode 100644 index 0000000..5036269 --- /dev/null +++ b/src/httpaste/view/viewport/ui/paste/public/search.html @@ -0,0 +1,14 @@ +{% extends 'frame/decorated.html' %} + +{% block header %} +

httpaste - Paste - Public

+

+ Public pastes are not indexed and can only be accessed by knowing their respective + paste id. +

+{% endblock %} +{% block content %} +{% include 'container/post_paste_form.html' %} + + +{% endblock %} \ No newline at end of file diff --git a/src/httpaste/view/viewport/ui/paste/search.html b/src/httpaste/view/viewport/ui/paste/search.html new file mode 100644 index 0000000..549bc74 --- /dev/null +++ b/src/httpaste/view/viewport/ui/paste/search.html @@ -0,0 +1,36 @@ +{% extends 'frame/decorated.html' %} + +{% block header %} +

httpaste - Paste

+ +

+ All pastes are symetrically encrypted server-side with an HMAC derived key + and SHA-256 hashing, a server-side salt and a randomly generated password. + Public paste’s passwords are derived from their ids. Private paste’s passwords + are randomly generated and stored inside a symetrically encrypted personal database, + with the encryption key also being derived through the same HMAC mechanism, + where a HTTP basic authentication act as the master password. + Paste ids, usernames, and any other identifiable attributes are only stored + inside the storage backend as keyed and salted BLAKE2 hashes. +

+

Note: + The initial creation of a private paste will prompt for login credentials. + If the login credentials are not known, they will be created automatically. + If it is required to authenticate with other credentials, clear your local + HTTP authentication cache. +

+{% endblock %} +{% block content %} +
+
Navigation
+ +
  • + Create a Private Paste +
    • +
  • + Create a Public Paste +
    • +     +
    + +{% endblock %} \ No newline at end of file diff --git a/src/httpaste/view/viewport/ui/search.html b/src/httpaste/view/viewport/ui/search.html new file mode 100644 index 0000000..c84ce4d --- /dev/null +++ b/src/httpaste/view/viewport/ui/search.html @@ -0,0 +1,41 @@ +{% extends 'frame/decorated.html' %} + +{% block header %} +

    httpaste - versatile HTTP pastebin

    +

    + This is the user interface of the hosted version of httpaste, + a program which offers an HTTP interface for storing public and private data (a.k.a. pastes), + commonly referred to as a pastebin application. It is inspired by + sprunge.us and ix.io. + It aims for a higher degree of privacy control than available commercial pastebin products. +

    +

    + httpaste features include: +

      +
    • Authenticated private and unlisted public pasting
      • +
    • Lifetime control (including Burn-After-Read expiration)
      • +
    • Encoded and Binary Data Upload
      • +
    • Syntax Higlighting
      • +
    • Output Formatting
      • +
    • Output Content-Type Control
      • +
    +

    +

    + A pseudo man page for CLI usage is available via HTTP GET of this host's root document. + (e.g. `$ curl httpaste.it`) +

    +{% endblock %} + +{% block content %} +
    +
    Navigation
    + +
  • + Paste +
    • +
  • + User +
    • +     +
    +{% endblock %} \ No newline at end of file diff --git a/src/httpaste/view/viewport/ui/user/search.html b/src/httpaste/view/viewport/ui/user/search.html new file mode 100644 index 0000000..0d2df0f --- /dev/null +++ b/src/httpaste/view/viewport/ui/user/search.html @@ -0,0 +1,16 @@ +{% extends 'frame/decorated.html' %} + +{% block header %} +

    httpaste - User

    +{% endblock %} +{% block content %} +
    +
    Navigation
    + +
  • + Clear Local HTTP Authentication Cache +
    • +     +
    + +{% endblock %} \ No newline at end of file diff --git a/src/httpaste/views/viewport/ui/user/session/search.html b/src/httpaste/view/viewport/ui/user/session/search.html similarity index 100% rename from src/httpaste/views/viewport/ui/user/session/search.html rename to src/httpaste/view/viewport/ui/user/session/search.html diff --git a/src/httpaste/views/container/get_paste_form.html b/src/httpaste/views/container/get_paste_form.html deleted file mode 100644 index d3f658a..0000000 --- a/src/httpaste/views/container/get_paste_form.html +++ /dev/null @@ -1,26 +0,0 @@ -
    -
    - - - - - - Pygments lexer short name (e.g. 'terraform', 'python') -
    -
    - - - - - Pygments formatter short name (e.g. 'html', 'terminal256') -
    -
    - - - - - Content-Type Header the server should return -
    - - -
    \ No newline at end of file diff --git a/src/httpaste/views/container/post_paste_form.html b/src/httpaste/views/container/post_paste_form.html deleted file mode 100644 index 31ed83c..0000000 --- a/src/httpaste/views/container/post_paste_form.html +++ /dev/null @@ -1,21 +0,0 @@ -
    -
    - - -

    - -
    -     - Either supply a past text, or upload a file. -
    -
    - - - - - Set a paste’s lifetime to make it expire after a specified amount of time.
    - The lifetime must be provided in minutes and cannot be less than 1
    (, unless lesser than 0).
    - A lifetime of 0 will evaluate to a lifetime 1. -
    - -
    \ No newline at end of file diff --git a/src/httpaste/views/frame/base.html b/src/httpaste/views/frame/base.html deleted file mode 100644 index 8d6a422..0000000 --- a/src/httpaste/views/frame/base.html +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - -
    - {% block content %}{% endblock %} -
    - - \ No newline at end of file diff --git a/src/httpaste/views/frame/decorated.html b/src/httpaste/views/frame/decorated.html deleted file mode 100644 index 7e13b26..0000000 --- a/src/httpaste/views/frame/decorated.html +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - -
    - {% block content %}{% endblock %} -
    - - \ No newline at end of file diff --git a/src/httpaste/views/viewport/ui/paste/get.html b/src/httpaste/views/viewport/ui/paste/get.html deleted file mode 100644 index 03a1538..0000000 --- a/src/httpaste/views/viewport/ui/paste/get.html +++ /dev/null @@ -1,27 +0,0 @@ -{% extends 'frame/base.html' %} - -{% block content %} - - Return -

    Paste Conditioner

    - {% if query['preview'] %} - Preview - -
    - {% else %} -

    Preview is disabled. -
    - This probably happened because the paste is set to expire after read. -
    - You can still proceed to condition the paste URL. -

    - {% endif %} - {% include 'container/get_paste_form.html' %} -
    -
    -

    Paste URLs

    - Formatted: {{paste_url}} -
    - Raw: {{raw_paste_url}} -
    -{% endblock %} \ No newline at end of file diff --git a/src/httpaste/views/viewport/ui/paste/private/search.html b/src/httpaste/views/viewport/ui/paste/private/search.html deleted file mode 100644 index 08bce72..0000000 --- a/src/httpaste/views/viewport/ui/paste/private/search.html +++ /dev/null @@ -1,5 +0,0 @@ -{% extends 'frame/base.html' %} - -{% block content %} - {% include 'container/post_paste_form.html' %} -{% endblock %} \ No newline at end of file diff --git a/src/httpaste/views/viewport/ui/paste/public/search.html b/src/httpaste/views/viewport/ui/paste/public/search.html deleted file mode 100644 index 08bce72..0000000 --- a/src/httpaste/views/viewport/ui/paste/public/search.html +++ /dev/null @@ -1,5 +0,0 @@ -{% extends 'frame/base.html' %} - -{% block content %} - {% include 'container/post_paste_form.html' %} -{% endblock %} \ No newline at end of file diff --git a/src/httpaste/views/viewport/ui/paste/search.html b/src/httpaste/views/viewport/ui/paste/search.html deleted file mode 100644 index 17cb192..0000000 --- a/src/httpaste/views/viewport/ui/paste/search.html +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - -

    - Create a Private Paste -

    -

    - Create a Public Paste -

    -

    - - Flush Local HTTP Authentication Cache - -

    - - \ No newline at end of file diff --git a/src/httpaste/views/viewport/ui/search.html b/src/httpaste/views/viewport/ui/search.html deleted file mode 100644 index ddfe158..0000000 --- a/src/httpaste/views/viewport/ui/search.html +++ /dev/null @@ -1,11 +0,0 @@ -{% extends 'frame/decorated.html' %} - -{% block content %} -
    -
    - httpaste - versatile HTTP pastebin (User Interface) - -
    -
    - -{% endblock %} \ No newline at end of file From 26aea68acb3e10409874ee649389836b74e7d5ac Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 22:32:46 +0200 Subject: [PATCH 46/55] chore(setup.cfg): update dependencies --- Pipfile.lock | 66 +++++++++++++++++++++++++++++++++++++++++++--------- setup.cfg | 1 + 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index fc24f88..34a19aa 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -292,6 +292,50 @@ "markers": "python_version >= '3.6'", "version": "==21.3" }, + "pillow": { + "hashes": [ + "sha256:01ce45deec9df310cbbee11104bae1a2a43308dd9c317f99235b6d3080ddd66e", + "sha256:0c51cb9edac8a5abd069fd0758ac0a8bfe52c261ee0e330f363548aca6893595", + "sha256:17869489de2fce6c36690a0c721bd3db176194af5f39249c1ac56d0bb0fcc512", + "sha256:21dee8466b42912335151d24c1665fcf44dc2ee47e021d233a40c3ca5adae59c", + "sha256:25023a6209a4d7c42154073144608c9a71d3512b648a2f5d4465182cb93d3477", + "sha256:255c9d69754a4c90b0ee484967fc8818c7ff8311c6dddcc43a4340e10cd1636a", + "sha256:35be4a9f65441d9982240e6966c1eaa1c654c4e5e931eaf580130409e31804d4", + "sha256:3f42364485bfdab19c1373b5cd62f7c5ab7cc052e19644862ec8f15bb8af289e", + "sha256:3fddcdb619ba04491e8f771636583a7cc5a5051cd193ff1aa1ee8616d2a692c5", + "sha256:463acf531f5d0925ca55904fa668bb3461c3ef6bc779e1d6d8a488092bdee378", + "sha256:4fe29a070de394e449fd88ebe1624d1e2d7ddeed4c12e0b31624561b58948d9a", + "sha256:55dd1cf09a1fd7c7b78425967aacae9b0d70125f7d3ab973fadc7b5abc3de652", + "sha256:5a3ecc026ea0e14d0ad7cd990ea7f48bfcb3eb4271034657dc9d06933c6629a7", + "sha256:5cfca31ab4c13552a0f354c87fbd7f162a4fafd25e6b521bba93a57fe6a3700a", + "sha256:66822d01e82506a19407d1afc104c3fcea3b81d5eb11485e593ad6b8492f995a", + "sha256:69e5ddc609230d4408277af135c5b5c8fe7a54b2bdb8ad7c5100b86b3aab04c6", + "sha256:6b6d4050b208c8ff886fd3db6690bf04f9a48749d78b41b7a5bf24c236ab0165", + "sha256:7a053bd4d65a3294b153bdd7724dce864a1d548416a5ef61f6d03bf149205160", + "sha256:82283af99c1c3a5ba1da44c67296d5aad19f11c535b551a5ae55328a317ce331", + "sha256:8782189c796eff29dbb37dd87afa4ad4d40fc90b2742704f94812851b725964b", + "sha256:8d79c6f468215d1a8415aa53d9868a6b40c4682165b8cb62a221b1baa47db458", + "sha256:97bda660702a856c2c9e12ec26fc6d187631ddfd896ff685814ab21ef0597033", + "sha256:a325ac71914c5c043fa50441b36606e64a10cd262de12f7a179620f579752ff8", + "sha256:a336a4f74baf67e26f3acc4d61c913e378e931817cd1e2ef4dfb79d3e051b481", + "sha256:a598d8830f6ef5501002ae85c7dbfcd9c27cc4efc02a1989369303ba85573e58", + "sha256:a5eaf3b42df2bcda61c53a742ee2c6e63f777d0e085bbc6b2ab7ed57deb13db7", + "sha256:aea7ce61328e15943d7b9eaca87e81f7c62ff90f669116f857262e9da4057ba3", + "sha256:af79d3fde1fc2e33561166d62e3b63f0cc3e47b5a3a2e5fea40d4917754734ea", + "sha256:c24f718f9dd73bb2b31a6201e6db5ea4a61fdd1d1c200f43ee585fc6dcd21b34", + "sha256:c5b0ff59785d93b3437c3703e3c64c178aabada51dea2a7f2c5eccf1bcf565a3", + "sha256:c7110ec1701b0bf8df569a7592a196c9d07c764a0a74f65471ea56816f10e2c8", + "sha256:c870193cce4b76713a2b29be5d8327c8ccbe0d4a49bc22968aa1e680930f5581", + "sha256:c9efef876c21788366ea1f50ecb39d5d6f65febe25ad1d4c0b8dff98843ac244", + "sha256:de344bcf6e2463bb25179d74d6e7989e375f906bcec8cb86edb8b12acbc7dfef", + "sha256:eb1b89b11256b5b6cad5e7593f9061ac4624f7651f7a8eb4dfa37caa1dfaa4d0", + "sha256:ed742214068efa95e9844c2d9129e209ed63f61baa4d54dbf4cf8b5e2d30ccf2", + "sha256:f401ed2bbb155e1ade150ccc63db1a4f6c1909d3d378f7d1235a44e90d75fb97", + "sha256:fb89397013cf302f282f0fc998bb7abf11d49dcff72c8ecb320f76ea6e2c5717" + ], + "markers": "python_version >= '3.7'", + "version": "==9.1.0" + }, "protobuf": { "hashes": [ "sha256:001c2160c03b6349c04de39cf1a58e342750da3632f6978a1634a3dcca1ec10e", @@ -339,11 +383,11 @@ }, "pyparsing": { "hashes": [ - "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea", - "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484" + "sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954", + "sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06" ], - "markers": "python_version >= '3.6'", - "version": "==3.0.7" + "markers": "python_full_version >= '3.6.8'", + "version": "==3.0.8" }, "pyrsistent": { "hashes": [ @@ -501,11 +545,11 @@ }, "pyparsing": { "hashes": [ - "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea", - "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484" + "sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954", + "sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06" ], - "markers": "python_version >= '3.6'", - "version": "==3.0.7" + "markers": "python_full_version >= '3.6.8'", + "version": "==3.0.8" }, "six": { "hashes": [ @@ -533,11 +577,11 @@ }, "virtualenv": { "hashes": [ - "sha256:1e8588f35e8b42c6ec6841a13c5e88239de1e6e4e4cedfd3916b306dc826ec66", - "sha256:8e5b402037287126e81ccde9432b95a8be5b19d36584f64957060a3488c11ca8" + "sha256:e617f16e25b42eb4f6e74096b9c9e37713cf10bf30168fb4a739f3fa8f898a3a", + "sha256:ef589a79795589aada0c1c5b319486797c03b67ac3984c48c669c0e4f50df3a5" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==20.14.0" + "version": "==20.14.1" } } } diff --git a/setup.cfg b/setup.cfg index d8b5b0e..78e0361 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,6 +23,7 @@ install_requires = connexion>=2.13.0,<3 cryptography>=36.0.2,<37 pygments>=2.11.2,<3 + Pillow>=9.1.0,<10 zip_safe = true package_dir = =src From f33ed12fb6e1758cc58dd455b1c636d71957e30b Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 22:40:38 +0200 Subject: [PATCH 47/55] fix(view): add package initializer wherever they went... --- src/httpaste/view/__init__.py | 0 src/httpaste/view/container/__init__.py | 0 src/httpaste/view/frame/__init__.py | 0 src/httpaste/view/viewport/__init__.py | 0 src/httpaste/view/viewport/ui/__init__.py | 0 src/httpaste/view/viewport/ui/paste/__init__.py | 0 src/httpaste/view/viewport/ui/paste/private/__init__.py | 0 src/httpaste/view/viewport/ui/paste/public/__init__.py | 0 src/httpaste/view/viewport/ui/user/__init__.py | 0 src/httpaste/view/viewport/ui/user/session/__init__.py | 0 10 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/httpaste/view/__init__.py create mode 100644 src/httpaste/view/container/__init__.py create mode 100644 src/httpaste/view/frame/__init__.py create mode 100644 src/httpaste/view/viewport/__init__.py create mode 100644 src/httpaste/view/viewport/ui/__init__.py create mode 100644 src/httpaste/view/viewport/ui/paste/__init__.py create mode 100644 src/httpaste/view/viewport/ui/paste/private/__init__.py create mode 100644 src/httpaste/view/viewport/ui/paste/public/__init__.py create mode 100644 src/httpaste/view/viewport/ui/user/__init__.py create mode 100644 src/httpaste/view/viewport/ui/user/session/__init__.py diff --git a/src/httpaste/view/__init__.py b/src/httpaste/view/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/view/container/__init__.py b/src/httpaste/view/container/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/view/frame/__init__.py b/src/httpaste/view/frame/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/view/viewport/__init__.py b/src/httpaste/view/viewport/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/view/viewport/ui/__init__.py b/src/httpaste/view/viewport/ui/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/view/viewport/ui/paste/__init__.py b/src/httpaste/view/viewport/ui/paste/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/view/viewport/ui/paste/private/__init__.py b/src/httpaste/view/viewport/ui/paste/private/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/view/viewport/ui/paste/public/__init__.py b/src/httpaste/view/viewport/ui/paste/public/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/view/viewport/ui/user/__init__.py b/src/httpaste/view/viewport/ui/user/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/httpaste/view/viewport/ui/user/session/__init__.py b/src/httpaste/view/viewport/ui/user/session/__init__.py new file mode 100644 index 0000000..e69de29 From 47cb58c9b14eb90f837147b3e1b25e9810c7ed16 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 16 Apr 2022 22:48:00 +0200 Subject: [PATCH 48/55] fix(Dockerfile): add missing dependencies for PILLOW --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d64fa0e..ae4aa6c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ WORKDIR /usr/local/src/httpaste COPY . . RUN apt-get update && \ - apt-get install -y libffi-dev gcc && \ + apt-get install -y libffi-dev gcc fontconfig && \ python3 -m pip install pipenv && \ python3 -m pipenv install --deploy --system --verbose && \ python3 setup.py install && \ @@ -27,4 +27,4 @@ FROM base as uwsgi ENTRYPOINT ["uwsgi", "--master", "--enable-threads", "--manage-script-name", "-w", "httpaste.wsgi:application"] -CMD ["-s", "/tmp/yourapplication.sock"] \ No newline at end of file +CMD ["-s", "/tmp/yourapplication.sock"] From 68d9240c0c79eafa25465a7c9973b0cdc1e5d585 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 17 Apr 2022 03:51:42 +0200 Subject: [PATCH 49/55] feat(router): establish shared router/view/controller global variables - add ssl warning --- src/httpaste/__init__.py | 17 +++++++++++++++ src/httpaste/controller/ui/__init__.py | 9 ++++++-- src/httpaste/controller/ui/paste/__init__.py | 13 +++++++++--- src/httpaste/controller/ui/paste/private.py | 10 ++++++--- src/httpaste/controller/ui/paste/public.py | 9 ++++++-- src/httpaste/controller/ui/user/__init__.py | 10 ++++++--- .../controller/ui/user/session/__init__.py | 9 ++++---- src/httpaste/helper/template.py | 21 ++++++++++++++++++- src/httpaste/helper/url.py | 20 ++++++++++++++++++ src/httpaste/server.py | 2 ++ src/httpaste/view/frame/decorated.html | 19 +++++++++++++++++ 11 files changed, 121 insertions(+), 18 deletions(-) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 9a062cf..8e93560 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -152,6 +152,7 @@ from httpaste.model import Config as ModelConfig from httpaste.backend import get_backend_config from httpaste.backend import Config as BackendConfig from httpaste.helper.config import get_configparser, CONFIGPATH_ENVIRON +from httpaste.helper.url import url_upgrade_to_https from httpaste.helper.http import ( BadRequestError, ForbiddenError, @@ -238,6 +239,22 @@ def get_flask_app(config: Config) -> FlaskApp: response.headers['WWW-Authenticate'] = 'Basic realm="private"' return response + @application.app.before_request + def before_request_func(): + from flask import request + + request._view = {} + + if config.server.request_ssl: + + https_url = url_upgrade_to_https(request.url, config.server.ssl_port) + + if https_url != request.url: + + print('Hallo') + + request._view['before_request__ssl_url'] = https_url + return application diff --git a/src/httpaste/controller/ui/__init__.py b/src/httpaste/controller/ui/__init__.py index 23890e6..6e740a6 100644 --- a/src/httpaste/controller/ui/__init__.py +++ b/src/httpaste/controller/ui/__init__.py @@ -1,6 +1,8 @@ -from httpaste.helper.template import views +from httpaste.helper.template import views, render_template_with_context from httpaste import __doc__ as man_page +from flask import current_app + def search(**kwargs): template = views.get_template("viewport/ui/search.html") @@ -13,4 +15,7 @@ def search(**kwargs): 'delete_session_url': '/ui/user/session/delete' } - return template.render(**variables), 200 \ No newline at end of file + with current_app.app_context(): + view_render = render_template_with_context(template, **variables) + + return view_render, 200 \ No newline at end of file diff --git a/src/httpaste/controller/ui/paste/__init__.py b/src/httpaste/controller/ui/paste/__init__.py index e743a4c..7d05d47 100644 --- a/src/httpaste/controller/ui/paste/__init__.py +++ b/src/httpaste/controller/ui/paste/__init__.py @@ -2,8 +2,9 @@ from io import BytesIO from base64 import b64encode from connexion import request +from flask import current_app -from httpaste.helper.template import views +from httpaste.helper.template import views, render_template_with_context from httpaste.helper.url import url_query_string, url_append_query_param from httpaste.helper.syntax import syntax_shortnames, format_shortnames from httpaste.helper.http import mime_types @@ -23,7 +24,10 @@ def search(**kwargs): 'delete_session_url': '/ui/user/session/delete' } - return template.render(**variables), 200 + with current_app.app_context(): + view_render = render_template_with_context(template, **variables) + + return view_render, 200 def post(**kwargs): @@ -93,4 +97,7 @@ def get(**kwargs): 'mime_types': mime_types() } - return template.render(**variables) \ No newline at end of file + with current_app.app_context(): + view_render = render_template_with_context(template, **variables) + + return view_render, 200 \ No newline at end of file diff --git a/src/httpaste/controller/ui/paste/private.py b/src/httpaste/controller/ui/paste/private.py index 4236f67..3f2fe65 100644 --- a/src/httpaste/controller/ui/paste/private.py +++ b/src/httpaste/controller/ui/paste/private.py @@ -1,4 +1,6 @@ -from httpaste.helper.template import views +from flask import current_app + +from httpaste.helper.template import views, render_template_with_context from httpaste.controller.ui.paste import post as post_proxy from httpaste.controller.ui.paste import get as get_proxy @@ -8,10 +10,12 @@ def search(**kwargs): variables = { 'paste_form_url': '/ui/paste/private', - 'user': kwargs.get('user') } - return template.render(**variables), 200 + with current_app.app_context(): + view_render = render_template_with_context(template, **variables) + + return view_render, 200 def post(**kwargs): diff --git a/src/httpaste/controller/ui/paste/public.py b/src/httpaste/controller/ui/paste/public.py index 0659d5c..3110c53 100644 --- a/src/httpaste/controller/ui/paste/public.py +++ b/src/httpaste/controller/ui/paste/public.py @@ -1,4 +1,6 @@ -from httpaste.helper.template import views +from flask import current_app + +from httpaste.helper.template import views, render_template_with_context from httpaste.controller.ui.paste import post as post_proxy from httpaste.controller.ui.paste import get as get_proxy @@ -10,7 +12,10 @@ def search(**kwargs): 'paste_form_url': '/ui/paste/public' } - return template.render(**variables), 200 + with current_app.app_context(): + view_render = render_template_with_context(template, **variables) + + return view_render, 200 def post(**kwargs): diff --git a/src/httpaste/controller/ui/user/__init__.py b/src/httpaste/controller/ui/user/__init__.py index 6c6d86a..0fd5766 100644 --- a/src/httpaste/controller/ui/user/__init__.py +++ b/src/httpaste/controller/ui/user/__init__.py @@ -1,5 +1,6 @@ -from httpaste.helper.template import views -from httpaste import __doc__ as man_page +from flask import current_app + +from httpaste.helper.template import views, render_template_with_context def search(**kwargs): @@ -9,4 +10,7 @@ def search(**kwargs): 'delete_session_url': '/ui/user/session/delete' } - return template.render(**variables), 200 \ No newline at end of file + with current_app.app_context(): + view_render = render_template_with_context(template, **variables) + + return view_render, 200 \ No newline at end of file diff --git a/src/httpaste/controller/ui/user/session/__init__.py b/src/httpaste/controller/ui/user/session/__init__.py index 54376bc..6f1eeee 100644 --- a/src/httpaste/controller/ui/user/session/__init__.py +++ b/src/httpaste/controller/ui/user/session/__init__.py @@ -1,4 +1,4 @@ -from httpaste.helper.template import views +from httpaste.helper.template import views, render_template_with_context from httpaste.controller.user.session import delete as raw_delete from connexion import request @@ -7,11 +7,12 @@ def search(**kwargs): template = views.get_template("viewport/ui/user/session/search.html") - print(request.path) - variables = {'session_delete_url': request.path + '/delete'} - return template.render(**variables), 200 + with current_app.app_context(): + view_render = render_template_with_context(template, **variables) + + return view_render, 200 def delete(**kwargs): diff --git a/src/httpaste/helper/template.py b/src/httpaste/helper/template.py index 1ae6d0e..3543ea9 100644 --- a/src/httpaste/helper/template.py +++ b/src/httpaste/helper/template.py @@ -1,6 +1,25 @@ from jinja2 import Environment, PackageLoader, select_autoescape +from collections import namedtuple views = Environment( loader=PackageLoader("httpaste", "view"), autoescape=select_autoescape() -) \ No newline at end of file +) + + +def render_template_with_context(template: object, **kwargs): + """render a template with global context variables + + the definition of a global context is abstract, it does neither apply + to Flask, nor to Jinja2 and only acts as a bridge for passing + variables from flask to jinja2, without having to define them within + each controller. + + :param template: jinja2 template object + """ + + from flask import request + + return template.render(**{**kwargs, **{ + 'flask': namedtuple('Flask', request._view.keys())(**request._view) + }}) \ No newline at end of file diff --git a/src/httpaste/helper/url.py b/src/httpaste/helper/url.py index 38528a1..cd0c800 100644 --- a/src/httpaste/helper/url.py +++ b/src/httpaste/helper/url.py @@ -1,3 +1,4 @@ +from typing import Optional from urllib.parse import urlparse, parse_qs @@ -18,3 +19,22 @@ def url_append_query_param(url:str, name: str, value:str): return urlcomps._replace(query=qs).geturl() + +def url_upgrade_to_https(url: str, port: Optional[int] = 443): + + urlcomps = urlparse(url) + + urlcomps = urlcomps._replace(scheme='https') + + if url != urlcomps.geturl(): + + hostname = urlcomps.netloc.rsplit(':', 1)[0] + + if port != 443: + netloc = ':'.join((hostname, str(port))) + else: + netloc = hostname + + urlcomps = urlcomps._replace(netloc=netloc) + + return urlcomps.geturl() \ No newline at end of file diff --git a/src/httpaste/server.py b/src/httpaste/server.py index 55c0542..14103e3 100755 --- a/src/httpaste/server.py +++ b/src/httpaste/server.py @@ -10,6 +10,8 @@ class Config(NamedTuple): """ swagger_ui: bool = True bind_address: str = None + request_ssl: bool = True + ssl_port: int = 443 def get_server_config(configIni: ConfigParser) -> Config: diff --git a/src/httpaste/view/frame/decorated.html b/src/httpaste/view/frame/decorated.html index d7c6cf7..e3983dc 100644 --- a/src/httpaste/view/frame/decorated.html +++ b/src/httpaste/view/frame/decorated.html @@ -15,10 +15,29 @@ margin-bottom: 1em; } + .blinking{ + animation:blinkingText 1.0s infinite; + animation-timing-function: step-start; + } + + .warning { + color: red; + } + + @keyframes blinkingText { + 0%{ color: red; } + 50%{ color: transparent; } + 100%{ color: red; } + }
    + {% if flask.before_request__ssl_url is defined %} +

    + WARNING: Communication not encrypted - SSL/TLS will not be enforced. Visit {{ flask.before_request__ssl_url }}, for SSL/TLS encryption. +

    + {% endif %} {% block header %}{% endblock %}
    From 56f46172ce8938cf8ccf9af72f90cf3feab83f4d Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 17 Apr 2022 03:53:51 +0200 Subject: [PATCH 50/55] feat(samples/httpaste.it/httpd) enable SSL --- samples/httpaste.it/docker-compose.yml | 2 ++ .../httpd/usr/local/apache2/conf/httpd.conf | 19 ++++++++++++++++++- .../httpd/usr/local/apache2/ssl/.gitignore | 2 ++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 samples/httpaste.it/httpd/usr/local/apache2/ssl/.gitignore diff --git a/samples/httpaste.it/docker-compose.yml b/samples/httpaste.it/docker-compose.yml index 2150c79..3c7cf0f 100644 --- a/samples/httpaste.it/docker-compose.yml +++ b/samples/httpaste.it/docker-compose.yml @@ -22,6 +22,7 @@ services: dockerfile: Dockerfile ports: - "80:80" + - "443:443" volumes: - type: volume @@ -30,6 +31,7 @@ services: volume: nocopy: true - ./httpd/usr/local/apache2/conf/httpd.conf:/usr/local/apache2/conf/httpd.conf + - ./httpd/usr/local/apache2/ssl:/usr/local/apache2/ssl tor: build: context: ./tor diff --git a/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf b/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf index 07c9156..feccc6b 100644 --- a/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf +++ b/samples/httpaste.it/httpd/usr/local/apache2/conf/httpd.conf @@ -18,7 +18,7 @@ LoadModule unixd_module modules/mod_unixd.so LoadModule access_compat_module modules/mod_access_compat.so LoadModule security2_module /usr/lib/apache2/modules/mod_security2.so LoadModule evasive20_module /usr/lib/apache2/modules/mod_evasive20.so - +LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so User www-data @@ -88,3 +88,20 @@ ServerName 127.0.0.1 SetEnv proxy-sendchunks ProxyPass "/" "unix:/shared/uwsgi.sock|uwsgi://localhost/" + + + Listen 0.0.0.0:443 + + + + #ProxyPreserveHost On + ServerName httpaste.it + ServerAlias localhost + SSLEngine on + SSLCertificateFile "ssl/certificate.crt" + SSLCertificateChainFile "ssl/ca_bundle.crt" + SSLCertificateKeyFile "ssl/private.key" + SetEnv proxy-sendchunks + ProxyPass "/" "unix:/shared/uwsgi.sock|uwsgi://localhost/" + + diff --git a/samples/httpaste.it/httpd/usr/local/apache2/ssl/.gitignore b/samples/httpaste.it/httpd/usr/local/apache2/ssl/.gitignore new file mode 100644 index 0000000..0d313d1 --- /dev/null +++ b/samples/httpaste.it/httpd/usr/local/apache2/ssl/.gitignore @@ -0,0 +1,2 @@ +*.key +*.crt \ No newline at end of file From 98586f4fd2accdcc839e51520ff55f49a9f79b54 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 17 Apr 2022 03:58:22 +0200 Subject: [PATCH 51/55] feat(samples/httpaste.it/tor): make hidden_service transferable --- samples/httpaste.it/docker-compose.yml | 1 + samples/httpaste.it/tor/var/lib/tor/hidden_service/.gitkeep | 1 + 2 files changed, 2 insertions(+) create mode 100644 samples/httpaste.it/tor/var/lib/tor/hidden_service/.gitkeep diff --git a/samples/httpaste.it/docker-compose.yml b/samples/httpaste.it/docker-compose.yml index 3c7cf0f..2dc9ace 100644 --- a/samples/httpaste.it/docker-compose.yml +++ b/samples/httpaste.it/docker-compose.yml @@ -38,5 +38,6 @@ services: dockerfile: Dockerfile volumes: - ./tor/etc/tor/torrc:/etc/tor/torrc + - ./tor/var/lib/tor/hidden_service:./tor/var/lib/tor/hidden_service volumes: system-shared: diff --git a/samples/httpaste.it/tor/var/lib/tor/hidden_service/.gitkeep b/samples/httpaste.it/tor/var/lib/tor/hidden_service/.gitkeep new file mode 100644 index 0000000..f59ec20 --- /dev/null +++ b/samples/httpaste.it/tor/var/lib/tor/hidden_service/.gitkeep @@ -0,0 +1 @@ +* \ No newline at end of file From 90fa8cd7b8d9af8e8dd40429dcccd7b7ed1fb3ea Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 17 Apr 2022 04:00:20 +0200 Subject: [PATCH 52/55] style: remove debug print statements --- src/httpaste/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 8e93560..24716b9 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -202,8 +202,6 @@ def get_flask_app(config: Config) -> FlaskApp: """get a flask app object """ - print(config.server.swagger_ui) - options = {"swagger_ui": config.server.swagger_ui} #context manager returns a pathlib.Path object @@ -251,8 +249,6 @@ def get_flask_app(config: Config) -> FlaskApp: if https_url != request.url: - print('Hallo') - request._view['before_request__ssl_url'] = https_url return application From ad4e7f4762a5b462722c2d0f27f2ea158f523b39 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 17 Apr 2022 04:31:49 +0200 Subject: [PATCH 53/55] fix(router): add SSL exemption for Tor hidden services --- samples/httpaste.it/docker-compose.yml | 2 +- src/httpaste/__init__.py | 2 +- src/httpaste/helper/url.py | 16 +++++++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/samples/httpaste.it/docker-compose.yml b/samples/httpaste.it/docker-compose.yml index 2dc9ace..ff6124c 100644 --- a/samples/httpaste.it/docker-compose.yml +++ b/samples/httpaste.it/docker-compose.yml @@ -38,6 +38,6 @@ services: dockerfile: Dockerfile volumes: - ./tor/etc/tor/torrc:/etc/tor/torrc - - ./tor/var/lib/tor/hidden_service:./tor/var/lib/tor/hidden_service + - ./tor/var/lib/tor/hidden_service:/tor/var/lib/tor/hidden_service volumes: system-shared: diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 24716b9..844f9cc 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -247,7 +247,7 @@ def get_flask_app(config: Config) -> FlaskApp: https_url = url_upgrade_to_https(request.url, config.server.ssl_port) - if https_url != request.url: + if https_url != request.url and not url_has_tld(request._view, 'onion'): request._view['before_request__ssl_url'] = https_url diff --git a/src/httpaste/helper/url.py b/src/httpaste/helper/url.py index cd0c800..c44c02f 100644 --- a/src/httpaste/helper/url.py +++ b/src/httpaste/helper/url.py @@ -37,4 +37,18 @@ def url_upgrade_to_https(url: str, port: Optional[int] = 443): urlcomps = urlcomps._replace(netloc=netloc) - return urlcomps.geturl() \ No newline at end of file + return urlcomps.geturl() + + +def url_has_tld(url:str, tld:str): + + urlcomps = urlparse(url) + + hostname = urlcomps.netloc.rsplit(':', 1)[0] + + hostname_levels = hostname.rsplit('.', 1) + + if len(hostname_levels) > 1 and hostname_levels[-1:][0] == tld: + return True + + return False \ No newline at end of file From 903e4370095dd4443ca5ef73187d65d8c3a70b3b Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 17 Apr 2022 04:34:06 +0200 Subject: [PATCH 54/55] fix(router): add missing import --- src/httpaste/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 844f9cc..3b0bc70 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -152,7 +152,7 @@ from httpaste.model import Config as ModelConfig from httpaste.backend import get_backend_config from httpaste.backend import Config as BackendConfig from httpaste.helper.config import get_configparser, CONFIGPATH_ENVIRON -from httpaste.helper.url import url_upgrade_to_https +from httpaste.helper.url import url_upgrade_to_https, url_has_tld from httpaste.helper.http import ( BadRequestError, ForbiddenError, From 5fa7c5c898f6706bc17630d174894d1cf36d3451 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sun, 17 Apr 2022 04:55:30 +0200 Subject: [PATCH 55/55] fix(router): replace faulty parameter --- src/httpaste/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpaste/__init__.py b/src/httpaste/__init__.py index 3b0bc70..6d779c2 100755 --- a/src/httpaste/__init__.py +++ b/src/httpaste/__init__.py @@ -247,7 +247,7 @@ def get_flask_app(config: Config) -> FlaskApp: https_url = url_upgrade_to_https(request.url, config.server.ssl_port) - if https_url != request.url and not url_has_tld(request._view, 'onion'): + if https_url != request.url and not url_has_tld(request.url, 'onion'): request._view['before_request__ssl_url'] = https_url