From f0c0188d58c5acf48ec43c8db78385df24503565 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 16:50:42 +0200 Subject: [PATCH 1/4] fix(model/paste): interpolate expiration paste model now consists of expiration instead of lifetime and timestamp. This is to ensure, that man-in-the-middle attackers cannot derive the origin of a paste through observing transport layer server request times. --- src/httpaste/backend/__init__.py | 2 +- src/httpaste/backend/file/paste.py | 28 ++++++++++------------- src/httpaste/backend/sqlite/paste.py | 12 ++++------ src/httpaste/backend/sqlite/paste.sql | 9 ++++---- src/httpaste/controller/paste/__init__.py | 4 ++-- src/httpaste/model/__init__.py | 13 ++++++++--- src/httpaste/model/paste.py | 26 +++++++++++++-------- 7 files changed, 50 insertions(+), 44 deletions(-) diff --git a/src/httpaste/backend/__init__.py b/src/httpaste/backend/__init__.py index 2e176d9..48978a0 100644 --- a/src/httpaste/backend/__init__.py +++ b/src/httpaste/backend/__init__.py @@ -26,7 +26,7 @@ class SQLite(Backend): def __init__(self, parameters: SqliteParameters): - parameters['connection'] = get_sqlite_connection(parameters) + parameters = SqliteParameters(parameters.path, get_sqlite_connection(parameters)) self.user = SqliteUser(parameters, User) self.paste = SqlitePaste(parameters, Paste) diff --git a/src/httpaste/backend/file/paste.py b/src/httpaste/backend/file/paste.py index d27b7ca..faefaf0 100644 --- a/src/httpaste/backend/file/paste.py +++ b/src/httpaste/backend/file/paste.py @@ -7,6 +7,15 @@ from pathlib import Path from ast import literal_eval +COLUMNS = [ + 'data', + 'data_hash', + 'sub', + 'expiration', + 'encoding' +] + + def load( proto: object, path: Path, @@ -22,13 +31,7 @@ def load( return None cells = {} - for column in [ - 'data', - 'data_hash', - 'sub', - 'timestamp', - 'lifetime', - 'encoding']: + for column in COLUMNS: cell = row.joinpath(column) @@ -56,8 +59,7 @@ def load( cells['sub'], cells['data'], cells['data_hash'], - cells['timestamp'], - cells['lifetime'], + cells['expiration'], cells['encoding']) @@ -68,13 +70,7 @@ def dump(model: object, path: Path, model_schema: type) -> None: row = path.joinpath(model.pid.hex()) row.mkdir(parents=True, exist_ok=True) - for column in [ - 'data', - 'data_hash', - 'sub', - 'timestamp', - 'lifetime', - 'encoding']: + for column in COLUMNS: cell = row.joinpath(column) cell_schema = getattr(model_schema, column) diff --git a/src/httpaste/backend/sqlite/paste.py b/src/httpaste/backend/sqlite/paste.py index 61f8cd1..150e3cb 100644 --- a/src/httpaste/backend/sqlite/paste.py +++ b/src/httpaste/backend/sqlite/paste.py @@ -11,7 +11,7 @@ def load(proto: object, connection: Connection, model_class: type): cur = connection.cursor() cur.execute( - 'SELECT pid, data, data_hash, sub, timestamp, lifetime, encoding FROM pastes WHERE pid=?', + 'SELECT pid, data, data_hash, sub, expiration, encoding FROM pastes WHERE pid=?', (proto.pid, )) @@ -24,8 +24,7 @@ def load(proto: object, connection: Connection, model_class: type): result['sub'], result['data'], result['data_hash'], - result['timestamp'], - result['lifetime'], + result['expiration'], result['encoding']) return None @@ -38,14 +37,13 @@ def dump(model: object, connection: Connection): cur = connection.cursor() cur.execute( - '''INSERT INTO pastes (pid, data, data_hash, sub, timestamp, lifetime, encoding) - VALUES (?,?,?,?,?,?,?)''', + '''INSERT INTO pastes (pid, data, data_hash, sub, expiration, encoding) + VALUES (?,?,?,?,?,?)''', (model.pid, model.data, model.data_hash, model.sub, - model.timestamp, - model.lifetime, + model.expiration, model.encoding)) connection.commit() diff --git a/src/httpaste/backend/sqlite/paste.sql b/src/httpaste/backend/sqlite/paste.sql index 5788d5b..7f9bb46 100644 --- a/src/httpaste/backend/sqlite/paste.sql +++ b/src/httpaste/backend/sqlite/paste.sql @@ -1,10 +1,9 @@ CREATE TABLE IF NOT EXISTS "pastes" ( - "id" BLOB NOT NULL UNIQUE, + "pid" BLOB NOT NULL UNIQUE, "data" BLOB NOT NULL, "data_hash" BLOB NOT NULL, "sub" BLOB UNIQUE, - "timestamp" INTEGER NOT NULL, - "lifetime" INTEGER NOT NULL, - "encoding" TEXT - PRIMARY KEY("id") + "expiration" INTEGER NOT NULL, + "encoding" TEXT, + PRIMARY KEY("pid") ); \ No newline at end of file diff --git a/src/httpaste/controller/paste/__init__.py b/src/httpaste/controller/paste/__init__.py index fcd6ab6..c8f4b87 100644 --- a/src/httpaste/controller/paste/__init__.py +++ b/src/httpaste/controller/paste/__init__.py @@ -71,7 +71,7 @@ def get(**kwargs): config.salt, config.hmac_iterations) try: - data, lifetime, encoding = call() + data, expiration, encoding = call() except paste_model.LifetimeError as e: if kwargs.get('user') is not None: paste_model.remove_safe(pid, sub, pkey, config.backend.paste, @@ -85,7 +85,7 @@ def get(**kwargs): raise ForbiddenError(str(e)) # burn after read - if lifetime < 0: + if expiration < 0: if kwargs.get('user') is not None: paste_model.remove_safe(pid, sub, pkey, config.backend.paste, config.salt, config.hmac_iterations) diff --git a/src/httpaste/model/__init__.py b/src/httpaste/model/__init__.py index 955ded4..b9d1b4f 100644 --- a/src/httpaste/model/__init__.py +++ b/src/httpaste/model/__init__.py @@ -12,6 +12,7 @@ class PasteDataSchema: sub = bytes timestamp = int lifetime = int + expiration = int encoding = str @@ -54,6 +55,14 @@ class PasteEncoding(PasteDataSchema.encoding): """ +class PasteExpiration(PasteDataSchema.expiration): + """Paste Expiration + + < 0: after first acccess + 0: never + """ + + class PasteLifetime(PasteDataSchema.lifetime): """Paste Lifetime """ @@ -128,8 +137,6 @@ class Paste(NamedTuple): #: paste data hash data_hash: Optional[PasteHash] = None #: paste timestamp - timestamp: Optional[PasteTimestamp] = None - #: paste lifetime - lifetime: Optional[PasteLifetime] = None + expiration: Optional[PasteExpiration] = None #: paste encoding encoding: Optional[PasteEncoding] = None diff --git a/src/httpaste/model/paste.py b/src/httpaste/model/paste.py index 5ad9dc8..04ea6a5 100755 --- a/src/httpaste/model/paste.py +++ b/src/httpaste/model/paste.py @@ -10,7 +10,7 @@ from httpaste.helper.crypto import dhash, shash, encrypt, decrypt from httpaste.helper.common import generate_random_string from httpaste.model import (Paste, PasteId, Sub, MasterKey, PasteKey, Salt, PasteData, PasteHash, PasteTimestamp, PasteSub, - PasteLifetime, PasteEncoding) + PasteLifetime, PasteEncoding, PasteExpiration) class NotFoundError(Exception): @@ -87,8 +87,7 @@ def load(proto: Paste, backend: object) -> Optional[Paste]: raise SubError('Paste not owned by user') - if model.lifetime >= 0 and model.timestamp + \ - (60 * model.lifetime) < int(time.time()): + if model.expiration > 0 and model.expiration < int(time.time()): raise LifetimeError('Paste expired') @@ -121,8 +120,7 @@ def load_safe( proto.sub, data, model.data_hash, - model.timestamp, - model.lifetime, + model.expiration, model.encoding) @@ -195,6 +193,11 @@ def create( sub = None timestamp = PasteTimestamp(int(time.time())) + if lifetime < 0: + expiration = -1 + else: + expiration = PasteExpiration(timestamp + (lifetime * 60)) + safe_data = PasteData(encrypt(data, pid, salt, hmac_iter)) model = Paste( @@ -202,8 +205,7 @@ def create( sub, safe_data, data_hash, - timestamp, - lifetime, + expiration, encoding) dump(model, backend) @@ -234,6 +236,11 @@ def create_safe(data: PasteData, safe_sub = PasteSub(shash(sub, data_hash, pid)) timestamp = PasteTimestamp(int(time.time())) + if lifetime < 0: + expiration = -1 + else: + expiration = PasteExpiration(timestamp + (lifetime * 60)) + safe_data = PasteData(encrypt(data, pkey, salt, hmac_iter)) dump(Paste( @@ -241,8 +248,7 @@ def create_safe(data: PasteData, safe_sub, safe_data, data_hash, - timestamp, - lifetime, + expiration, encoding ), backend) @@ -279,7 +285,7 @@ def get(pid: PasteId, backend: object, salt: Salt = Config.salt, hmac_iter: int data = decrypt(model.data, pid, salt, hmac_iter) - return PasteData(data), model.lifetime, model.encoding + return PasteData(data), model.expiration, model.encoding def get_safe( From bd7af0585011ccbce7d7a105d6c913de0ed3579a Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 17:59:04 +0200 Subject: [PATCH 2/4] feat(mode/user): implement complex index to accomodate more personalized user data, the index can now hold more than just paste constraints. It was planned to implement an authentication expiration, however, this does not make sense in an HTTP basic auth context. --- src/httpaste/backend/file/user.py | 18 ++++++--- src/httpaste/model/__init__.py | 6 ++- src/httpaste/model/user.py | 61 +++++++++++++++++++++---------- 3 files changed, 58 insertions(+), 27 deletions(-) diff --git a/src/httpaste/backend/file/user.py b/src/httpaste/backend/file/user.py index 02255b2..72c9256 100644 --- a/src/httpaste/backend/file/user.py +++ b/src/httpaste/backend/file/user.py @@ -6,6 +6,12 @@ acting as cells. from pathlib import Path from ast import literal_eval +COLUMNS = [ + 'sub', + 'key_hash', + 'index', +] + def load( proto: object, @@ -22,15 +28,17 @@ def load( return None cells = {} - for column in ['key_hash', 'index']: + for column in COLUMNS[1:]: cell = row.joinpath(column) - if getattr(model_schema, column) == bytes: - + if not cell.exists(): + cells[column] = None + elif getattr(model_schema, column) == bytes: cells[column] = cell.read_bytes() + elif getattr(model_schema, column) == str: + cells[column] = cell.read_text() else: - cells[column] = literal_eval(cell.read_text()) return model_class( @@ -46,7 +54,7 @@ def dump(model: object, path: Path, model_schema: object) -> None: row = path.joinpath(model.sub.hex()) row.mkdir(parents=True, exist_ok=True) - for column in ['key_hash', 'index']: + for column in COLUMNS[1:]: cell = row.joinpath(column) diff --git a/src/httpaste/model/__init__.py b/src/httpaste/model/__init__.py index 955ded4..e41e4c1 100644 --- a/src/httpaste/model/__init__.py +++ b/src/httpaste/model/__init__.py @@ -1,6 +1,6 @@ """Model """ -from typing import NamedTuple, Optional, Dict, Union +from typing import NamedTuple, Optional, Dict, Union, Any, TypedDict class PasteDataSchema: @@ -89,9 +89,11 @@ class Sub(UserDataSchema.sub): """ -class Index(Dict[PasteId, PasteKey]): +class Index(TypedDict): """User Paste Index """ + auth_expires: int + pastes: Dict[str, Dict[str, Any]] class SerializedIndex(UserDataSchema.index): diff --git a/src/httpaste/model/user.py b/src/httpaste/model/user.py index 7639127..b925063 100755 --- a/src/httpaste/model/user.py +++ b/src/httpaste/model/user.py @@ -2,6 +2,7 @@ """user model interface """ import json +from time import time from typing import Optional from httpaste import Config @@ -34,7 +35,7 @@ class IndexError(Exception): """ -def load( +def _load( proto: User, master_key: str, backend: object, @@ -54,16 +55,19 @@ def load( return None try: - return User( - *model[:-1], - Index(**json.loads(decrypt(model.index, master_key, salt, hmac_iter))) - ) + serialized_data = decrypt(model.index, master_key, salt, hmac_iter) except DecryptionError as e: - raise IndexError('unable to decrypt user index') from e + else: + data = json.loads(serialized_data) + + return User( + *model[:-1], + Index(**data) + ) -def dump( +def _dump( model: User, key: MasterKey, backend: object, @@ -77,7 +81,7 @@ def dump( :param salt: randomization salt """ - if not isinstance(model.index, Index): + if model.index is not None and not isinstance(model.index, dict): raise BaseException('index serialization pre-processing not allowed.') @@ -102,11 +106,13 @@ def load_paste_key( :param salt: randomization salt """ - for k, v in load(User(sub), key, backend, salt, hmac_iter).index.items(): + model = _load(User(sub), key, backend, salt, hmac_iter) + + for k, v in model.index.get('pastes').items(): if bytes.fromhex(k) == pid: - return PasteKey(bytes.fromhex(v)) + return PasteKey(bytes.fromhex(v.get('key'))) return None @@ -128,12 +134,13 @@ def dump_paste_key( :param backend: user model backend """ - model = load(User(sub), key, backend, salt, hmac_iter) + model = _load(User(sub), key, backend, salt, hmac_iter) - dump(User(*model[:-1], Index({ - **model.index, - **{pid.hex(): pkey.hex()} - })), key, backend, salt, hmac_iter) + model.index.setdefault('pastes', {})[pid.hex()] = { + 'key': pkey.hex() + } + + _dump(model, key, backend, salt, hmac_iter) def authenticate( @@ -154,22 +161,36 @@ def authenticate( proto = User(sub) + bogus_decline_msg = 'unable to authenticate' + try: - model = load(proto, key, backend, salt, hmac_iter) + model = _load(proto, key, backend, salt, hmac_iter) except IndexError as e: - raise AuthenticationError('you dun goofed') + raise AuthenticationError(bogus_decline_msg) from e if not model: - model = User(sub, key_hash, Index({})) - dump(model, key, backend, salt, hmac_iter) + data = { + 'auth_expires': int(time()) + (1 * 60) + } + + model = User(sub, key_hash, Index(data)) + _dump(model, key, backend, salt, hmac_iter) else: if model.key_hash != key_hash: - raise AuthenticationError('you dun goofed') + raise AuthenticationError(bogus_decline_msg) return { 'sub': sub, 'master_key': key } + + +__all__ = [ + AuthenticationError, + load_paste_key, + dump_paste_key, + authenticate +] \ No newline at end of file From 27bc790d5d860e06d18e319f2a497f16f0ff8ee5 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 18:47:11 +0200 Subject: [PATCH 3/4] feat: add backend sanitizing --- src/httpaste/__main__.py | 22 +++++++++++++++++++++- src/httpaste/backend/file/__init__.py | 14 ++++++++++++++ src/httpaste/backend/file/paste.py | 17 +++++++++++++++++ src/httpaste/backend/file/user.py | 5 +++++ src/httpaste/backend/sqlite/__init__.py | 8 ++++++++ src/httpaste/backend/sqlite/paste.py | 12 ++++++++++++ src/httpaste/backend/sqlite/user.py | 5 +++++ 7 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/httpaste/__main__.py b/src/httpaste/__main__.py index 399417f..4a6aae4 100644 --- a/src/httpaste/__main__.py +++ b/src/httpaste/__main__.py @@ -92,6 +92,20 @@ def command_init_backend(**kwargs): config.backend.paste.init() +def command_sanitize_backend(**kwargs): + """sanitize the backend + """ + + from httpaste import load_config + + config, _ = load_config(kwargs.get('config')) + + config, _ = load_config(kwargs.get('config')) + + config.backend.user.sanitize() + config.backend.paste.sanitize() + + def parser(): p = argparse.ArgumentParser(description='Process some integers.') @@ -121,6 +135,11 @@ def parser(): help=command_init_backend.__doc__) p_init_backend.add_argument('--config', '-c', required=True) + p_sanitize_backend = sp.add_parser( + 'sanitize-backend', + help=command_sanitize_backend.__doc__) + p_sanitize_backend.add_argument('--config', '-c', required=True) + return p @@ -136,7 +155,8 @@ def main(): 'cgi': command_cgi, 'fcgi': command_fcgi, 'default-config': command_default_config, - 'init-backend': command_init_backend + 'init-backend': command_init_backend, + 'sanitize-backend': command_sanitize_backend }[kwargs.pop('command')](**kwargs) diff --git a/src/httpaste/backend/file/__init__.py b/src/httpaste/backend/file/__init__.py index a8d1126..9352f8f 100644 --- a/src/httpaste/backend/file/__init__.py +++ b/src/httpaste/backend/file/__init__.py @@ -58,6 +58,13 @@ class User(object): return user.init(self.path) + def sanitize(self): + + if self.path.exists(): + return user.sanitize(self.path, self.model_class, self.model_schema) + + return None + class Paste(object): """Filesystem paste model backend @@ -96,3 +103,10 @@ class Paste(object): def init(self): return paste.init(self.path) + + def sanitize(self): + + if self.path.exists(): + return paste.sanitize(self.path, self.model_class, self.model_schema) + + return None \ No newline at end of file diff --git a/src/httpaste/backend/file/paste.py b/src/httpaste/backend/file/paste.py index faefaf0..dbeb341 100644 --- a/src/httpaste/backend/file/paste.py +++ b/src/httpaste/backend/file/paste.py @@ -5,6 +5,7 @@ acting as cells. """ from pathlib import Path from ast import literal_eval +from time import time COLUMNS = [ @@ -98,6 +99,22 @@ def init(path: Path): return None +def sanitize(path: Path, model_class: type, model_schema: type): + + for row in path.iterdir(): + + expiration_cell = row.joinpath('expiration') + + if not expiration_cell.exists(): + continue + + expiration = literal_eval(expiration_cell.read_text()) + + if expiration < int(time()) and expiration > 0: + + delete(model_class(bytes.fromhex(row.name)), path) + + def _rm_tree(pth: Path): for child in pth.iterdir(): if child.is_file(): diff --git a/src/httpaste/backend/file/user.py b/src/httpaste/backend/file/user.py index 72c9256..21e32d0 100644 --- a/src/httpaste/backend/file/user.py +++ b/src/httpaste/backend/file/user.py @@ -82,6 +82,11 @@ def init(path: Path): return None +def sanitize(path: Path, model_class: type, model_schema: type): + + return None + + def _rm_tree(pth: Path): for child in pth.iterdir(): if child.is_file(): diff --git a/src/httpaste/backend/sqlite/__init__.py b/src/httpaste/backend/sqlite/__init__.py index 8bd14e5..5680f5c 100644 --- a/src/httpaste/backend/sqlite/__init__.py +++ b/src/httpaste/backend/sqlite/__init__.py @@ -45,6 +45,10 @@ class User(object): return user.init(self.connection) + def sanitize(self): + + return user.sanitize(self.connection, self.model_class) + class Paste(object): """SQLite paste model backend @@ -74,6 +78,10 @@ class Paste(object): return paste.init(self.connection) + def sanitize(self): + + return paste.sanitize(self.connection, self.model_class) + def get_connection(parameters: Parameters): """get an sqlite connection object diff --git a/src/httpaste/backend/sqlite/paste.py b/src/httpaste/backend/sqlite/paste.py index 150e3cb..bb0eb17 100644 --- a/src/httpaste/backend/sqlite/paste.py +++ b/src/httpaste/backend/sqlite/paste.py @@ -2,6 +2,7 @@ """ from os import path from sqlite3 import Connection +from time import time def load(proto: object, connection: Connection, model_class: type): @@ -67,3 +68,14 @@ def init(connection: Connection): cur.execute(fh.read()) connection.commit() + + +def sanitize(connection: Connection, model_class: type) -> bool: + + cur = connection.cursor() + + cur.execute('''SELECT pid FROM pastes WHERE expiration < ? AND expiration > 0''', (int(time()),)) + + for row in cur.fetchall(): + + delete(model_class(row['pid'])) \ No newline at end of file diff --git a/src/httpaste/backend/sqlite/user.py b/src/httpaste/backend/sqlite/user.py index 423b797..11fc53e 100644 --- a/src/httpaste/backend/sqlite/user.py +++ b/src/httpaste/backend/sqlite/user.py @@ -53,3 +53,8 @@ def init(connection: Connection): cur.execute(fh.read()) connection.commit() + + +def sanitize(connection: Connection, model_class) -> bool: + + return None \ No newline at end of file From dc5c644f29ab52621cbe77a99e41c4dc29a56a81 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 2 Apr 2022 18:50:19 +0200 Subject: [PATCH 4/4] fix(model/paste): remove faulty var from get_safe() --- src/httpaste/model/paste.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/httpaste/model/paste.py b/src/httpaste/model/paste.py index 04ea6a5..f7f856d 100755 --- a/src/httpaste/model/paste.py +++ b/src/httpaste/model/paste.py @@ -300,4 +300,4 @@ def get_safe( model = load_safe(Paste(pid, sub), pkey, backend, salt, hmac_iter) - return PasteData(model.data), model.lifetime, model.encoding + return PasteData(model.data), model.expiration, model.encoding