From de19212a719b0bc6ea5455b3e32072a76d021df6 Mon Sep 17 00:00:00 2001 From: Mia Herkt Date: Fri, 27 Sep 2024 17:39:18 +0200 Subject: PEP8 compliance --- migrations/env.py | 1 + migrations/versions/0659d7b9eea8_.py | 4 -- migrations/versions/0cd36ecdd937_.py | 30 ++++------ .../versions/30bfe33aa328_add_file_size_field.py | 5 +- .../versions/5cda1743b92d_add_request_filters.py | 67 +++++++++++----------- migrations/versions/5cee97aab219_.py | 7 +-- migrations/versions/7e246705da6a_.py | 4 -- migrations/versions/939a08e1d6e5_.py | 38 +++++++----- ...766afb7d2_store_user_agent_string_with_files.py | 6 -- migrations/versions/e2e816056589_.py | 4 -- 10 files changed, 75 insertions(+), 91 deletions(-) (limited to 'migrations') diff --git a/migrations/env.py b/migrations/env.py index 4593816..46a8e20 100755 --- a/migrations/env.py +++ b/migrations/env.py @@ -81,6 +81,7 @@ def run_migrations_online(): finally: connection.close() + if context.is_offline_mode(): run_migrations_offline() else: diff --git a/migrations/versions/0659d7b9eea8_.py b/migrations/versions/0659d7b9eea8_.py index 2ef2151..1763219 100644 --- a/migrations/versions/0659d7b9eea8_.py +++ b/migrations/versions/0659d7b9eea8_.py @@ -15,12 +15,8 @@ import sqlalchemy as sa def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.add_column('file', sa.Column('mgmt_token', sa.String(), nullable=True)) - # ### end Alembic commands ### def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.drop_column('file', 'mgmt_token') - # ### end Alembic commands ### diff --git a/migrations/versions/0cd36ecdd937_.py b/migrations/versions/0cd36ecdd937_.py index 349cab9..6e9c87a 100644 --- a/migrations/versions/0cd36ecdd937_.py +++ b/migrations/versions/0cd36ecdd937_.py @@ -15,28 +15,22 @@ import sqlalchemy as sa def upgrade(): - ### commands auto generated by Alembic - please adjust! ### op.create_table('URL', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('url', sa.UnicodeText(), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('url') - ) + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('url', sa.UnicodeText(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('url')) op.create_table('file', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('sha256', sa.String(), nullable=True), - sa.Column('ext', sa.UnicodeText(), nullable=True), - sa.Column('mime', sa.UnicodeText(), nullable=True), - sa.Column('addr', sa.UnicodeText(), nullable=True), - sa.Column('removed', sa.Boolean(), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('sha256') - ) - ### end Alembic commands ### + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('sha256', sa.String(), nullable=True), + sa.Column('ext', sa.UnicodeText(), nullable=True), + sa.Column('mime', sa.UnicodeText(), nullable=True), + sa.Column('addr', sa.UnicodeText(), nullable=True), + sa.Column('removed', sa.Boolean(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('sha256')) def downgrade(): - ### commands auto generated by Alembic - please adjust! ### op.drop_table('file') op.drop_table('URL') - ### end Alembic commands ### diff --git a/migrations/versions/30bfe33aa328_add_file_size_field.py b/migrations/versions/30bfe33aa328_add_file_size_field.py index e6ac279..b1032c4 100644 --- a/migrations/versions/30bfe33aa328_add_file_size_field.py +++ b/migrations/versions/30bfe33aa328_add_file_size_field.py @@ -19,6 +19,7 @@ from pathlib import Path Base = automap_base() + def upgrade(): op.add_column('file', sa.Column('size', sa.BigInteger(), nullable=True)) bind = op.get_bind() @@ -34,8 +35,8 @@ def upgrade(): p = storage / f.sha256 if p.is_file(): updates.append({ - "id" : f.id, - "size" : p.stat().st_size + "id": f.id, + "size": p.stat().st_size }) session.bulk_update_mappings(File, updates) diff --git a/migrations/versions/5cda1743b92d_add_request_filters.py b/migrations/versions/5cda1743b92d_add_request_filters.py index 3992e14..b03f1aa 100644 --- a/migrations/versions/5cda1743b92d_add_request_filters.py +++ b/migrations/versions/5cda1743b92d_add_request_filters.py @@ -19,47 +19,48 @@ import ipaddress Base = automap_base() + def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.create_table('request_filter', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('type', sa.String(length=20), nullable=False), - sa.Column('comment', sa.UnicodeText(), nullable=True), - sa.Column('addr', sa.LargeBinary(length=16), nullable=True), - sa.Column('net', sa.Text(), nullable=True), - sa.Column('regex', sa.UnicodeText(), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('addr') - ) - with op.batch_alter_table('request_filter', schema=None) as batch_op: - batch_op.create_index(batch_op.f('ix_request_filter_type'), ['type'], unique=False) + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('type', sa.String(length=20), nullable=False), + sa.Column('comment', sa.UnicodeText(), nullable=True), + sa.Column('addr', sa.LargeBinary(length=16), + nullable=True), + sa.Column('net', sa.Text(), nullable=True), + sa.Column('regex', sa.UnicodeText(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('addr')) - # ### end Alembic commands ### + with op.batch_alter_table('request_filter', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_request_filter_type'), ['type'], + unique=False) bind = op.get_bind() Base.prepare(autoload_with=bind) RequestFilter = Base.classes.request_filter session = Session(bind=bind) - if "FHOST_UPLOAD_BLACKLIST" in current_app.config: - if current_app.config["FHOST_UPLOAD_BLACKLIST"]: - with current_app.open_instance_resource(current_app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl: - for l in bl.readlines(): - if not l.startswith("#"): - l = l.strip() - if l.endswith(":"): - # old implementation uses str.startswith, - # which does not translate to networks - current_app.logger.warning(f"Ignored address: {l}") - continue - - flt = RequestFilter(type="addr", addr=ipaddress.ip_address(l).packed) - session.add(flt) - - if "FHOST_MIME_BLACKLIST" in current_app.config: - for mime in current_app.config["FHOST_MIME_BLACKLIST"]: - flt = RequestFilter(type="mime", regex=mime) - session.add(flt) + blp = current_app.config.get("FHOST_UPLOAD_BLACKLIST") + if blp: + with current_app.open_instance_resource(blp, "r") as bl: + for line in bl.readlines(): + if not line.startswith("#"): + line = line.strip() + if line.endswith(":"): + # old implementation uses str.startswith, + # which does not translate to networks + current_app.logger.warning( + f"Ignored address: {line}") + continue + + addr = ipaddress.ip_address(line).packed + flt = RequestFilter(type="addr", addr=addr) + session.add(flt) + + for mime in current_app.config.get("FHOST_MIME_BLACKLIST", []): + flt = RequestFilter(type="mime", regex=mime) + session.add(flt) session.commit() @@ -72,9 +73,7 @@ def upgrade(): def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('request_filter', schema=None) as batch_op: batch_op.drop_index(batch_op.f('ix_request_filter_type')) op.drop_table('request_filter') - # ### end Alembic commands ### diff --git a/migrations/versions/5cee97aab219_.py b/migrations/versions/5cee97aab219_.py index 6c1a16b..4ebee5f 100644 --- a/migrations/versions/5cee97aab219_.py +++ b/migrations/versions/5cee97aab219_.py @@ -15,12 +15,9 @@ import sqlalchemy as sa def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('file', sa.Column('last_vscan', sa.DateTime(), nullable=True)) - # ### end Alembic commands ### + op.add_column('file', sa.Column('last_vscan', sa.DateTime(), + nullable=True)) def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.drop_column('file', 'last_vscan') - # ### end Alembic commands ### diff --git a/migrations/versions/7e246705da6a_.py b/migrations/versions/7e246705da6a_.py index 33dbf79..e8617b3 100644 --- a/migrations/versions/7e246705da6a_.py +++ b/migrations/versions/7e246705da6a_.py @@ -15,12 +15,8 @@ import sqlalchemy as sa def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.add_column('file', sa.Column('nsfw_score', sa.Float(), nullable=True)) - # ### end Alembic commands ### def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.drop_column('file', 'nsfw_score') - # ### end Alembic commands ### diff --git a/migrations/versions/939a08e1d6e5_.py b/migrations/versions/939a08e1d6e5_.py index e389b41..9e69758 100644 --- a/migrations/versions/939a08e1d6e5_.py +++ b/migrations/versions/939a08e1d6e5_.py @@ -21,24 +21,29 @@ from sqlalchemy.orm import Session import os import time -""" For a file of a given size, determine the largest allowed lifespan of that file -Based on the current app's configuration: Specifically, the MAX_CONTENT_LENGTH, as well -as FHOST_{MIN,MAX}_EXPIRATION. +""" +For a file of a given size, determine the largest allowed lifespan of that file + +Based on the current app's configuration: +Specifically, the MAX_CONTENT_LENGTH, as well as FHOST_{MIN,MAX}_EXPIRATION. -This lifespan may be shortened by a user's request, but no files should be allowed to -expire at a point after this number. +This lifespan may be shortened by a user's request, but no files should be +allowed to expire at a point after this number. Value returned is a duration in milliseconds. """ def get_max_lifespan(filesize: int) -> int: - min_exp = current_app.config.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000) - max_exp = current_app.config.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000) - max_size = current_app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024) + cfg = current_app.config + min_exp = cfg.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000) + max_exp = cfg.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000) + max_size = cfg.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024) return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3) + Base = automap_base() + def upgrade(): op.add_column('file', sa.Column('expiration', sa.BigInteger())) @@ -48,14 +53,14 @@ def upgrade(): session = Session(bind=bind) storage = Path(current_app.config["FHOST_STORAGE_PATH"]) - current_time = time.time() * 1000; + current_time = time.time() * 1000 # List of file hashes which have not expired yet # This could get really big for some servers try: unexpired_files = os.listdir(storage) except FileNotFoundError: - return # There are no currently unexpired files + return # There are no currently unexpired files # Calculate an expiration date for all existing files @@ -65,7 +70,7 @@ def upgrade(): sa.not_(File.removed) ) ) - updates = [] # We coalesce updates to the database here + updates = [] # We coalesce updates to the database here # SQLite has a hard limit on the number of variables so we # need to do this the slow way @@ -74,13 +79,18 @@ def upgrade(): for file in files: file_path = storage / file.sha256 stat = os.stat(file_path) - max_age = get_max_lifespan(stat.st_size) # How long the file is allowed to live, in ms - file_birth = stat.st_mtime * 1000 # When the file was created, in ms - updates.append({'id': file.id, 'expiration': int(file_birth + max_age)}) + # How long the file is allowed to live, in ms + max_age = get_max_lifespan(stat.st_size) + # When the file was created, in ms + file_birth = stat.st_mtime * 1000 + updates.append({ + 'id': file.id, + 'expiration': int(file_birth + max_age)}) # Apply coalesced updates session.bulk_update_mappings(File, updates) session.commit() + def downgrade(): op.drop_column('file', 'expiration') diff --git a/migrations/versions/dd0766afb7d2_store_user_agent_string_with_files.py b/migrations/versions/dd0766afb7d2_store_user_agent_string_with_files.py index 4af7680..914d31a 100644 --- a/migrations/versions/dd0766afb7d2_store_user_agent_string_with_files.py +++ b/migrations/versions/dd0766afb7d2_store_user_agent_string_with_files.py @@ -15,16 +15,10 @@ import sqlalchemy as sa def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('file', schema=None) as batch_op: batch_op.add_column(sa.Column('ua', sa.UnicodeText(), nullable=True)) - # ### end Alembic commands ### - def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table('file', schema=None) as batch_op: batch_op.drop_column('ua') - - # ### end Alembic commands ### diff --git a/migrations/versions/e2e816056589_.py b/migrations/versions/e2e816056589_.py index 7c31ba9..b803ede 100644 --- a/migrations/versions/e2e816056589_.py +++ b/migrations/versions/e2e816056589_.py @@ -15,12 +15,8 @@ import sqlalchemy as sa def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.add_column('file', sa.Column('secret', sa.String(), nullable=True)) - # ### end Alembic commands ### def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.drop_column('file', 'secret') - # ### end Alembic commands ### -- cgit v1.2.3