Implement request filters

This moves preexisting blacklists to the database, and adds the
following filter types:

    * IP address
    * IP network
    * MIME type
    * User agent

In addition, IP address handling is now done with the ipaddress
module.

2 files changed, 158 insertions, 0 deletions

diff --git a/migrations/versions/5cda1743b92d_add_request_filters.py b/migrations/versions/5cda1743b92d_add_request_filters.py
new file mode 100644
index 0000000..3992e14
--- /dev/null
+++ b/migrations/versions/5cda1743b92d_add_request_filters.py
@@ -0,0 +1,80 @@
+"""Add request filters
+
+Revision ID: 5cda1743b92d
+Revises: dd0766afb7d2
+Create Date: 2024-09-27 12:13:16.845981
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '5cda1743b92d'
+down_revision = 'dd0766afb7d2'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.ext.automap import automap_base
+from sqlalchemy.orm import Session
+from flask import current_app
+import ipaddress
+
+Base = automap_base()
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('request_filter',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('type', sa.String(length=20), nullable=False),
+    sa.Column('comment', sa.UnicodeText(), nullable=True),
+    sa.Column('addr', sa.LargeBinary(length=16), nullable=True),
+    sa.Column('net', sa.Text(), nullable=True),
+    sa.Column('regex', sa.UnicodeText(), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('addr')
+    )
+    with op.batch_alter_table('request_filter', schema=None) as batch_op:
+        batch_op.create_index(batch_op.f('ix_request_filter_type'), ['type'], unique=False)
+
+    # ### end Alembic commands ###
+
+    bind = op.get_bind()
+    Base.prepare(autoload_with=bind)
+    RequestFilter = Base.classes.request_filter
+    session = Session(bind=bind)
+
+    if "FHOST_UPLOAD_BLACKLIST" in current_app.config:
+        if current_app.config["FHOST_UPLOAD_BLACKLIST"]:
+            with current_app.open_instance_resource(current_app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl:
+                for l in bl.readlines():
+                    if not l.startswith("#"):
+                        l = l.strip()
+                        if l.endswith(":"):
+                            # old implementation uses str.startswith,
+                            # which does not translate to networks
+                            current_app.logger.warning(f"Ignored address: {l}")
+                            continue
+
+                        flt = RequestFilter(type="addr", addr=ipaddress.ip_address(l).packed)
+                        session.add(flt)
+
+    if "FHOST_MIME_BLACKLIST" in current_app.config:
+        for mime in current_app.config["FHOST_MIME_BLACKLIST"]:
+            flt = RequestFilter(type="mime", regex=mime)
+            session.add(flt)
+
+    session.commit()
+
+    w = "Entries in your host and MIME blacklists have been migrated to " \
+        "request filters and stored in the databaes, where possible. " \
+        "The corresponding files and config options may now be deleted. " \
+        "Note that you may have to manually restore them if you wish to " \
+        "revert this with a db downgrade operation."
+    current_app.logger.warning(w)
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('request_filter', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_request_filter_type'))
+
+    op.drop_table('request_filter')
+    # ### end Alembic commands ###
diff --git a/migrations/versions/d9a53a28ba54_change_file_addr_to_ipaddress_type.py b/migrations/versions/d9a53a28ba54_change_file_addr_to_ipaddress_type.py
new file mode 100644
index 0000000..2a46a27
--- /dev/null
+++ b/migrations/versions/d9a53a28ba54_change_file_addr_to_ipaddress_type.py
@@ -0,0 +1,78 @@
+"""Change File.addr to IPAddress type
+
+Revision ID: d9a53a28ba54
+Revises: 5cda1743b92d
+Create Date: 2024-09-27 14:03:06.764764
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'd9a53a28ba54'
+down_revision = '5cda1743b92d'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.ext.automap import automap_base
+from sqlalchemy.orm import Session
+from flask import current_app
+import ipaddress
+
+Base = automap_base()
+
+
+def upgrade():
+    with op.batch_alter_table('file', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('addr_tmp', sa.LargeBinary(16),
+                                      nullable=True))
+
+    bind = op.get_bind()
+    Base.prepare(autoload_with=bind)
+    File = Base.classes.file
+    session = Session(bind=bind)
+
+    updates = []
+    stmt = sa.select(File).where(sa.not_(File.addr == None))
+    for f in session.scalars(stmt.execution_options(yield_per=1000)):
+        addr = ipaddress.ip_address(f.addr)
+        if type(addr) is ipaddress.IPv6Address:
+            addr = addr.ipv4_mapped or addr
+
+        updates.append({
+            "id": f.id,
+            "addr_tmp": addr.packed
+        })
+    session.execute(sa.update(File), updates)
+
+    with op.batch_alter_table('file', schema=None) as batch_op:
+        batch_op.drop_column('addr')
+        batch_op.alter_column('addr_tmp', new_column_name='addr')
+
+
+def downgrade():
+    with op.batch_alter_table('file', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('addr_tmp', sa.UnicodeText,
+                                      nullable=True))
+
+    bind = op.get_bind()
+    Base.prepare(autoload_with=bind)
+    File = Base.classes.file
+    session = Session(bind=bind)
+
+    updates = []
+    stmt = sa.select(File).where(sa.not_(File.addr == None))
+    for f in session.scalars(stmt.execution_options(yield_per=1000)):
+        addr = ipaddress.ip_address(f.addr)
+        if type(addr) is ipaddress.IPv6Address:
+            addr = addr.ipv4_mapped or addr
+
+        updates.append({
+            "id": f.id,
+            "addr_tmp": addr.compressed
+        })
+
+    session.execute(sa.update(File), updates)
+
+    with op.batch_alter_table('file', schema=None) as batch_op:
+        batch_op.drop_column('addr')
+        batch_op.alter_column('addr_tmp', new_column_name='addr')
+