From 0b60f0c82cebf67a3c6a5d66105c7fb1bbf7c594 Mon Sep 17 00:00:00 2001 From: Nemanja Trivic Date: Tue, 27 Feb 2024 13:14:23 +0100 Subject: [PATCH 01/13] implemented DynamoDBSessionInterface and tests. --- requirements/dev.txt | 2 + src/flask_session/__init__.py | 15 ++++ src/flask_session/defaults.py | 4 + src/flask_session/dynamodb/__init__.py | 1 + src/flask_session/dynamodb/dynamodb.py | 106 +++++++++++++++++++++++++ tests/test_dynamodb.py | 52 ++++++++++++ 6 files changed, 180 insertions(+) create mode 100644 src/flask_session/dynamodb/__init__.py create mode 100644 src/flask_session/dynamodb/dynamodb.py create mode 100644 tests/test_dynamodb.py diff --git a/requirements/dev.txt b/requirements/dev.txt index 68c084d0..14205475 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -15,4 +15,6 @@ redis pymemcache Flask-SQLAlchemy pymongo +boto3 +mypy_boto3_dynamodb diff --git a/src/flask_session/__init__.py b/src/flask_session/__init__.py index 12baa39f..0f9a5ee6 100644 --- a/src/flask_session/__init__.py +++ b/src/flask_session/__init__.py @@ -104,6 +104,12 @@ def _get_interface(self, app): "SESSION_CLEANUP_N_REQUESTS", Defaults.SESSION_CLEANUP_N_REQUESTS ) + # DynamoDB settings + SESSION_DYNAMODB = config.get("SESSION_DYNAMODB", Defaults.SESSION_DYNAMODB) + SESSION_DYNAMODB_TABLE = config.get( + "SESSION_DYNAMODB_TABLE", Defaults.SESSION_DYNAMODB_TABLE + ) + common_params = { "app": app, "key_prefix": SESSION_KEY_PREFIX, @@ -165,6 +171,15 @@ def _get_interface(self, app): bind_key=SESSION_SQLALCHEMY_BIND_KEY, cleanup_n_requests=SESSION_CLEANUP_N_REQUESTS, ) + elif SESSION_TYPE == "dynamodb": + from .dynamodb import DynamoDBSessionInterface + + session_interface = DynamoDBSessionInterface( + **common_params, + client=SESSION_DYNAMODB, + table_name=SESSION_DYNAMODB_TABLE, + ) + else: raise ValueError(f"Unrecognized value for SESSION_TYPE: {SESSION_TYPE}") diff --git a/src/flask_session/defaults.py b/src/flask_session/defaults.py index 4823a851..d785ea81 100644 --- a/src/flask_session/defaults.py +++ b/src/flask_session/defaults.py @@ -39,3 +39,7 @@ class Defaults: SESSION_SQLALCHEMY_SEQUENCE = None SESSION_SQLALCHEMY_SCHEMA = None SESSION_SQLALCHEMY_BIND_KEY = None + + # DynamoDB settings + SESSION_DYNAMODB = None + SESSION_DYNAMODB_TABLE = "FlaskSession" diff --git a/src/flask_session/dynamodb/__init__.py b/src/flask_session/dynamodb/__init__.py new file mode 100644 index 00000000..a63048f3 --- /dev/null +++ b/src/flask_session/dynamodb/__init__.py @@ -0,0 +1 @@ +from .dynamodb import DynamoDBSession, DynamoDBSessionInterface # noqa: F401 diff --git a/src/flask_session/dynamodb/dynamodb.py b/src/flask_session/dynamodb/dynamodb.py new file mode 100644 index 00000000..e91c4aff --- /dev/null +++ b/src/flask_session/dynamodb/dynamodb.py @@ -0,0 +1,106 @@ +from decimal import Decimal +import warnings +from datetime import datetime +from datetime import timedelta as TimeDelta +from typing import Optional +from mypy_boto3_dynamodb import DynamoDBServiceResource +import boto3 + +try: + import cPickle as pickle +except ImportError: + import pickle + +from datetime import datetime, timezone + +from itsdangerous import want_bytes + +from ..base import ServerSideSession, ServerSideSessionInterface +from ..defaults import Defaults + + +class DynamoDBSession(ServerSideSession): + pass + + +class DynamoDBSessionInterface(ServerSideSessionInterface): + """A Session interface that uses dynamodb as backend. (`boto3` required) + + :param client: A ``DynamoDBServiceResource`` instance. + :param key_prefix: A prefix that is added to all MongoDB store keys. + :param use_signer: Whether to sign the session id cookie or not. + :param permanent: Whether to use permanent session or not. + :param sid_length: The length of the generated session id in bytes. + :param table_name: DynamoDB table name to store the session. + + .. versionadded:: 0.6 + The `sid_length` parameter was added. + + .. versionadded:: 0.2 + The `use_signer` parameter was added. + """ + + serializer = pickle + session_class = DynamoDBSession + + def __init__( + self, + client: Optional[DynamoDBServiceResource] = Defaults.SESSION_DYNAMODB, + key_prefix: str = Defaults.SESSION_KEY_PREFIX, + use_signer: bool = Defaults.SESSION_USE_SIGNER, + permanent: bool = Defaults.SESSION_PERMANENT, + sid_length: int = Defaults.SESSION_SID_LENGTH, + serialization_format: str = Defaults.SESSION_SERIALIZATION_FORMAT, + table_name: str = Defaults.SESSION_DYNAMODB_TABLE, + ): + if client is None: + warnings.warn( + "No valid MongoClient instance provided, attempting to create a new instance on localhost with default settings.", + RuntimeWarning, + stacklevel=1, + ) + client = boto3.resource("dynamodb", endpoint_url="http://localhost:8000") + try: + client.meta.client.update_time_to_live( + TableName=self.table_name, + TimeToLiveSpecification={ + "Enabled": True, + "AttributeName": "expiration", + }, + ) + except AttributeError: + pass + + self.client = client + self.store = client.Table(table_name) + super().__init__(self.store, key_prefix, use_signer, permanent, sid_length) + + def _retrieve_session_data(self, store_id: str) -> Optional[dict]: + # Get the saved session (document) from the database + document = self.store.get_item(Key={"id": store_id})["Item"] + if document: + serialized_session_data = want_bytes(document["val"].value) + return self.serializer.decode(serialized_session_data) + return None + + def _delete_session(self, store_id: str) -> None: + self.store.delete_item(Key={"id": store_id}) + + def _upsert_session( + self, session_lifetime: TimeDelta, session: ServerSideSession, store_id: str + ) -> None: + storage_expiration_datetime = datetime.utcnow() + session_lifetime + # Serialize the session data + serialized_session_data = self.serializer.encode(session) + print(storage_expiration_datetime.timestamp()) + + self.store.update_item( + Key={ + "id": store_id, + }, + UpdateExpression="SET val = :value, expiration = :exp", + ExpressionAttributeValues={ + ":value": serialized_session_data, + ":exp": Decimal(storage_expiration_datetime.timestamp()), + }, + ) diff --git a/tests/test_dynamodb.py b/tests/test_dynamodb.py new file mode 100644 index 00000000..d3c7b1b7 --- /dev/null +++ b/tests/test_dynamodb.py @@ -0,0 +1,52 @@ +import datetime +import json +from contextlib import contextmanager +import time + +import flask +from flask_session.dynamodb import DynamoDBSession +from itsdangerous import want_bytes +import boto3 + + +class TestMongoSession: + """This requires package: boto3""" + + @contextmanager + def setup_dynamodb(self): + self.client = boto3.resource("dynamodb", endpoint_url="http://localhost:8000") + self.store = self.client.Table("flask-session") + try: + scan = self.store.scan() + with self.store.batch_writer() as batch: + for each in scan["Items"]: + batch.delete_item( + Key={ + "id": each["id"], + } + ) + yield + finally: + scan = self.store.scan() + with self.store.batch_writer() as batch: + for each in scan["Items"]: + batch.delete_item( + Key={ + "id": each["id"], + } + ) + pass + + def test_dynamodb_default(self, app_utils): + with self.setup_dynamodb(): + app = app_utils.create_app( + { + "SESSION_TYPE": "dynamodb", + "SESSION_DYNAMODB": self.client, + "SESSION_DYNAMODB_TABLE": "flask-sessions", + } + ) + + with app.test_request_context(): + assert isinstance(flask.session, DynamoDBSession) + app_utils.test_session(app) From 1f8acb38a326fba24cdc836e38de24cd9d441dd3 Mon Sep 17 00:00:00 2001 From: Nemanja Trivic Date: Tue, 27 Feb 2024 21:26:40 +0100 Subject: [PATCH 02/13] Corrected table name. --- tests/test_dynamodb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dynamodb.py b/tests/test_dynamodb.py index d3c7b1b7..e7e17e85 100644 --- a/tests/test_dynamodb.py +++ b/tests/test_dynamodb.py @@ -15,7 +15,7 @@ class TestMongoSession: @contextmanager def setup_dynamodb(self): self.client = boto3.resource("dynamodb", endpoint_url="http://localhost:8000") - self.store = self.client.Table("flask-session") + self.store = self.client.Table("FlaskSession") try: scan = self.store.scan() with self.store.batch_writer() as batch: @@ -43,7 +43,7 @@ def test_dynamodb_default(self, app_utils): { "SESSION_TYPE": "dynamodb", "SESSION_DYNAMODB": self.client, - "SESSION_DYNAMODB_TABLE": "flask-sessions", + "SESSION_DYNAMODB_TABLE": "FlaskSession", } ) From 37bff6e5aa9a9585f55996f1e90afe50fc8e89ba Mon Sep 17 00:00:00 2001 From: Nemanja Trivic Date: Tue, 27 Feb 2024 21:32:11 +0100 Subject: [PATCH 03/13] Removing monogdb mentions. --- src/flask_session/dynamodb/dynamodb.py | 4 ++-- tests/test_dynamodb.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/flask_session/dynamodb/dynamodb.py b/src/flask_session/dynamodb/dynamodb.py index e91c4aff..6266e20a 100644 --- a/src/flask_session/dynamodb/dynamodb.py +++ b/src/flask_session/dynamodb/dynamodb.py @@ -27,7 +27,7 @@ class DynamoDBSessionInterface(ServerSideSessionInterface): """A Session interface that uses dynamodb as backend. (`boto3` required) :param client: A ``DynamoDBServiceResource`` instance. - :param key_prefix: A prefix that is added to all MongoDB store keys. + :param key_prefix: A prefix that is added to all DynamoDB store keys. :param use_signer: Whether to sign the session id cookie or not. :param permanent: Whether to use permanent session or not. :param sid_length: The length of the generated session id in bytes. @@ -55,7 +55,7 @@ def __init__( ): if client is None: warnings.warn( - "No valid MongoClient instance provided, attempting to create a new instance on localhost with default settings.", + "No valid DynamoDBServiceResource instance provided, attempting to create a new instance on localhost:8000.", RuntimeWarning, stacklevel=1, ) diff --git a/tests/test_dynamodb.py b/tests/test_dynamodb.py index e7e17e85..00e3457c 100644 --- a/tests/test_dynamodb.py +++ b/tests/test_dynamodb.py @@ -9,7 +9,7 @@ import boto3 -class TestMongoSession: +class TestDynamoDBSession: """This requires package: boto3""" @contextmanager From 7cea1697f368074fd8d9ec16575556159554c2a3 Mon Sep 17 00:00:00 2001 From: Nemanja Trivic Date: Wed, 28 Feb 2024 06:34:49 +0100 Subject: [PATCH 04/13] Added url parameter for local testing. --- src/flask_session/__init__.py | 4 ++++ src/flask_session/defaults.py | 1 + src/flask_session/dynamodb/dynamodb.py | 4 +++- tests/test_dynamodb.py | 5 ++++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/flask_session/__init__.py b/src/flask_session/__init__.py index 0f9a5ee6..8903ad36 100644 --- a/src/flask_session/__init__.py +++ b/src/flask_session/__init__.py @@ -109,6 +109,9 @@ def _get_interface(self, app): SESSION_DYNAMODB_TABLE = config.get( "SESSION_DYNAMODB_TABLE", Defaults.SESSION_DYNAMODB_TABLE ) + SESSION_DYNAMODB_URL = config.get( + "SESSION_DYNAMODB_URL", Defaults.SESSION_DYNAMODB_URL + ) common_params = { "app": app, @@ -178,6 +181,7 @@ def _get_interface(self, app): **common_params, client=SESSION_DYNAMODB, table_name=SESSION_DYNAMODB_TABLE, + url=SESSION_DYNAMODB_URL, ) else: diff --git a/src/flask_session/defaults.py b/src/flask_session/defaults.py index d785ea81..5a525db2 100644 --- a/src/flask_session/defaults.py +++ b/src/flask_session/defaults.py @@ -43,3 +43,4 @@ class Defaults: # DynamoDB settings SESSION_DYNAMODB = None SESSION_DYNAMODB_TABLE = "FlaskSession" + SESSION_DYNAMODB_URL = "http://localhost:8000" diff --git a/src/flask_session/dynamodb/dynamodb.py b/src/flask_session/dynamodb/dynamodb.py index 6266e20a..5216d24d 100644 --- a/src/flask_session/dynamodb/dynamodb.py +++ b/src/flask_session/dynamodb/dynamodb.py @@ -32,6 +32,7 @@ class DynamoDBSessionInterface(ServerSideSessionInterface): :param permanent: Whether to use permanent session or not. :param sid_length: The length of the generated session id in bytes. :param table_name: DynamoDB table name to store the session. + :param url: DynamoDB URL for local testing. .. versionadded:: 0.6 The `sid_length` parameter was added. @@ -52,6 +53,7 @@ def __init__( sid_length: int = Defaults.SESSION_SID_LENGTH, serialization_format: str = Defaults.SESSION_SERIALIZATION_FORMAT, table_name: str = Defaults.SESSION_DYNAMODB_TABLE, + url: str = Defaults.SESSION_DYNAMODB_URL, ): if client is None: warnings.warn( @@ -59,7 +61,7 @@ def __init__( RuntimeWarning, stacklevel=1, ) - client = boto3.resource("dynamodb", endpoint_url="http://localhost:8000") + client = boto3.resource("dynamodb", endpoint_url=url) try: client.meta.client.update_time_to_live( TableName=self.table_name, diff --git a/tests/test_dynamodb.py b/tests/test_dynamodb.py index 00e3457c..f634905e 100644 --- a/tests/test_dynamodb.py +++ b/tests/test_dynamodb.py @@ -4,6 +4,7 @@ import time import flask +from flask_session.defaults import Defaults from flask_session.dynamodb import DynamoDBSession from itsdangerous import want_bytes import boto3 @@ -14,7 +15,9 @@ class TestDynamoDBSession: @contextmanager def setup_dynamodb(self): - self.client = boto3.resource("dynamodb", endpoint_url="http://localhost:8000") + self.client = boto3.resource( + "dynamodb", endpoint_url=Defaults.SESSION_DYNAMODB_URL + ) self.store = self.client.Table("FlaskSession") try: scan = self.store.scan() From 3928290cbf393d1ec189c352fcd4ab11759cb8d5 Mon Sep 17 00:00:00 2001 From: Nemanja Trivic Date: Wed, 28 Feb 2024 06:39:46 +0100 Subject: [PATCH 05/13] Using default table in tests. --- tests/test_dynamodb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_dynamodb.py b/tests/test_dynamodb.py index f634905e..c1af8533 100644 --- a/tests/test_dynamodb.py +++ b/tests/test_dynamodb.py @@ -18,7 +18,7 @@ def setup_dynamodb(self): self.client = boto3.resource( "dynamodb", endpoint_url=Defaults.SESSION_DYNAMODB_URL ) - self.store = self.client.Table("FlaskSession") + self.store = self.client.Table(Defaults.SESSION_DYNAMODB_TABLE) try: scan = self.store.scan() with self.store.batch_writer() as batch: @@ -46,7 +46,6 @@ def test_dynamodb_default(self, app_utils): { "SESSION_TYPE": "dynamodb", "SESSION_DYNAMODB": self.client, - "SESSION_DYNAMODB_TABLE": "FlaskSession", } ) From 1ad3eb07304622f9883cd7f4b309ea6784cb7868 Mon Sep 17 00:00:00 2001 From: Nemanja Trivic Date: Wed, 28 Feb 2024 14:55:26 +0100 Subject: [PATCH 06/13] Refactored get instead of literal dict retrieval. --- src/flask_session/dynamodb/dynamodb.py | 4 ++-- tests/test_dynamodb.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/flask_session/dynamodb/dynamodb.py b/src/flask_session/dynamodb/dynamodb.py index 5216d24d..be1a8c2e 100644 --- a/src/flask_session/dynamodb/dynamodb.py +++ b/src/flask_session/dynamodb/dynamodb.py @@ -79,9 +79,9 @@ def __init__( def _retrieve_session_data(self, store_id: str) -> Optional[dict]: # Get the saved session (document) from the database - document = self.store.get_item(Key={"id": store_id})["Item"] + document = self.store.get_item(Key={"id": store_id}).get("Item") if document: - serialized_session_data = want_bytes(document["val"].value) + serialized_session_data = want_bytes(document.get("val").value) return self.serializer.decode(serialized_session_data) return None diff --git a/tests/test_dynamodb.py b/tests/test_dynamodb.py index c1af8533..77b522d1 100644 --- a/tests/test_dynamodb.py +++ b/tests/test_dynamodb.py @@ -22,20 +22,20 @@ def setup_dynamodb(self): try: scan = self.store.scan() with self.store.batch_writer() as batch: - for each in scan["Items"]: + for each in scan.get("Items"): batch.delete_item( Key={ - "id": each["id"], + "id": each.get("id"), } ) yield finally: scan = self.store.scan() with self.store.batch_writer() as batch: - for each in scan["Items"]: + for each in scan.get("Items"): batch.delete_item( Key={ - "id": each["id"], + "id": each.get("id"), } ) pass From 68415e3ed648946072221aeab8838bb354005361 Mon Sep 17 00:00:00 2001 From: Nemanja Trivic Date: Thu, 29 Feb 2024 00:16:14 +0100 Subject: [PATCH 07/13] Automatic table creation and capacity parameters. --- src/flask_session/__init__.py | 8 ++++++++ src/flask_session/defaults.py | 2 ++ src/flask_session/dynamodb/dynamodb.py | 22 +++++++++++++++++++++- tests/test_dynamodb.py | 22 +++++++++++----------- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/src/flask_session/__init__.py b/src/flask_session/__init__.py index 8903ad36..cbf94a56 100644 --- a/src/flask_session/__init__.py +++ b/src/flask_session/__init__.py @@ -112,6 +112,12 @@ def _get_interface(self, app): SESSION_DYNAMODB_URL = config.get( "SESSION_DYNAMODB_URL", Defaults.SESSION_DYNAMODB_URL ) + SESSION_DYNAMODB_READ = config.get( + "SESSION_DYNAMODB_READ", Defaults.SESSION_DYNAMODB_READ + ) + SESSION_DYNAMODB_WRITE = config.get( + "SESSION_DYNAMODB_WRITE", Defaults.SESSION_DYNAMODB_WRITE + ) common_params = { "app": app, @@ -182,6 +188,8 @@ def _get_interface(self, app): client=SESSION_DYNAMODB, table_name=SESSION_DYNAMODB_TABLE, url=SESSION_DYNAMODB_URL, + read_capacity=SESSION_DYNAMODB_READ, + write_capacity=SESSION_DYNAMODB_WRITE, ) else: diff --git a/src/flask_session/defaults.py b/src/flask_session/defaults.py index 5a525db2..5bd221a4 100644 --- a/src/flask_session/defaults.py +++ b/src/flask_session/defaults.py @@ -44,3 +44,5 @@ class Defaults: SESSION_DYNAMODB = None SESSION_DYNAMODB_TABLE = "FlaskSession" SESSION_DYNAMODB_URL = "http://localhost:8000" + SESSION_DYNAMODB_READ = 5 + SESSION_DYNAMODB_WRITE = 5 diff --git a/src/flask_session/dynamodb/dynamodb.py b/src/flask_session/dynamodb/dynamodb.py index be1a8c2e..e5aa1292 100644 --- a/src/flask_session/dynamodb/dynamodb.py +++ b/src/flask_session/dynamodb/dynamodb.py @@ -33,6 +33,8 @@ class DynamoDBSessionInterface(ServerSideSessionInterface): :param sid_length: The length of the generated session id in bytes. :param table_name: DynamoDB table name to store the session. :param url: DynamoDB URL for local testing. + :param read_capacity: DynamoDB table read capacity units. + :param write_capacity: DynamoDB table write capacity units. .. versionadded:: 0.6 The `sid_length` parameter was added. @@ -54,7 +56,10 @@ def __init__( serialization_format: str = Defaults.SESSION_SERIALIZATION_FORMAT, table_name: str = Defaults.SESSION_DYNAMODB_TABLE, url: str = Defaults.SESSION_DYNAMODB_URL, + read_capacity: int = Defaults.SESSION_DYNAMODB_READ, + write_capacity: int = Defaults.SESSION_DYNAMODB_WRITE, ): + if client is None: warnings.warn( "No valid DynamoDBServiceResource instance provided, attempting to create a new instance on localhost:8000.", @@ -62,7 +67,22 @@ def __init__( stacklevel=1, ) client = boto3.resource("dynamodb", endpoint_url=url) + try: + client.create_table( + AttributeDefinitions=[ + {"AttributeName": "id", "AttributeType": "S"}, + ], + TableName=table_name, + KeySchema=[ + {"AttributeName": "id", "KeyType": "HASH"}, + ], + ProvisionedThroughput={ + "ReadCapacityUnits": read_capacity, + "WriteCapacityUnits": write_capacity, + }, + ) + client.meta.client.update_time_to_live( TableName=self.table_name, TimeToLiveSpecification={ @@ -70,7 +90,7 @@ def __init__( "AttributeName": "expiration", }, ) - except AttributeError: + except (AttributeError, client.meta.client.exceptions.ResourceInUseException): pass self.client = client diff --git a/tests/test_dynamodb.py b/tests/test_dynamodb.py index 77b522d1..24f9f54d 100644 --- a/tests/test_dynamodb.py +++ b/tests/test_dynamodb.py @@ -18,8 +18,8 @@ def setup_dynamodb(self): self.client = boto3.resource( "dynamodb", endpoint_url=Defaults.SESSION_DYNAMODB_URL ) - self.store = self.client.Table(Defaults.SESSION_DYNAMODB_TABLE) try: + self.store = self.client.Table(Defaults.SESSION_DYNAMODB_TABLE) scan = self.store.scan() with self.store.batch_writer() as batch: for each in scan.get("Items"): @@ -28,17 +28,17 @@ def setup_dynamodb(self): "id": each.get("id"), } ) - yield - finally: - scan = self.store.scan() - with self.store.batch_writer() as batch: - for each in scan.get("Items"): - batch.delete_item( - Key={ - "id": each.get("id"), - } - ) + except self.client.meta.client.exceptions.ResourceNotFoundException: pass + yield + scan = self.store.scan() + with self.store.batch_writer() as batch: + for each in scan.get("Items"): + batch.delete_item( + Key={ + "id": each.get("id"), + } + ) def test_dynamodb_default(self, app_utils): with self.setup_dynamodb(): From 1f5d5d361de8bd19969977b4efe6a465ebc3445f Mon Sep 17 00:00:00 2001 From: Lex Date: Thu, 21 Mar 2024 22:32:16 +1000 Subject: [PATCH 08/13] Add test action image for dynamodb --- .github/workflows/test.yaml | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 03448275..95283532 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,20 +2,21 @@ name: Run unittests on: [push, pull_request] jobs: unittests: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [3.8, 3.9, 3.10, 3.11, 3.12] - services: - mongodb: - image: mongo - ports: - - 27017:27017 - steps: - - uses: actions/checkout@v4 - - uses: supercharge/redis-github-action@1.5.0 - - uses: niden/actions-memcached@v7 - - name: Install testing requirements - run: pip3 install -r requirements/dev.txt - - name: Run tests - run: pytest tests + runs-on: ubuntu-latest + services: + mongodb: + image: mongo + ports: + - 27017:27017 + dynamodb: + image: amazon/dynamodb-local + ports: + - 8000:8000 + steps: + - uses: actions/checkout@v4 + - uses: supercharge/redis-github-action@1.5.0 + - uses: niden/actions-memcached@v7 + - name: Install testing requirements + run: pip3 install -r requirements/dev.txt + - name: Run tests + run: pytest tests From 14d6a1596942a18673761f17a6df811c28277bc5 Mon Sep 17 00:00:00 2001 From: Lex Date: Thu, 21 Mar 2024 22:34:57 +1000 Subject: [PATCH 09/13] Typo --- .github/workflows/test.yaml | 39 ++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 95283532..3b57e6e9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,21 +2,24 @@ name: Run unittests on: [push, pull_request] jobs: unittests: - runs-on: ubuntu-latest - services: - mongodb: - image: mongo - ports: - - 27017:27017 - dynamodb: - image: amazon/dynamodb-local - ports: - - 8000:8000 - steps: - - uses: actions/checkout@v4 - - uses: supercharge/redis-github-action@1.5.0 - - uses: niden/actions-memcached@v7 - - name: Install testing requirements - run: pip3 install -r requirements/dev.txt - - name: Run tests - run: pytest tests + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8, 3.9, 3.10, 3.11, 3.12] + services: + mongodb: + image: mongo + ports: + - 27017:27017 + dynamodb: + image: amazon/dynamodb-local + ports: + - 8000:8000 + steps: + - uses: actions/checkout@v4 + - uses: supercharge/redis-github-action@1.5.0 + - uses: niden/actions-memcached@v7 + - name: Install testing requirements + run: pip3 install -r requirements/dev.txt + - name: Run tests + run: pytest tests From ab9a756a18de87033cf6d3b2d86c6dc46e006800 Mon Sep 17 00:00:00 2001 From: Lex Date: Fri, 22 Mar 2024 20:44:16 +1000 Subject: [PATCH 10/13] Tidy up dynamo db, add docker and requirements --- docker-compose.yml | 16 ++++++--- pyproject.toml | 2 ++ src/flask_session/__init__.py | 12 ------- src/flask_session/defaults.py | 5 +-- src/flask_session/dynamodb/dynamodb.py | 50 +++++++++++++------------- tests/test_dynamodb.py | 12 +++---- 6 files changed, 45 insertions(+), 52 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c5838f6a..7864e9d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,18 @@ version: '3.8' services: + dynamodb-local: + image: "amazon/dynamodb-local:latest" + container_name: dynamodb-local + ports: + - "8000:8000" + environment: + - AWS_ACCESS_KEY_ID=dummy + - AWS_SECRET_ACCESS_KEY=dummy + - AWS_DEFAULT_REGION=us-west-2 + mongo: image: mongo:latest - environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: example ports: - "27017:27017" volumes: @@ -26,4 +33,5 @@ services: volumes: postgres_data: mongo_data: - redis_data: \ No newline at end of file + redis_data: + dynamodb_data: \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index de127b1e..49c2a4f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,5 +85,7 @@ dev-dependencies = [ "sphinx>=7.1.2", "furo>=2024.1.29", "sphinx-favicon>=1.0.1", + "boto3>=1.34.68", + "mypy_boto3_dynamodb>=1.34.67", "pymemcache>=4.0.0", ] diff --git a/src/flask_session/__init__.py b/src/flask_session/__init__.py index cbf94a56..0f9a5ee6 100644 --- a/src/flask_session/__init__.py +++ b/src/flask_session/__init__.py @@ -109,15 +109,6 @@ def _get_interface(self, app): SESSION_DYNAMODB_TABLE = config.get( "SESSION_DYNAMODB_TABLE", Defaults.SESSION_DYNAMODB_TABLE ) - SESSION_DYNAMODB_URL = config.get( - "SESSION_DYNAMODB_URL", Defaults.SESSION_DYNAMODB_URL - ) - SESSION_DYNAMODB_READ = config.get( - "SESSION_DYNAMODB_READ", Defaults.SESSION_DYNAMODB_READ - ) - SESSION_DYNAMODB_WRITE = config.get( - "SESSION_DYNAMODB_WRITE", Defaults.SESSION_DYNAMODB_WRITE - ) common_params = { "app": app, @@ -187,9 +178,6 @@ def _get_interface(self, app): **common_params, client=SESSION_DYNAMODB, table_name=SESSION_DYNAMODB_TABLE, - url=SESSION_DYNAMODB_URL, - read_capacity=SESSION_DYNAMODB_READ, - write_capacity=SESSION_DYNAMODB_WRITE, ) else: diff --git a/src/flask_session/defaults.py b/src/flask_session/defaults.py index 5bd221a4..7f890d6e 100644 --- a/src/flask_session/defaults.py +++ b/src/flask_session/defaults.py @@ -42,7 +42,4 @@ class Defaults: # DynamoDB settings SESSION_DYNAMODB = None - SESSION_DYNAMODB_TABLE = "FlaskSession" - SESSION_DYNAMODB_URL = "http://localhost:8000" - SESSION_DYNAMODB_READ = 5 - SESSION_DYNAMODB_WRITE = 5 + SESSION_DYNAMODB_TABLE = "Sessions" diff --git a/src/flask_session/dynamodb/dynamodb.py b/src/flask_session/dynamodb/dynamodb.py index e5aa1292..fbd7678c 100644 --- a/src/flask_session/dynamodb/dynamodb.py +++ b/src/flask_session/dynamodb/dynamodb.py @@ -1,18 +1,12 @@ -from decimal import Decimal import warnings from datetime import datetime from datetime import timedelta as TimeDelta +from decimal import Decimal from typing import Optional -from mypy_boto3_dynamodb import DynamoDBServiceResource -import boto3 - -try: - import cPickle as pickle -except ImportError: - import pickle - -from datetime import datetime, timezone +import boto3 +from mypy_boto3_dynamodb.service_resource import DynamoDBServiceResource +from flask import Flask from itsdangerous import want_bytes from ..base import ServerSideSession, ServerSideSessionInterface @@ -32,9 +26,6 @@ class DynamoDBSessionInterface(ServerSideSessionInterface): :param permanent: Whether to use permanent session or not. :param sid_length: The length of the generated session id in bytes. :param table_name: DynamoDB table name to store the session. - :param url: DynamoDB URL for local testing. - :param read_capacity: DynamoDB table read capacity units. - :param write_capacity: DynamoDB table write capacity units. .. versionadded:: 0.6 The `sid_length` parameter was added. @@ -43,21 +34,18 @@ class DynamoDBSessionInterface(ServerSideSessionInterface): The `use_signer` parameter was added. """ - serializer = pickle session_class = DynamoDBSession def __init__( self, + app: Flask, client: Optional[DynamoDBServiceResource] = Defaults.SESSION_DYNAMODB, key_prefix: str = Defaults.SESSION_KEY_PREFIX, use_signer: bool = Defaults.SESSION_USE_SIGNER, permanent: bool = Defaults.SESSION_PERMANENT, - sid_length: int = Defaults.SESSION_SID_LENGTH, + sid_length: int = Defaults.SESSION_ID_LENGTH, serialization_format: str = Defaults.SESSION_SERIALIZATION_FORMAT, table_name: str = Defaults.SESSION_DYNAMODB_TABLE, - url: str = Defaults.SESSION_DYNAMODB_URL, - read_capacity: int = Defaults.SESSION_DYNAMODB_READ, - write_capacity: int = Defaults.SESSION_DYNAMODB_WRITE, ): if client is None: @@ -66,7 +54,13 @@ def __init__( RuntimeWarning, stacklevel=1, ) - client = boto3.resource("dynamodb", endpoint_url=url) + client = boto3.resource( + "dynamodb", + endpoint_url="http://localhost:8000", + region_name="us-west-2", + aws_access_key_id="dummy", + aws_secret_access_key="dummy", + ) try: client.create_table( @@ -77,12 +71,9 @@ def __init__( KeySchema=[ {"AttributeName": "id", "KeyType": "HASH"}, ], - ProvisionedThroughput={ - "ReadCapacityUnits": read_capacity, - "WriteCapacityUnits": write_capacity, - }, + BillingMode="PAY_PER_REQUEST", ) - + client.meta.client.get_waiter("table_exists").wait(TableName=table_name) client.meta.client.update_time_to_live( TableName=self.table_name, TimeToLiveSpecification={ @@ -91,11 +82,19 @@ def __init__( }, ) except (AttributeError, client.meta.client.exceptions.ResourceInUseException): + # TTL already exists, or table already exists pass self.client = client self.store = client.Table(table_name) - super().__init__(self.store, key_prefix, use_signer, permanent, sid_length) + super().__init__( + app, + key_prefix, + use_signer, + permanent, + sid_length, + serialization_format, + ) def _retrieve_session_data(self, store_id: str) -> Optional[dict]: # Get the saved session (document) from the database @@ -114,7 +113,6 @@ def _upsert_session( storage_expiration_datetime = datetime.utcnow() + session_lifetime # Serialize the session data serialized_session_data = self.serializer.encode(session) - print(storage_expiration_datetime.timestamp()) self.store.update_item( Key={ diff --git a/tests/test_dynamodb.py b/tests/test_dynamodb.py index 24f9f54d..a0e1f1fb 100644 --- a/tests/test_dynamodb.py +++ b/tests/test_dynamodb.py @@ -1,13 +1,9 @@ -import datetime -import json from contextlib import contextmanager -import time +import boto3 import flask from flask_session.defaults import Defaults from flask_session.dynamodb import DynamoDBSession -from itsdangerous import want_bytes -import boto3 class TestDynamoDBSession: @@ -16,7 +12,11 @@ class TestDynamoDBSession: @contextmanager def setup_dynamodb(self): self.client = boto3.resource( - "dynamodb", endpoint_url=Defaults.SESSION_DYNAMODB_URL + "dynamodb", + endpoint_url="http://localhost:8000", + region_name="us-west-2", + aws_access_key_id="dummy", + aws_secret_access_key="dummy", ) try: self.store = self.client.Table(Defaults.SESSION_DYNAMODB_TABLE) From a92914ed1196acc2eab4d562996ac944413fb19c Mon Sep 17 00:00:00 2001 From: Lex Date: Fri, 22 Mar 2024 22:12:34 +1000 Subject: [PATCH 11/13] Add docs --- docs/api.rst | 3 ++- docs/config.rst | 15 +++++++++++++++ docs/installation.rst | 2 ++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index 8779f4d6..b31eceb5 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -19,4 +19,5 @@ Anything documented here is part of the public API that Flask-Session provides, .. autoclass:: flask_session.filesystem.FileSystemSessionInterface .. autoclass:: flask_session.cachelib.CacheLibSessionInterface .. autoclass:: flask_session.mongodb.MongoDBSessionInterface -.. autoclass:: flask_session.sqlalchemy.SqlAlchemySessionInterface \ No newline at end of file +.. autoclass:: flask_session.sqlalchemy.SqlAlchemySessionInterface +.. autoclass:: flask_session.dynamodb.DynamoDBSessionInterface \ No newline at end of file diff --git a/docs/config.rst b/docs/config.rst index 44ff02af..e38f8b05 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -211,6 +211,21 @@ SqlAlchemy Default: ``None`` +Dynamodb +~~~~~~~~~~~~~~~~~~~~~~~ + +.. py:data:: SESSION_DYNAMODB + + A ``boto3.resource`` instance. + + Default: Instance connected to ``'localhost:8000'`` + +.. py:data:: SESSION_DYNAMODB_TABLE_NAME + + The name of the table you want to use. + + Default: ``'Sessions'`` + .. deprecated:: 0.7.0 ``SESSION_FILE_DIR``, ``SESSION_FILE_THRESHOLD``, ``SESSION_FILE_MODE``. Use ``SESSION_CACHELIB`` instead. diff --git a/docs/installation.rst b/docs/installation.rst index 1bf19d40..fd00504c 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -42,6 +42,8 @@ Flask-Session has an increasing number of directly supported storage and client - pymongo_ * - SQL Alchemy - flask-sqlalchemy_ + * - DynamoDB + - boto3_ Other libraries may work if they use the same commands as the ones listed above. From 17369528b45d9b5a59003ec006ede35ced40f694 Mon Sep 17 00:00:00 2001 From: Lex Date: Sun, 24 Mar 2024 18:38:04 +1000 Subject: [PATCH 12/13] Fix docs requirements --- requirements/dev.txt | 1 - requirements/docs.in | 4 +++- requirements/docs.txt | 26 ++++++++++++++++++++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 14205475..dd433cac 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -16,5 +16,4 @@ pymemcache Flask-SQLAlchemy pymongo boto3 -mypy_boto3_dynamodb diff --git a/requirements/docs.in b/requirements/docs.in index 7da2e193..211cd708 100644 --- a/requirements/docs.in +++ b/requirements/docs.in @@ -7,4 +7,6 @@ redis cachelib pymongo flask_sqlalchemy -pymemcache \ No newline at end of file +pymemcache +boto3 +mypy_boto3_dynamodb \ No newline at end of file diff --git a/requirements/docs.txt b/requirements/docs.txt index 84bfc531..81adad9c 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -12,6 +12,12 @@ beautifulsoup4==4.12.3 # via furo blinker==1.7.0 # via flask +boto3==1.34.69 + # via -r requirements/docs.in +botocore==1.34.69 + # via + # boto3 + # s3transfer cachelib==0.12.0 # via -r requirements/docs.in certifi==2023.5.7 @@ -40,10 +46,16 @@ jinja2==3.1.2 # via # flask # sphinx +jmespath==1.0.1 + # via + # boto3 + # botocore markupsafe==2.1.2 # via # jinja2 # werkzeug +mypy-boto3-dynamodb==1.34.67 + # via -r requirements/docs.in packaging==23.1 # via sphinx pygments==2.15.1 @@ -54,10 +66,16 @@ pymemcache==4.0.0 # via -r requirements/docs.in pymongo==4.6.2 # via -r requirements/docs.in +python-dateutil==2.9.0.post0 + # via botocore redis==5.0.1 # via -r requirements/docs.in requests==2.30.0 # via sphinx +s3transfer==0.10.1 + # via boto3 +six==1.16.0 + # via python-dateutil snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 @@ -87,8 +105,12 @@ sphinxcontrib-serializinghtml==1.1.5 sqlalchemy==2.0.27 # via flask-sqlalchemy typing-extensions==4.10.0 - # via sqlalchemy + # via + # mypy-boto3-dynamodb + # sqlalchemy urllib3==2.0.2 - # via requests + # via + # botocore + # requests werkzeug==3.0.1 # via flask From 5a3413b9b1d67a4116404f23022b58326a60fbdf Mon Sep 17 00:00:00 2001 From: Lex Date: Sun, 24 Mar 2024 20:48:09 +1000 Subject: [PATCH 13/13] Add changelog and contributor --- CHANGES.rst | 7 +++++-- CONTRIBUTORS.md | 1 + requirements/dev.txt | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 652f916c..5b89698e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,10 +1,13 @@ -0.7.2 - 2024-03-21 +0.8.0 ------------------ +Added +~~~~~~~ +- Add DynamoDB session interface (`#214 `_). + Fixed ~~~~~ - Include prematurely removed ``cachelib`` dependency. Will be removed in 1.0.0 to be an optional dependency (`#223 `_). -- Note 0.7.1 was not released due to a publishing error. 0.7.0 - 2024-03-18 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index f8aafac8..d0c4c815 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1,5 +1,6 @@ ## Contributors +- [necat1](https://github.com/necat1) - [nebolax](https://github.com/nebolax) - [Taragolis](https://github.com/Taragolis) - [Lxstr](https://github.com/Lxstr) diff --git a/requirements/dev.txt b/requirements/dev.txt index dd433cac..14205475 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -16,4 +16,5 @@ pymemcache Flask-SQLAlchemy pymongo boto3 +mypy_boto3_dynamodb