From 6e85490ae3f6b8348cdb9a967cf05a8eb1c81446 Mon Sep 17 00:00:00 2001 From: patrick o'leary Date: Wed, 20 Dec 2023 17:44:42 -0500 Subject: [PATCH 1/4] Fix for sqlalchemy 2 and text upgrades --- luigi/db_task_history.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/luigi/db_task_history.py b/luigi/db_task_history.py index e1eabcb7d1..40bce65b9b 100644 --- a/luigi/db_task_history.py +++ b/luigi/db_task_history.py @@ -49,6 +49,8 @@ import sqlalchemy.orm import sqlalchemy.orm.collections from sqlalchemy.engine import reflection +from sqlalchemy import text + Base = sqlalchemy.ext.declarative.declarative_base() logger = logging.getLogger('luigi-interface') @@ -251,25 +253,26 @@ def _upgrade_schema(engine): # Upgrade 1. Add task_id column and index to tasks if 'task_id' not in [x['name'] for x in inspector.get_columns('tasks')]: logger.warning('Upgrading DbTaskHistory schema: Adding tasks.task_id') - conn.execute('ALTER TABLE tasks ADD COLUMN task_id VARCHAR(200)') - conn.execute('CREATE INDEX ix_task_id ON tasks (task_id)') + conn.execute(text('ALTER TABLE tasks ADD COLUMN task_id VARCHAR(200)')) + conn.execute(text('CREATE INDEX ix_task_id ON tasks (task_id)')) # Upgrade 2. Alter value column to be TEXT, note that this is idempotent so no if-guard if 'mysql' in engine.dialect.name: - conn.execute('ALTER TABLE task_parameters MODIFY COLUMN value TEXT') + conn.execute(text('ALTER TABLE task_parameters MODIFY COLUMN value TEXT')) elif 'oracle' in engine.dialect.name: - conn.execute('ALTER TABLE task_parameters MODIFY value TEXT') + conn.execute(text('ALTER TABLE task_parameters MODIFY value TEXT')) elif 'mssql' in engine.dialect.name: - conn.execute('ALTER TABLE task_parameters ALTER COLUMN value TEXT') + conn.execute(text('ALTER TABLE task_parameters ALTER COLUMN value TEXT')) elif 'postgresql' in engine.dialect.name: if str([x for x in inspector.get_columns('task_parameters') if x['name'] == 'value'][0]['type']) != 'TEXT': - conn.execute('ALTER TABLE task_parameters ALTER COLUMN value TYPE TEXT') + conn.execute(text('ALTER TABLE task_parameters ALTER COLUMN value TYPE TEXT')) elif 'sqlite' in engine.dialect.name: # SQLite does not support changing column types. A database file will need # to be used to pickup this migration change. - for i in conn.execute('PRAGMA table_info(task_parameters);').fetchall(): - if i['name'] == 'value' and i['type'] != 'TEXT': + for i in conn.execute(text('PRAGMA table_info(task_parameters);')).fetchall(): + x = i._asdict() + if x['name'] == 'value' and x['type'] != 'TEXT': logger.warning( 'SQLite can not change column types. Please use a new database ' 'to pickup column type changes.' From 46b60cb48657e27ec7d97cb0cf3a493c2b65a046 Mon Sep 17 00:00:00 2001 From: patrick o'leary Date: Wed, 20 Dec 2023 19:03:24 -0500 Subject: [PATCH 2/4] updating tox to sqlalchemy 2.0 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4208e98eca..a6c3f21c0a 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ deps = boto3>=1.11.0 pyhive[presto]==0.6.1 s3transfer>=0.3,<4.0 - sqlalchemy<1.4 + sqlalchemy>=2.0,<2.1 elasticsearch>=1.0.0,<2.0.0 psutil<4.0 cdh,hdp: hdfs>=2.0.4,<3.0.0 From 46563d92bd92b22c3c4a0a19b56d40143103e454 Mon Sep 17 00:00:00 2001 From: patrick o'leary Date: Thu, 21 Dec 2023 15:09:32 -0500 Subject: [PATCH 3/4] support for both sqlalchemy 1 & 2 --- luigi/db_task_history.py | 9 ++++++++- tox.ini | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/luigi/db_task_history.py b/luigi/db_task_history.py index 40bce65b9b..5a0e7b9a13 100644 --- a/luigi/db_task_history.py +++ b/luigi/db_task_history.py @@ -49,12 +49,19 @@ import sqlalchemy.orm import sqlalchemy.orm.collections from sqlalchemy.engine import reflection -from sqlalchemy import text Base = sqlalchemy.ext.declarative.declarative_base() logger = logging.getLogger('luigi-interface') +if sqlalchemy.__version__.startswith('2'): + logger.warning('SQLAlchemy 2.x is not tested with luigi.db_task_history.DbTaskHistory') + from sqlalchemy import text + +else : + def text(sql): + return sql + class DbTaskHistory(task_history.TaskHistory): """ diff --git a/tox.ini b/tox.ini index a6c3f21c0a..33c4d32379 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ deps = boto3>=1.11.0 pyhive[presto]==0.6.1 s3transfer>=0.3,<4.0 - sqlalchemy>=2.0,<2.1 + sqlalchemy<2.1 elasticsearch>=1.0.0,<2.0.0 psutil<4.0 cdh,hdp: hdfs>=2.0.4,<3.0.0 From 84411fec61e9118e9b6a56203d73cbd375f8d6db Mon Sep 17 00:00:00 2001 From: patrick o'leary Date: Thu, 21 Dec 2023 15:25:00 -0500 Subject: [PATCH 4/4] fixing flake8 ws formatting --- luigi/db_task_history.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/luigi/db_task_history.py b/luigi/db_task_history.py index 5a0e7b9a13..cf71b92aed 100644 --- a/luigi/db_task_history.py +++ b/luigi/db_task_history.py @@ -58,10 +58,10 @@ logger.warning('SQLAlchemy 2.x is not tested with luigi.db_task_history.DbTaskHistory') from sqlalchemy import text -else : +else: def text(sql): return sql - + class DbTaskHistory(task_history.TaskHistory): """