feat: support SQLAlchemy 1.4 (#191)

* WIP: update tests for SQLAlchemy 1.4

* feat: support SQLAlchemy 1.4

* fix migration bugs

* test: update yaml

* install 1.4

* fix the dialect name

* fix caching

* fix temp tables

* update test suite

* fix pytest

* del unused

* fix user agent

* fix test

* fix reflection test

* fix test

* fix tests

* lint fix

* fix test

* migration fix

* fix migration

* check

* fix test

* fix migrations

* fix migrations test

* change default py version

* minor fixes

* test workflow changes

* new tests skip

* kokoro session fix

Author: Ilya Gurov

.github/sync-repo-settings.yaml

@@ -9,7 +9,8 @@ branchProtectionRules:
   requiredStatusCheckContexts:
     - 'lint'
     - 'unit'
-    - 'compliance_tests'
+    - 'compliance_tests_13'
+    - 'compliance_tests_14'
     - 'migration_tests'
     - 'cla/google'
     - 'Kokoro'

.github/workflows/test_suite.yml

@@ -38,7 +38,7 @@ jobs:
           SPANNER_EMULATOR_HOST: localhost:9010
           GOOGLE_CLOUD_PROJECT: appdev-soda-spanner-staging
 
-  compliance_tests:
+  compliance_tests_13:
     runs-on: ubuntu-latest
 
     services:
@@ -57,7 +57,31 @@ jobs:
       - name: Install nox
         run: python -m pip install nox
       - name: Run Compliance Tests
-        run: nox -s compliance_test
+        run: nox -s compliance_test_13
+        env:
+          SPANNER_EMULATOR_HOST: localhost:9010
+          GOOGLE_CLOUD_PROJECT: appdev-soda-spanner-staging
+
+  compliance_tests_14:
+    runs-on: ubuntu-latest
+
+    services:
+      emulator-0:
+        image: gcr.io/cloud-spanner-emulator/emulator:latest
+        ports:
+          - 9010:9010
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install nox
+        run: python -m pip install nox
+      - name: Run Compliance Tests
+        run: nox -s compliance_test_14
         env:
           SPANNER_EMULATOR_HOST: localhost:9010
           GOOGLE_CLOUD_PROJECT: appdev-soda-spanner-staging

.kokoro/presubmit/compliance.cfg

@@ -3,5 +3,5 @@
 # Only run this nox session.
 env_vars: {
     key: "NOX_SESSION"
-    value: "compliance_test"
+    value: "compliance_test_13"
 }

create_test_config.py

@@ -21,7 +21,7 @@
 def set_test_config(project, instance):
     config = configparser.ConfigParser()
     url = (
-        f"spanner:///projects/{project}/instances/{instance}/"
+        f"spanner+spanner:///projects/{project}/instances/{instance}/"
         "databases/compliance-test"
     )
     config.add_section("db")
@@ -38,4 +38,4 @@ def main(argv):
 
 
 if __name__ == "__main__":
-   main(sys.argv[1:])
+    main(sys.argv[1:])

create_test_database.py

@@ -82,19 +82,18 @@ def create_test_instance():
 
     instance = CLIENT.instance(instance_id, instance_config, labels=labels)
 
-
     try:
-      created_op = instance.create()
-      created_op.result(1800)  # block until completion
+        created_op = instance.create()
+        created_op.result(1800)  # block until completion
     except AlreadyExists:
-      pass  # instance was already created
+        pass  # instance was already created
 
     try:
-      database = instance.database("compliance-test")
-      created_op = database.create()
-      created_op.result(1800)
+        database = instance.database("compliance-test")
+        created_op = database.create()
+        created_op.result(1800)
     except AlreadyExists:
-      pass  # instance was already created
+        pass  # instance was already created
 
     set_test_config(PROJECT, instance_id)
 

google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py

@@ -495,7 +495,7 @@ class SpannerDialect(DefaultDialect):
     Represents an API layer to control Cloud Spanner database with SQLAlchemy API.
     """
 
-    name = "spanner"
+    name = "spanner+spanner"
     driver = "spanner"
     positional = False
     paramstyle = "format"
@@ -512,6 +512,7 @@ class SpannerDialect(DefaultDialect):
     supports_native_enum = True
     supports_native_boolean = True
     supports_native_decimal = True
+    supports_statement_cache = True
 
     ddl_compiler = SpannerDDLCompiler
     preparer = SpannerIdentifierPreparer

noxfile.py

@@ -64,8 +64,7 @@ class = StreamHandler
     op.alter_column(
         'account',
         'name',
-        existing_type=sa.String(50),
-        nullable=True,
+        existing_type=sa.String(70),
     )"""
 
 
@@ -114,7 +113,7 @@ def lint_setup_py(session):
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
-def compliance_test(session):
+def compliance_test_13(session):
     """Run SQLAlchemy dialect compliance test suite."""
 
     # Check the value of `RUN_COMPLIANCE_TESTS` env var. It defaults to true.
@@ -132,7 +131,6 @@ def compliance_test(session):
         "pytest", "pytest-cov", "pytest-asyncio",
     )
 
-    session.install("pytest")
     session.install("mock")
     session.install("-e", ".[tracing]")
     session.run("python", "create_test_database.py")
@@ -145,7 +143,46 @@ def compliance_test(session):
         "--cov-config=.coveragerc",
         "--cov-report=",
         "--cov-fail-under=0",
-        "test",
+        "--asyncio-mode=auto",
+        "test/test_suite_13.py",
+    )
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def compliance_test_14(session):
+    """Run SQLAlchemy dialect compliance test suite."""
+
+    # Check the value of `RUN_COMPLIANCE_TESTS` env var. It defaults to true.
+    if os.environ.get("RUN_COMPLIANCE_TESTS", "true") == "false":
+        session.skip("RUN_COMPLIANCE_TESTS is set to false, skipping")
+    # Sanity check: Only run tests if the environment variable is set.
+    if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", "") and not os.environ.get(
+        "SPANNER_EMULATOR_HOST", ""
+    ):
+        session.skip(
+            "Credentials or emulator host must be set via environment variable"
+        )
+
+    session.install(
+        "pytest", "pytest-cov", "pytest-asyncio",
+    )
+
+    session.install("mock")
+    session.install("-e", ".[tracing]")
+    session.run("python", "create_test_database.py")
+
+    session.install("sqlalchemy>=1.4")
+
+    session.run(
+        "py.test",
+        "--cov=google.cloud.sqlalchemy_spanner",
+        "--cov=tests",
+        "--cov-append",
+        "--cov-config=.coveragerc",
+        "--cov-report=",
+        "--cov-fail-under=0",
+        "--asyncio-mode=auto",
+        "test/test_suite_14.py",
     )
 
 
@@ -180,7 +217,7 @@ def migration_test(session):
         "GOOGLE_CLOUD_PROJECT", os.getenv("PROJECT_ID", "emulator-test-project"),
     )
     db_url = (
-        f"spanner:///projects/{project}/instances/"
+        f"spanner+spanner:///projects/{project}/instances/"
         "sqlalchemy-dialect-test/databases/compliance-test"
     )
 

setup.py

@@ -60,7 +60,7 @@
     description=description,
     entry_points={
         "sqlalchemy.dialects": [
-            "spanner = google.cloud.sqlalchemy_spanner:SpannerDialect"
+            "spanner.spanner = google.cloud.sqlalchemy_spanner:SpannerDialect"
         ]
     },
     install_requires=dependencies,