From 27f331e7c07995c4de9482aa9962deff5289210c Mon Sep 17 00:00:00 2001 From: Shanika Kuruppu Date: Thu, 14 Oct 2021 09:59:53 +1100 Subject: [PATCH 1/7] test: get project from env var --- migration_test_cleanup.py | 11 ++++++++++- noxfile.py | 10 +++++++++- samples/conftest.py | 16 ++++++++++++++-- setup.cfg | 3 --- test/test_suite.py | 36 +++++++++++++++++++++++------------- 5 files changed, 56 insertions(+), 20 deletions(-) diff --git a/migration_test_cleanup.py b/migration_test_cleanup.py index 01a9a528..9c5f92a2 100644 --- a/migration_test_cleanup.py +++ b/migration_test_cleanup.py @@ -20,12 +20,21 @@ from google.cloud import spanner +project = os.getenv( + "GOOGLE_CLOUD_PROJECT", + os.getenv("PROJECT_ID", "emulator-test-project"), +) +db_url = ( + f"spanner:///projects/{project}/instances/" + "sqlalchemy-dialect-test/databases/compliance-test" +) + config = configparser.ConfigParser() if os.path.exists("test.cfg"): config.read("test.cfg") else: config.read("setup.cfg") -db_url = config.get("db", "default") +db_url = config.get("db", "default", fallback=db_url) project = re.findall(r"projects(.*?)instances", db_url) instance_id = re.findall(r"instances(.*?)databases", db_url) diff --git a/noxfile.py b/noxfile.py index 1e5eb9ed..2a83841d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -147,12 +147,20 @@ def migration_test(session): session.install("-e", ".") session.install("alembic") + project = os.getenv( + "GOOGLE_CLOUD_PROJECT", os.getenv("PROJECT_ID", "emulator-test-project"), + ) + db_url = ( + f"spanner:///projects/{project}/instances/" + "sqlalchemy-dialect-test/databases/compliance-test" + ) + config = configparser.ConfigParser() if os.path.exists("test.cfg"): config.read("test.cfg") else: config.read("setup.cfg") - db_url = config.get("db", "default") + db_url = config.get("db", "default", fallback=db_url) session.run("alembic", "init", "test_migration") diff --git a/samples/conftest.py b/samples/conftest.py index f3f0fcaa..5a4f622e 100644 --- a/samples/conftest.py +++ b/samples/conftest.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import configparser import datetime +import os import uuid import pytest @@ -30,11 +32,21 @@ @pytest.fixture def db_url(): - return ( - "spanner:///projects/appdev-soda-spanner-staging/instances/" + project = os.getenv( + "GOOGLE_CLOUD_PROJECT", os.getenv("PROJECT_ID", "emulator-test-project"), + ) + db_url = ( + f"spanner:///projects/{project}/instances/" "sqlalchemy-dialect-test/databases/compliance-test" ) + config = configparser.ConfigParser() + if os.path.exists("test.cfg"): + config.read("test.cfg") + else: + config.read("setup.cfg") + return config.get("db", "default", fallback=db_url) + @pytest.fixture def table_id(): diff --git a/setup.cfg b/setup.cfg index 7050a471..7b1c6800 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,3 @@ python_files=test/*test_*.py [sqla_testing] requirement_cls=google.cloud.sqlalchemy_spanner.requirements:Requirements profile_file=test/profiles.txt - -[db] -default=spanner:///projects/appdev-soda-spanner-staging/instances/sqlalchemy-dialect-test/databases/compliance-test \ No newline at end of file diff --git a/test/test_suite.py b/test/test_suite.py index 500114c3..37f1491f 100644 --- a/test/test_suite.py +++ b/test/test_suite.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import configparser from datetime import timezone import decimal import operator @@ -115,6 +116,24 @@ config.test_schema = "" +PROJECT = os.getenv( + "GOOGLE_CLOUD_PROJECT", os.getenv("PROJECT_ID", "emulator-test-project"), +) +DB_URL = ( + f"spanner:///projects/{PROJECT}/instances/" + "sqlalchemy-dialect-test/databases/compliance-test" +) + + +def get_db_url(): + config = configparser.ConfigParser() + if os.path.exists("test.cfg"): + config.read("test.cfg") + else: + config.read("setup.cfg") + return config.get("db", "default", fallback=DB_URL) + + class EscapingTest(_EscapingTest): @provide_metadata def test_percent_sign_round_trip(self): @@ -678,7 +697,7 @@ def define_temp_tables(cls, metadata): Column("foo", sqlalchemy.INT), sqlalchemy.Index("user_tmp_uq", "name", unique=True), sqlalchemy.Index("user_tmp_ix", "foo"), - **kw + **kw, ) if ( testing.requires.view_reflection.enabled @@ -1508,10 +1527,7 @@ class InterleavedTablesTest(fixtures.TestBase): """ def setUp(self): - self._engine = create_engine( - "spanner:///projects/appdev-soda-spanner-staging/instances/" - "sqlalchemy-dialect-test/databases/compliance-test" - ) + self._engine = create_engine(get_db_url()) self._metadata = MetaData(bind=self._engine) def test_interleave(self): @@ -1560,10 +1576,7 @@ class UserAgentTest(fixtures.TestBase): """Check that SQLAlchemy dialect uses correct user agent.""" def setUp(self): - self._engine = create_engine( - "spanner:///projects/appdev-soda-spanner-staging/instances/" - "sqlalchemy-dialect-test/databases/compliance-test" - ) + self._engine = create_engine(get_db_url()) self._metadata = MetaData(bind=self._engine) def test_user_agent(self): @@ -1583,10 +1596,7 @@ class ExecutionOptionsTest(fixtures.TestBase): """ def setUp(self): - self._engine = create_engine( - "spanner:///projects/appdev-soda-spanner-staging/instances/" - "sqlalchemy-dialect-test/databases/compliance-test" - ) + self._engine = create_engine(get_db_url()) self._metadata = MetaData(bind=self._engine) self._table = Table( From 8e87e1b4ef95249e27b638f92d00be684ef8ce61 Mon Sep 17 00:00:00 2001 From: Shanika Kuruppu Date: Thu, 14 Oct 2021 11:43:03 +1100 Subject: [PATCH 2/7] chore: remove RUN_SYSTEM_TEST env var --- .kokoro/presubmit/presubmit.cfg | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.kokoro/presubmit/presubmit.cfg b/.kokoro/presubmit/presubmit.cfg index b158096f..18a4c353 100644 --- a/.kokoro/presubmit/presubmit.cfg +++ b/.kokoro/presubmit/presubmit.cfg @@ -1,7 +1 @@ # Format: //devtools/kokoro/config/proto/build.proto - -# Disable system tests. -env_vars: { - key: "RUN_SYSTEM_TESTS" - value: "false" -} From a70169e82f18c2056d8a898fb63af4a48c5a8244 Mon Sep 17 00:00:00 2001 From: Shanika Kuruppu Date: Tue, 19 Oct 2021 12:59:10 +1100 Subject: [PATCH 3/7] test: longer timeout for creating instances/dbs --- create_test_database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_test_database.py b/create_test_database.py index a635b7ff..a30bcb50 100644 --- a/create_test_database.py +++ b/create_test_database.py @@ -82,11 +82,11 @@ def create_test_instance(): instance = CLIENT.instance(instance_id, instance_config, labels=labels) created_op = instance.create() - created_op.result(120) # block until completion + created_op.result(1800) # block until completion database = instance.database("compliance-test") created_op = database.create() - created_op.result(120) + created_op.result(1800) config = configparser.ConfigParser() url = "spanner:///projects/{project}/instances/{instance_id}/databases/compliance-test".format( From f4044bc09a6846329ea55f4aea5f230249795e82 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Tue, 19 Oct 2021 14:35:55 +0300 Subject: [PATCH 4/7] Update sqlalchemy_spanner.py --- google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py b/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py index a9abec59..d65fc0a9 100644 --- a/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py +++ b/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py @@ -394,7 +394,7 @@ class SpannerDialect(DefaultDialect): execute_sequence_format = list supports_alter = True - supports_sane_rowcount = True + supports_sane_rowcount = False supports_sane_multi_rowcount = False supports_default_values = False supports_sequences = True From 8ff6e434aa9c2e984a87ef6fb1f7389d0810def2 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Tue, 19 Oct 2021 14:36:19 +0300 Subject: [PATCH 5/7] Update requirements.py --- google/cloud/sqlalchemy_spanner/requirements.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/google/cloud/sqlalchemy_spanner/requirements.py b/google/cloud/sqlalchemy_spanner/requirements.py index 1b929847..97fbd546 100644 --- a/google/cloud/sqlalchemy_spanner/requirements.py +++ b/google/cloud/sqlalchemy_spanner/requirements.py @@ -17,6 +17,14 @@ class Requirements(SuiteRequirements): + @property + def sane_rowcount(self): + return exclusions.closed() + + @property + def sane_multi_rowcount(self): + return exclusions.closed() + @property def foreign_key_constraint_name_reflection(self): return exclusions.open() From bc6d9457ea749d1e9b044327a30a12a2281288c9 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Wed, 27 Oct 2021 10:33:31 +0300 Subject: [PATCH 6/7] Delete config.yml --- .circleci/config.yml | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 4ba15aef..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: 2.1 - -orbs: - python: circleci/python@1.4.0 - -jobs: - build-and-test: - executor: python/default - steps: - - checkout - - run: sudo apt install python3.8 - - run: echo $GCLOUD_SERVICE_KEY > "$GOOGLE_APPLICATION_CREDENTIALS" - - run: python3 -m pip install nox - - run: python3 -m nox - -workflows: - main: - jobs: - - build-and-test From a4db7f9c0348064e63504e591f7ea0ea8fc5b611 Mon Sep 17 00:00:00 2001 From: Shanika Kuruppu Date: Wed, 3 Nov 2021 10:27:05 +1100 Subject: [PATCH 7/7] fix: refactored according to review feedback --- migration_test_cleanup.py | 43 ++++++++++++++++----------------------- noxfile.py | 2 +- test/_helpers.py | 20 ++++++++++++++++++ test/test_suite.py | 20 +----------------- 4 files changed, 39 insertions(+), 46 deletions(-) diff --git a/migration_test_cleanup.py b/migration_test_cleanup.py index 9c5f92a2..62266359 100644 --- a/migration_test_cleanup.py +++ b/migration_test_cleanup.py @@ -14,33 +14,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import configparser -import os import re +import sys from google.cloud import spanner -project = os.getenv( - "GOOGLE_CLOUD_PROJECT", - os.getenv("PROJECT_ID", "emulator-test-project"), -) -db_url = ( - f"spanner:///projects/{project}/instances/" - "sqlalchemy-dialect-test/databases/compliance-test" -) - -config = configparser.ConfigParser() -if os.path.exists("test.cfg"): - config.read("test.cfg") -else: - config.read("setup.cfg") -db_url = config.get("db", "default", fallback=db_url) - -project = re.findall(r"projects(.*?)instances", db_url) -instance_id = re.findall(r"instances(.*?)databases", db_url) - -client = spanner.Client(project="".join(project).replace("/", "")) -instance = client.instance(instance_id="".join(instance_id).replace("/", "")) -database = instance.database("compliance-test") - -database.update_ddl(["DROP TABLE account", "DROP TABLE alembic_version"]).result(120) + +def main(argv): + db_url = argv[0] + + project = re.findall(r"projects(.*?)instances", db_url) + instance_id = re.findall(r"instances(.*?)databases", db_url) + + client = spanner.Client(project="".join(project).replace("/", "")) + instance = client.instance(instance_id="".join(instance_id).replace("/", "")) + database = instance.database("compliance-test") + + database.update_ddl(["DROP TABLE account", "DROP TABLE alembic_version"]).result(120) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/noxfile.py b/noxfile.py index 2a83841d..04b2875d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -189,7 +189,7 @@ def migration_test(session): # clearing the migration data os.remove("alembic.ini") shutil.rmtree("test_migration") - session.run("python", "migration_test_cleanup.py") + session.run("python", "migration_test_cleanup.py", db_url) if os.path.exists("test.cfg"): os.remove("test.cfg") diff --git a/test/_helpers.py b/test/_helpers.py index c9d9ba74..dd18a149 100644 --- a/test/_helpers.py +++ b/test/_helpers.py @@ -5,7 +5,9 @@ # https://developers.google.com/open-source/licenses/bsd +import configparser import mock +import os from sqlalchemy.testing import fixtures try: @@ -29,6 +31,24 @@ _TEST_OT_PROVIDER_INITIALIZED = False +PROJECT = os.getenv( + "GOOGLE_CLOUD_PROJECT", os.getenv("PROJECT_ID", "emulator-test-project"), +) +DB_URL = ( + f"spanner:///projects/{PROJECT}/instances/" + "sqlalchemy-dialect-test/databases/compliance-test" +) + + +def get_db_url(): + config = configparser.ConfigParser() + if os.path.exists("test.cfg"): + config.read("test.cfg") + else: + config.read("setup.cfg") + return config.get("db", "default", fallback=DB_URL) + + def get_test_ot_exporter(): global _TEST_OT_EXPORTER diff --git a/test/test_suite.py b/test/test_suite.py index 37f1491f..ea40bf09 100644 --- a/test/test_suite.py +++ b/test/test_suite.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import configparser from datetime import timezone import decimal import operator @@ -112,28 +111,11 @@ UnicodeTextTest as _UnicodeTextTest, _UnicodeFixture as _UnicodeFixtureTest, ) +from test._helpers import get_db_url config.test_schema = "" -PROJECT = os.getenv( - "GOOGLE_CLOUD_PROJECT", os.getenv("PROJECT_ID", "emulator-test-project"), -) -DB_URL = ( - f"spanner:///projects/{PROJECT}/instances/" - "sqlalchemy-dialect-test/databases/compliance-test" -) - - -def get_db_url(): - config = configparser.ConfigParser() - if os.path.exists("test.cfg"): - config.read("test.cfg") - else: - config.read("setup.cfg") - return config.get("db", "default", fallback=DB_URL) - - class EscapingTest(_EscapingTest): @provide_metadata def test_percent_sign_round_trip(self):