From df0c488467d7f68cae8b05a14d72a0ebf222440a Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Fri, 12 Mar 2021 20:22:42 +0000 Subject: [PATCH 1/4] fix(composer): use separate AIRFLOW_HOME per session fixes #5482 --- composer/conftest.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/composer/conftest.py b/composer/conftest.py index 08ddda63c48..d278dcc561c 100644 --- a/composer/conftest.py +++ b/composer/conftest.py @@ -13,6 +13,9 @@ # limitations under the License. +import os +import tempfile + import pytest @@ -21,5 +24,16 @@ @pytest.fixture(scope="session") def airflow_database(): import airflow.utils.db + + # We use separate directory for local db path per session. + tmp_dir = tempfile.TemporaryDirectory() + os.environ['AIRFLOW_HOME'] = tmp_dir.name + print(f"Setting AIRFLOW_HOME to {tmp_dir.name}") + # reset both resets and initializes a new database airflow.utils.db.resetdb(rbac=None) # this command will change in Airflow 2.0 + + yield + + # cleaning up + tmp_dir.cleanup() From 4e7dc8bc12f64e826510a02ab69ae0c33a1be0fe Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Fri, 12 Mar 2021 20:32:27 +0000 Subject: [PATCH 2/4] add a commit for triggering builds in blog and workflow subdirs. --- .../unit-test-dags-cloud-build/dags/example_dag_test.py | 1 + composer/workflows/quickstart_test.py | 1 + 2 files changed, 2 insertions(+) diff --git a/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/dags/example_dag_test.py b/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/dags/example_dag_test.py index 1f84015264f..186e03897ce 100644 --- a/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/dags/example_dag_test.py +++ b/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/dags/example_dag_test.py @@ -21,6 +21,7 @@ @pytest.fixture(autouse=True, scope="function") +# The fixture `airflow_database` lives in composer/conftest.py. def set_variables(airflow_database): models.Variable.set('gcp_project', PROJECT_ID) yield diff --git a/composer/workflows/quickstart_test.py b/composer/workflows/quickstart_test.py index b0b284a8b14..bb513f6c126 100644 --- a/composer/workflows/quickstart_test.py +++ b/composer/workflows/quickstart_test.py @@ -19,6 +19,7 @@ @pytest.fixture(autouse=True, scope="function") +# The fixture `airflow_database` lives in composer/conftest.py. def set_variables(airflow_database): models.Variable.set('gcs_bucket', 'example_bucket') From 851e4ccd2f3a11ac12802dec8400fa4709ff353c Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Fri, 12 Mar 2021 21:58:48 +0000 Subject: [PATCH 3/4] set AIRFLOW_HOME in noxfile_config.py --- .../noxfile_config.py | 15 ++++++++++++--- composer/conftest.py | 17 +++++++++-------- composer/workflows/noxfile_config.py | 10 +++++++++- 3 files changed, 30 insertions(+), 12 deletions(-) rename composer/blog/{ => gcp-tech-blog/unit-test-dags-cloud-build}/noxfile_config.py (84%) diff --git a/composer/blog/noxfile_config.py b/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/noxfile_config.py similarity index 84% rename from composer/blog/noxfile_config.py rename to composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/noxfile_config.py index 4400d71d381..9d405561ee7 100644 --- a/composer/blog/noxfile_config.py +++ b/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/noxfile_config.py @@ -1,4 +1,4 @@ -# Copyright 2021 Google LLC +# Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,13 +20,22 @@ # The source of truth: # https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py +import tempfile + + +# Airflow creates a config file at the installation, so we want to set +# `AIRFLOW_HOME` envvar before running pytest. + +_tmpdir = tempfile.TemporaryDirectory() + + TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. # Skipping for Python 3.9 due to numpy compilation failure. "ignored_versions": ["2.7", "3.9"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them - "enforce_type_hints": False, + "enforce_type_hints": True, # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string @@ -35,5 +44,5 @@ # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - "envs": {}, + "envs": {"AIRFLOW_HOME": _tmpdir.name}, } diff --git a/composer/conftest.py b/composer/conftest.py index d278dcc561c..908edb39419 100644 --- a/composer/conftest.py +++ b/composer/conftest.py @@ -25,15 +25,16 @@ def airflow_database(): import airflow.utils.db - # We use separate directory for local db path per session. - tmp_dir = tempfile.TemporaryDirectory() - os.environ['AIRFLOW_HOME'] = tmp_dir.name - print(f"Setting AIRFLOW_HOME to {tmp_dir.name}") + # We use separate directory for local db path per session + # by setting AIRFLOW_HOME env var, which is done in noxfile_config.py. + + assert('AIRFLOW_HOME' in os.environ) + + airflow_home = os.environ["AIRFLOW_HOME"] + airflow_db = f"{airflow_home}/airflow.db" # reset both resets and initializes a new database airflow.utils.db.resetdb(rbac=None) # this command will change in Airflow 2.0 - yield - - # cleaning up - tmp_dir.cleanup() + # Making sure we are using a data file there. + assert(os.path.isfile(airflow_db)) diff --git a/composer/workflows/noxfile_config.py b/composer/workflows/noxfile_config.py index 4400d71d381..b9fd08da2e4 100644 --- a/composer/workflows/noxfile_config.py +++ b/composer/workflows/noxfile_config.py @@ -20,6 +20,14 @@ # The source of truth: # https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py +import tempfile + + +# Airflow creates a config file at the installation, so we want to set +# `AIRFLOW_HOME` envvar before running pytest. + +_tmpdir = tempfile.TemporaryDirectory() + TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. # Skipping for Python 3.9 due to numpy compilation failure. @@ -35,5 +43,5 @@ # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - "envs": {}, + "envs": {"AIRFLOW_HOME": _tmpdir.name}, } From 85055dce2749b9ac46b900ddc39d99be80b2eae3 Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Fri, 12 Mar 2021 22:02:45 +0000 Subject: [PATCH 4/4] fix errors in noxfile_config.py --- .../unit-test-dags-cloud-build/noxfile_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/noxfile_config.py b/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/noxfile_config.py index 9d405561ee7..31a17ad81f9 100644 --- a/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/noxfile_config.py +++ b/composer/blog/gcp-tech-blog/unit-test-dags-cloud-build/noxfile_config.py @@ -1,4 +1,4 @@ -# Copyright 2020 Google LLC +# Copyright 2021 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ "ignored_versions": ["2.7", "3.9"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them - "enforce_type_hints": True, + "enforce_type_hints": False, # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string