add tmp location

Author: derrickaw

sdks/python/apache_beam/yaml/extended_tests/databases/bigquery_dynamic_destinations.yaml

@@ -20,6 +20,11 @@ fixtures:
     type: "apache_beam.yaml.integration_tests.temp_bigquery_table"
     config:
       project: "apache-beam-testing"
+  - name: TEMP_DIR_0
+    # Need distributed filesystem to be able to read and write from a container.
+    type: "apache_beam.yaml.integration_tests.gcs_temp_dir"
+    config:
+      bucket: "gs://temp-storage-for-end-to-end-tests/temp-it"
 
 pipelines:
   - pipeline:
@@ -33,61 +38,60 @@ pipelines:
               - {id: 2, name: "Bob", country: "UK"}
               - {id: 3, name: "Charlie", country: "CN"}
               - {id: 4, name: "David", country: "US"}
-# Blocked by https://github.com/apache/beam/issues/30950
-  #       - type: WriteToBigQuery
-  #         name: WriteWithDynamicDestinations
-  #         config:
-  #           # NOTE: This will create 3 tables: BQ_TABLE_US, BQ_TABLE_UK, BQ_TABLE_CN
-  #           # This is because the table name is dynamically generated using the country field.
-  #           # The {{country}} syntax is used to dynamically generate the table name.
-  #           # For this testing example we are using {{country}} to dynamically generate the table name.
-  #           # In production, we would use {country} to dynamically generate the table name.
-  #           table: "{BQ_TABLE}_{{country}}"
-  #           create_disposition: CREATE_IF_NEEDED
-  #           write_disposition: WRITE_APPEND
-  #   options:
-  #     project: "apache-beam-testing"
-  #     temp_location: "gs://temp-storage-for-end-to-end-tests/temp-it"
+        - type: WriteToBigQuery
+          name: WriteWithDynamicDestinations
+          config:
+            # NOTE: This will create 3 tables: BQ_TABLE_US, BQ_TABLE_UK, BQ_TABLE_CN
+            # This is because the table name is dynamically generated using the country field.
+            # The {{country}} syntax is used to dynamically generate the table name.
+            # For this testing example we are using {{country}} to dynamically generate the table name.
+            # In production, we would use {country} to dynamically generate the table name.
+            table: "{BQ_TABLE}_{{country}}"
+            create_disposition: CREATE_IF_NEEDED
+            write_disposition: WRITE_APPEND
+    options:
+      project: "apache-beam-testing"
+      temp_location: "{TEMP_DIR_0}"
 
-  # - pipeline:
-  #     type: chain
-  #     transforms:
-  #       - type: ReadFromBigQuery
-  #         config:
-  #           table: "{BQ_TABLE}_US"
-  #       - type: AssertEqual
-  #         config:
-  #           elements:
-  #             - {id: 1, name: "Alice", country: "US"}
-  #             - {id: 4, name: "David", country: "US"}
-  #   options:
-  #     project: "apache-beam-testing"
-  #     temp_location: "gs://temp-storage-for-end-to-end-tests/temp-it"
+  - pipeline:
+      type: chain
+      transforms:
+        - type: ReadFromBigQuery
+          config:
+            table: "{BQ_TABLE}_US"
+        - type: AssertEqual
+          config:
+            elements:
+              - {id: 1, name: "Alice", country: "US"}
+              - {id: 4, name: "David", country: "US"}
+    options:
+      project: "apache-beam-testing"
+      temp_location: "{TEMP_DIR_0}"
 
-  # - pipeline:
-  #     type: chain
-  #     transforms:
-  #       - type: ReadFromBigQuery
-  #         config:
-  #           table: "{BQ_TABLE}_UK"
-  #       - type: AssertEqual
-  #         config:
-  #           elements:
-  #             - {id: 2, name: "Bob", country: "UK"}
-  #   options:
-  #     project: "apache-beam-testing"
-  #     temp_location: "gs://temp-storage-for-end-to-end-tests/temp-it"
+  - pipeline:
+      type: chain
+      transforms:
+        - type: ReadFromBigQuery
+          config:
+            table: "{BQ_TABLE}_UK"
+        - type: AssertEqual
+          config:
+            elements:
+              - {id: 2, name: "Bob", country: "UK"}
+    options:
+      project: "apache-beam-testing"
+      temp_location: "{TEMP_DIR_0}"
 
-  # - pipeline:
-  #     type: chain
-  #     transforms:
-  #       - type: ReadFromBigQuery
-  #         config:
-  #           table: "{BQ_TABLE}_CN"
-  #       - type: AssertEqual
-  #         config:
-  #           elements:
-  #             - {id: 3, name: "Charlie", country: "CN"}
-  #   options:
-  #     project: "apache-beam-testing"
-  #     temp_location: "gs://temp-storage-for-end-to-end-tests/temp-it"
+  - pipeline:
+      type: chain
+      transforms:
+        - type: ReadFromBigQuery
+          config:
+            table: "{BQ_TABLE}_CN"
+        - type: AssertEqual
+          config:
+            elements:
+              - {id: 3, name: "Charlie", country: "CN"}
+    options:
+      project: "apache-beam-testing"
+      temp_location: "{TEMP_DIR_0}"