Module composer

@pulumi/gcp > composer

class Environment

extends CustomResource

An environment for running orchestration tasks.

Environments run Apache Airflow software on Google infrastructure.

To get more information about Environments, see:

Warning: We STRONGLY recommend you read the GCP guides as the Environment resource requires a long deployment process and involves several layers of GCP infrastructure, including a Kubernetes Engine cluster, Cloud Storage, and Compute networking resources. Due to limitations of the API, Terraform will not be able to automatically find or manage many of these underlying resources. In particular:

  • It can take up to one hour to create or update an environment resource. In addition, GCP may only detect some errors in configuration when they are used (e.g. ~40-50 minutes into the creation process), and is prone to limited error reporting. If you encounter confusing or uninformative errors, please verify your configuration is valid against GCP Cloud Composer before filing bugs against the Terraform provider.
  • Environments create Google Cloud Storage buckets that do not get cleaned up automatically on environment deletion. More about Composer’s use of Cloud Storage.

Example Usage

Basic Usage

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const test = new gcp.composer.Environment("test", {
    region: "us-central1",
});

With GKE and Compute Resource Dependencies

NOTE To use service accounts, you need to give role/composer.worker to the service account on any resources that may be created for the environment (i.e. at a project level). This will probably require an explicit dependency on the IAM policy binding (see google_project_iam_member below).

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const testNetwork = new gcp.compute.Network("test", {
    autoCreateSubnetworks: false,
});
const testAccount = new gcp.serviceAccount.Account("test", {
    accountId: "composer-env-account",
    displayName: "Test Service Account for Composer Environment",
});
const testSubnetwork = new gcp.compute.Subnetwork("test", {
    ipCidrRange: "10.2.0.0/16",
    network: testNetwork.selfLink,
    region: "us-central1",
});
const composer_worker = new gcp.projects.IAMMember("composer-worker", {
    member: testAccount.email.apply(email => `serviceAccount:${email}`),
    role: "roles/composer.worker",
});
const testEnvironment = new gcp.composer.Environment("test", {
    config: {
        nodeConfig: {
            machineType: "n1-standard-1",
            network: testNetwork.selfLink,
            serviceAccount: testAccount.name,
            subnetwork: testSubnetwork.selfLink,
            zone: "us-central1-a",
        },
        nodeCount: 4,
    },
    region: "us-central1",
}, {dependsOn: [composer_worker]});

With Software (Airflow) Config

import * as pulumi from "@pulumi/pulumi";
import * as gcp from "@pulumi/gcp";

const test = new gcp.composer.Environment("test", {
    config: {
        softwareConfig: {
            airflowConfigOverrides: {
                "core-load_example": "True",
            },
            envVariables: {
                FOO: "bar",
            },
            pypiPackages: {
                numpy: "",
                scipy: "==1.1.0",
            },
        },
    },
    region: "us-central1",
});

constructor

new Environment(name: string, args?: EnvironmentArgs, opts?: pulumi.CustomResourceOptions)

Create a Environment resource with the given unique name, arguments, and options.

  • name The unique name of the resource.
  • args The arguments to use to populate this resource's properties.
  • opts A bag of options that control this resource's behavior.

method get

public static get(name: string, id: pulumi.Input<pulumi.ID>, state?: EnvironmentState, opts?: pulumi.CustomResourceOptions): Environment

Get an existing Environment resource’s state with the given name, ID, and optional extra properties used to qualify the lookup.

method getProvider

getProvider(moduleMember: string): ProviderResource | undefined

method isInstance

static isInstance(obj: any): boolean

Returns true if the given object is an instance of CustomResource. This is designed to work even when multiple copies of the Pulumi SDK have been loaded into the same process.

property config

public config: pulumi.Output<{
    airflowUri: string;
    dagGcsPrefix: string;
    gkeCluster: string;
    nodeConfig: {
        diskSizeGb: number;
        machineType: string;
        network: string;
        oauthScopes: string[];
        serviceAccount: string;
        subnetwork: string;
        tags: string[];
        zone: string;
    };
    nodeCount: number;
    softwareConfig: {
        airflowConfigOverrides: {[key: string]: string};
        envVariables: {[key: string]: string};
        imageVersion: string;
        pypiPackages: {[key: string]: string};
    };
}>;

property id

id: Output<ID>;

id is the provider-assigned unique ID for this managed resource. It is set during deployments and may be missing (undefined) during planning phases.

property labels

public labels: pulumi.Output<{[key: string]: string} | undefined>;

property name

public name: pulumi.Output<string>;

property project

public project: pulumi.Output<string>;

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

property region

public region: pulumi.Output<string | undefined>;

property urn

urn: Output<URN>;

urn is the stable logical URN used to distinctly address a resource, both before and after deployments.

interface EnvironmentArgs

The set of arguments for constructing a Environment resource.

property config

config?: pulumi.Input<{
    airflowUri: pulumi.Input<string>;
    dagGcsPrefix: pulumi.Input<string>;
    gkeCluster: pulumi.Input<string>;
    nodeConfig: pulumi.Input<{
        diskSizeGb: pulumi.Input<number>;
        machineType: pulumi.Input<string>;
        network: pulumi.Input<string>;
        oauthScopes: pulumi.Input<pulumi.Input<string>[]>;
        serviceAccount: pulumi.Input<string>;
        subnetwork: pulumi.Input<string>;
        tags: pulumi.Input<pulumi.Input<string>[]>;
        zone: pulumi.Input<string>;
    }>;
    nodeCount: pulumi.Input<number>;
    softwareConfig: pulumi.Input<{
        airflowConfigOverrides: pulumi.Input<{[key: string]: pulumi.Input<string>}>;
        envVariables: pulumi.Input<{[key: string]: pulumi.Input<string>}>;
        imageVersion: pulumi.Input<string>;
        pypiPackages: pulumi.Input<{[key: string]: pulumi.Input<string>}>;
    }>;
}>;

property labels

labels?: pulumi.Input<{[key: string]: pulumi.Input<string>}>;

property name

name?: pulumi.Input<string>;

property project

project?: pulumi.Input<string>;

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

property region

region?: pulumi.Input<string>;

interface EnvironmentState

Input properties used for looking up and filtering Environment resources.

property config

config?: pulumi.Input<{
    airflowUri: pulumi.Input<string>;
    dagGcsPrefix: pulumi.Input<string>;
    gkeCluster: pulumi.Input<string>;
    nodeConfig: pulumi.Input<{
        diskSizeGb: pulumi.Input<number>;
        machineType: pulumi.Input<string>;
        network: pulumi.Input<string>;
        oauthScopes: pulumi.Input<pulumi.Input<string>[]>;
        serviceAccount: pulumi.Input<string>;
        subnetwork: pulumi.Input<string>;
        tags: pulumi.Input<pulumi.Input<string>[]>;
        zone: pulumi.Input<string>;
    }>;
    nodeCount: pulumi.Input<number>;
    softwareConfig: pulumi.Input<{
        airflowConfigOverrides: pulumi.Input<{[key: string]: pulumi.Input<string>}>;
        envVariables: pulumi.Input<{[key: string]: pulumi.Input<string>}>;
        imageVersion: pulumi.Input<string>;
        pypiPackages: pulumi.Input<{[key: string]: pulumi.Input<string>}>;
    }>;
}>;

property labels

labels?: pulumi.Input<{[key: string]: pulumi.Input<string>}>;

property name

name?: pulumi.Input<string>;

property project

project?: pulumi.Input<string>;

The ID of the project in which the resource belongs. If it is not provided, the provider project is used.

property region

region?: pulumi.Input<string>;