parsing plugin, allow separate projects for vpc, monitoring, logging

modules/gcve-monitoring/main.tf

@@ -20,7 +20,7 @@ locals {
   sa_gcve_monitoring_roles = toset([
     "roles/secretmanager.secretAccessor",
     "roles/monitoring.admin",
-    "roles/logging.logWriter",
+    "roles/logging.logWriter"
   ])
 }
 
@@ -30,6 +30,12 @@ resource "google_project_service" "enable_destination_api" {
   disable_on_destroy = false
 }
 
+resource "google_project_service" "enable_cloudresourcemanager_api" {
+  project            = var.project
+  service            = "cloudresourcemanager.googleapis.com"
+  disable_on_destroy = false
+}
+
 resource "google_service_account" "sa_gcve_monitoring" {
   project    = var.project
   account_id = var.sa_gcve_monitoring
@@ -42,6 +48,14 @@ resource "google_project_iam_member" "gcve_monitoring_permissions" {
   member   = "serviceAccount:${google_service_account.sa_gcve_monitoring.email}"
 }
 
+# It is not an issue if var.project==var.logging_project, this resource is non-authoritative
+resource "google_project_iam_member" "gcve_monitoring_permissions_logging_project" {
+  for_each = local.sa_gcve_monitoring_roles
+  role     = each.key
+  project  = var.logging_project
+  member   = "serviceAccount:${google_service_account.sa_gcve_monitoring.email}"
+}
+
 #tfsec:ignore:vm-disk-encryption-customer-key
 
 # provide single Monitoring instance with healing capabilities.
@@ -102,12 +116,12 @@ resource "google_compute_health_check" "tcp_healthcheck" {
 data "google_compute_subnetwork" "gcve-subnetwork" {
   name    = var.subnetwork
   region  = var.gcve_region
-  project = var.project
+  project = var.vpc_project
 }
 
 #tfsec:ignore:google-compute-no-public-ingress
 resource "google_compute_firewall" "healthcheck" {
-  project = var.project
+  project = var.vpc_project
   name    = "gcve-mon-hc-rule"
   network = data.google_compute_subnetwork.gcve-subnetwork.network
 
@@ -146,11 +160,13 @@ resource "google_compute_instance_template" "vm_mon_tpl" {
     {
       endpoint_agent                 = "${local.base_gcve_agent_endpoint}/artifacts/bpagent-headless-vmware.tar.gz"
       endpoint_install               = "${local.base_gcve_agent_endpoint}/installer/install.sh"
+      parsing_plugins_git_url        = var.parsing_plugins_git_url
       gcloud_secret_vsphere_server   = "${local.base_gcloud_secret_manager}${var.secret_vsphere_server}"
       gcloud_secret_vsphere_user     = "${local.base_gcloud_secret_manager}${var.secret_vsphere_user}"
       gcloud_secret_vsphere_password = "${local.base_gcloud_secret_manager}${var.secret_vsphere_password}"
       gcve_region                    = var.gcve_region
       project_id                     = var.project
+      logging_project                = var.logging_project
   })
 
   disk {
@@ -167,7 +183,7 @@ resource "google_compute_instance_template" "vm_mon_tpl" {
 
   network_interface {
     subnetwork         = var.subnetwork
-    subnetwork_project = var.project
+    subnetwork_project = var.vpc_project
   }
 
   lifecycle {

modules/gcve-monitoring/scripts/installer.sh

@@ -17,7 +17,7 @@
 # https://github.com/GoogleCloudPlatform/monitoring-dashboard-samples/tree/master/terraform/agents/bindplane
 
 # Install prerequisites
-sudo apt-get install -y rsync
+sudo apt-get install -y rsync git
 
 # Setting up Cloud Monitoring with a standalone agent
 # https://cloud.google.com/vmware-engine/docs/environment/howto-cloud-monitoring-standalone
@@ -26,6 +26,12 @@ curl -s ${endpoint_install} -o /tmp/install.sh
 sudo chmod +x /tmp/install.sh
 sudo /tmp/install.sh /tmp/agent.tar.gz
 
+echo "installing gcve plugins for agent"
+git clone ${parsing_plugins_git_url} /tmp/filters
+sudo cp /tmp/filters/*.yaml /opt/bpagent/log_agent/plugins/
+sudo cp /tmp/filters/config_sample/log_agent.example.yaml /opt/bpagent/config/log_agent.yaml
+
+
 # Configure the agent to access your private cloud for metrics
 sudo cp /opt/bpagent/config/metrics/examples/vmware_vcenter.yaml /opt/bpagent/config/metrics/sources
 gcloud config set project ${project_id}
@@ -35,8 +41,7 @@ sudo sed -i "s/password:.*$/password: $(${gcloud_secret_vsphere_password})/g" /o
 sudo sed -i "s/# region:.*$/region: ${gcve_region}/g" /opt/bpagent/config/metrics/sources/vmware_vcenter.yaml
 
 #Configure the agent to access the service account for reporting
-sudo cp /opt/bpagent/config/log_agent.example.yaml /opt/bpagent/config/log_agent.yaml
-sudo sed -i "s/project_id:.*$/project_id: ${project_id}/g" /opt/bpagent/config/log_agent.yaml
+sudo sed -i "s/project_id:.*$/project_id: ${logging_project}/g" /opt/bpagent/config/log_agent.yaml
 sudo sed -i "s/credentials_file:.*$/#credentials_file: /g" /opt/bpagent/config/log_agent.yaml
 
 sudo systemctl stop bpagent

modules/gcve-monitoring/variables.tf

@@ -19,6 +19,16 @@ variable "project" {
   type        = string
 }
 
+variable "vpc_project" {
+  description = "The GCP project where the pre-existing VPC is, with PSA to GCVE already configured. This is usually a hub project with a shared vpc."
+  type        = string
+}
+
+variable "logging_project" {
+  description = "The GCP project where the logs will be stored. It could be the same or different as the monitoring project."
+  type        = string
+}
+
 variable "vm_mon_name" {
   description = "GCE VM name where GCVE monitoring agent will run"
   type        = string
@@ -103,3 +113,8 @@ variable "create_dashboards" {
   default     = true
 }
 
+variable "parsing_plugins_git_url" {
+  description = "vmware parsing plugins git url. i.e. https://github.com/alessandrolorusso/stanza-filters-gcve.git"
+  type        = string
+}
+

stages/01-privatecloud/01b-monitoring/monitoring.tf

@@ -18,6 +18,8 @@ module "gcve-monitoring" {
   source                  = "../../../modules/gcve-monitoring"
   gcve_region             = var.gcve_region
   project                 = var.project
+  vpc_project             = var.vpc_project
+  logging_project         = var.logging_project
   secret_vsphere_server   = var.secret_vsphere_server
   secret_vsphere_user     = var.secret_vsphere_user
   secret_vsphere_password = var.secret_vsphere_password
@@ -27,4 +29,5 @@ module "gcve-monitoring" {
   sa_gcve_monitoring      = var.sa_gcve_monitoring
   subnetwork              = var.subnetwork
   create_dashboards       = var.create_dashboards
+  parsing_plugins_git_url = var.parsing_plugins_git_url
 }

stages/01-privatecloud/01b-monitoring/terraform.tfvars.example

@@ -14,17 +14,19 @@
  * limitations under the License.
  */
 
-gcve_region             = "us-central1"
-project                 = "my-example-project"
-secret_vsphere_server   = "gcve-mon-vsphere-server"
-secret_vsphere_user     = "gcve-mon-vsphere-user"
-secret_vsphere_password = "gcve-mon-vsphere-password"
-vm_mon_name             = "gcve-monitoring-vm"
-vm_mon_type             = "e2-small"
-vm_mon_zone             = "us-central1-a"
-sa_gcve_monitoring      = "sa-gcve-monitoring"
-vpc                     = "my-network"
-subnetwork              = "my-subnetwork"
-vcenter_ip_address      = "10.0.0.6"
-nsx_manager_ip_address  = "10.0.0.11"
-create_dashboards       = true
+gcve_region                  = "us-central1"
+project                      = "my-example-project"
+vpc_project                  = "my-vpc-project"
+logging_project              = "my-logging-project"
+secret_vsphere_server        = "gcve-mon-vsphere-server"
+secret_vsphere_user          = "gcve-mon-vsphere-user"
+secret_vsphere_password      = "gcve-mon-vsphere-password"
+vm_mon_name                  = "gcve-monitoring-vm"
+vm_mon_type                  = "e2-small"
+vm_mon_zone                  = "us-central1-a"
+sa_gcve_monitoring           = "sa-gcve-monitoring"
+vpc                          = "my-network"
+subnetwork                   = "my-subnetwork"
+vcenter_ip_address           = "10.0.0.6"
+nsx_cidr                     = "10.0.0.11"
+create_dashboards            = true

stages/01-privatecloud/01b-monitoring/variables.tf

@@ -19,6 +19,16 @@ variable "project" {
   type        = string
 }
 
+variable "vpc_project" {
+  description = "The GCP project where the pre-existing VPC is, with PSA to GCVE already configured. This is usually a hub project with a shared vpc."
+  type        = string
+}
+
+variable "logging_project" {
+  description = "The GCP project where the logs will be stored. It could be the same or different as the monitoring project."
+  type        = string
+}
+
 variable "vm_mon_name" {
   description = "GCE VM name where GCVE monitoring agent will run"
   type        = string
@@ -36,7 +46,7 @@ variable "vm_mon_zone" {
 }
 
 variable "vpc" {
-  description = "VPC where the VM will be deployed to"
+  description = "VPC where the VM will be deployed to. This is usually a shared vpc in a hub project."
   type        = string
 }
 
@@ -75,8 +85,8 @@ variable "vcenter_ip_address" {
   type        = string
 }
 
-variable "nsx_manager_ip_address" {
-  description = "IP address of NSX Manager - used to for ingress firewall rule of syslog connector"
+variable "nsx_cidr" {
+  description = "CIDR range that includes all NSX appliances and edge nodes. Also all esxi nodes if dfw logs are expected. Setting this to the NSX manager IP will *not* work, most syslog messages do not have that source"
   type        = string
 }
 
@@ -85,3 +95,10 @@ variable "create_dashboards" {
   type        = bool
   default     = true
 }
+
+
+variable "parsing_plugins_git_url" {
+  description = "vmware parsing plugins git url. i.e. https://github.com/alessandrolorusso/stanza-filters-gcve.git"
+  type        = string
+  default     = "https://github.com/alessandrolorusso/stanza-filters-gcve.git"
+}

stages/01-privatecloud/01b-monitoring/vpc_firewall.tf

@@ -23,15 +23,16 @@ data "google_service_account" "monitoring_sa" {
 }
 
 resource "google_compute_firewall" "gcvesyslog" {
-  project = var.project
+  project = var.vpc_project
   name    = "gcve-syslog-rule"
   network = var.vpc
 
   allow {
     protocol = "tcp"
-    ports    = ["5142"]
+    # 5142 for vCenter, 5140 for NSX, 514 for NSX DFW
+    ports = ["5142", "5140", "514"]
   }
 
-  source_ranges           = [var.vcenter_ip_address, var.nsx_manager_ip_address]
+  source_ranges           = [var.vcenter_ip_address, var.nsx_cidr]
   target_service_accounts = [data.google_service_account.monitoring_sa.email]
 }