Merge branch 'main' into rm-forecast-generic

openclimatefix · Jan 3, 2025 · 2fcf940 · 2fcf940
2 parents f7d14ca + 4d6eadd
commit 2fcf940
Show file tree

Hide file tree

Showing 12 changed files with 83 additions and 20 deletions.
diff --git a/terraform/india/production/main.tf b/terraform/india/production/main.tf
@@ -414,7 +414,7 @@ module "forecast-ad" {
 
 # 5.0
 module "airflow" {
-  source                    = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/airflow?ref=f0ecf51"
+  source                    = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/airflow?ref=7e27e14"
   aws-environment           = local.environment
   aws-region                = local.region
   aws-domain                = local.domain

diff --git a/terraform/modules/networking/ec2_bastion/user_data.sh b/terraform/modules/networking/ec2_bastion/user_data.sh
@@ -18,4 +18,4 @@ echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC8RIkirOd6pkHU6U0+8csUIoBGNuxSj5In63
 echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFrgNxs10xZEpzKSOAcG1cI6pQBhTlCcj41dB2jXLa3w [email protected]" >> /home/ec2-user/.ssh/authorized_keys
 echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOmGsEaesnmsM/np9W7G8vHuxIZBjum/NO3Zk0MT+soO [email protected]" >> /home/ec2-user/.ssh/authorized_keys
 echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC0xop1hz1UsyqUUIf9416nGc2GCqm/exfnBGHDd+x8IOXJoGHc5fCM+pKpBaWjLFvOrXB/hx1qZTYFPCdygyzH7KPRIj22VBqgA26DIxd8/0aw4ookNCwOtWFJtm8AVibEffiMAl67zjMsUUZ0pGOM7w9/Zp2ZDr3TjBXKgDOkwLSu0y+66AagYl1dI7TFEAIKrAJQ6ZupYaTBAYxbekB97nYtrc1pUCR+707tm4U+eVjjni0MaQxiD83g35Ij1sOH+avijXTQQGSPJAMxnzEyUArdscg5Pc+qBQ/n5g9o9Sc2k8hwrxPDl4nzfHh9dUtqH7lMTTnjVc6SPdA6wPFY/rXJKk+9xPvphNo4wKZVdMCeVhHwx/9XN7JxKHSI4AQDCRUJ6WhoJFMaMdOziqnxc92sqHiUr1dwgFmKHsUmKm2pwU1JVmQCC+z0WjSbP5EVap8Vxu33t1XbpMowlLWe+Q1wxPR8n77bOqNMNY00bTKu0ki5eDgzHyvXfIQmFJk= preci@VINAYA" >> /home/ec2-user/.ssh/authorized_keys
-echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILYW49tR+g0gTMYhy4RHoY1SC94TSolqqjMnscaqDn3$ [email protected]" >> /home/ec2-user/.ssh/authorized_keys
+echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBPanFrUOmPJOK4ZMh5UQusuDRoCoDwDu1TjywNnOqCD" >> /home/ec2-user/.ssh/authorized_keys
diff --git a/terraform/modules/services/airflow/dags/india/forecast-site-dag.py b/terraform/modules/services/airflow/dags/india/forecast-site-dag.py
@@ -51,7 +51,7 @@
 
     latest_only >> [forecast]
 
-with DAG(f'{region}-ad-forecast', schedule_interval=f"0,30 * * * *", default_args=default_args, concurrency=10, max_active_tasks=10) as dag:
+with DAG(f'{region}-ad-forecast', schedule_interval=f"0,15,30,45 * * * *", default_args=default_args, concurrency=10, max_active_tasks=10) as dag:
     dag.doc_md = "Run the forecast for client AD"
 
     latest_only = LatestOnlyOperator(task_id="latest_only")

diff --git a/terraform/modules/services/airflow/dags/uk/dayafter-dag.py b/terraform/modules/services/airflow/dags/uk/dayafter-dag.py
@@ -38,7 +38,7 @@
 
     national_day_after = EcsRunTaskOperator(
         task_id=f'{region}-national-day-after',
-        task_definition='national-day-after',
+        task_definition='pvlive-national-day-after',
         cluster=cluster,
         overrides={},
         awslogs_region="eu-west-1",
@@ -66,7 +66,7 @@
 
     gsp_day_after = EcsRunTaskOperator(
         task_id=f'{region}-gsp-day-after',
-        task_definition='gsp-day-after',
+        task_definition='pvlive-gsp-day-after',
         cluster=cluster,
         overrides={},
         launch_type="FARGATE",

diff --git a/...dules/services/airflow/dags/uk/gsp-dag.py → ...es/services/airflow/dags/uk/pvlive-dag.py b/...dules/services/airflow/dags/uk/gsp-dag.py → ...es/services/airflow/dags/uk/pvlive-dag.py
@@ -39,7 +39,7 @@
 
     gsp_consumer = EcsRunTaskOperator(
         task_id=f'{region}-gsp-consumer',
-        task_definition='gsp',
+        task_definition='pvlive',
         cluster=cluster,
         overrides={},
         launch_type="FARGATE",

diff --git a/terraform/modules/services/eb_app/eb.tf b/terraform/modules/services/eb_app/eb.tf
@@ -87,7 +87,7 @@ resource "aws_elastic_beanstalk_environment" "eb-environment" {
   setting {
     namespace = "aws:autoscaling:asg"
     name      = "MaxSize"
-    value     = "1"
+    value     = var.max_ec2_count
     resource  = ""
   }
 

diff --git a/terraform/modules/services/eb_app/variables.tf b/terraform/modules/services/eb_app/variables.tf
@@ -97,3 +97,8 @@ variable "s3_bucket" {
   EOT
 }
 
+variable "max_ec2_count" {
+    type = number
+    description = "Maximum number of EC2 instances in the EB environment"
+    default = 1
+}
diff --git a/terraform/modules/storage/database-pair/rds.tf b/terraform/modules/storage/database-pair/rds.tf
@@ -2,7 +2,7 @@
 
 resource "aws_db_instance" "db-forecast" {
   allocated_storage            = 150
-  max_allocated_storage        = 200
+  max_allocated_storage        = 300
   engine                       = "postgres"
   engine_version               = var.engine_version
   instance_class               = "db.t3.medium"

diff --git a/terraform/modules/storage/open-data-pvnet-s3/bucket.tf b/terraform/modules/storage/open-data-pvnet-s3/bucket.tf
@@ -0,0 +1,11 @@
+# Public S3 bucket
+
+# Bucket itself
+# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket
+resource "aws_s3_bucket" "bucket" {
+  bucket = "ocf-open-data-pvnet"
+
+  tags = {
+    Name        = "Open_Data_PVNet"
+  }
+}
diff --git a/terraform/modules/storage/open-data-pvnet-s3/iam.tf b/terraform/modules/storage/open-data-pvnet-s3/iam.tf
@@ -0,0 +1,39 @@
+# Make IAM policy to write to the s3-public bucket
+
+data "aws_iam_policy_document" "open_data_pvnet_write_policy_description" {
+  version = "2012-10-17"
+  statement {
+    actions = [
+      "s3:ListBucket",
+      "s3:GetObject",
+      "s3:PutObject",
+      "s3:DeleteObject"
+    ]
+    resources = [aws_s3_bucket.bucket.arn, "${aws_s3_bucket.bucket.arn}/*"]
+    effect = "Allow"
+  }
+}
+
+# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy
+resource "aws_iam_policy" "open_data_pvnet_write_policy" {
+  name        = "s3-open_data_pvnet_write_policy"
+  description = "Policy to write to bucket: ${aws_s3_bucket.bucket.bucket}"
+
+  # Terraform's "jsonencode" function converts a
+  # Terraform expression result to valid JSON syntax.
+  policy = data.aws_iam_policy_document.open_data_pvnet_write_policy_description.json
+
+}
+
+
+# resource group
+resource "aws_iam_group" "open_data_pvnet_write_group" {
+  name = "open_data_pvnet_write_group"
+}
+
+# attach policy to group
+resource "aws_iam_policy_attachment" "open_data_pvnet_write_policy_attachment" {
+  name       = "s3-write-attachment"
+  policy_arn = aws_iam_policy.open_data_pvnet_write_policy.arn
+  groups      = [aws_iam_group.open_data_pvnet_write_group.name]
+}
diff --git a/terraform/nowcasting/development/main.tf b/terraform/nowcasting/development/main.tf
@@ -18,9 +18,9 @@ The componentes ares:
 3.4 - Satellite Consumer
 3.5 - Satellite Data Tailor Clean up
 3.6 - PV Consumer
-3.7 - GSP Consumer (From PVLive)
-3.8 - GSP Consumer - GSP Day After
-3.9 - GSP Consumer - National Day After
+3.7 - PVLive Consumer (From PVLive)
+3.8 - PVLive Consumer - GSP Day After
+3.9 - PVLive Consumer - National Day After
 4.1 - Metrics
 4.2 - Forecast PVnet 1
 4.3 - Forecast National XG
@@ -35,6 +35,7 @@ The componentes ares:
 6.3 - PVSite ML bucket
 6.4 - PVSite Forecast
 6.5 - PVSite Database Clean Up
+7.1 - Open Data PVnet
 
 Variables used across all modules
 ======*/
@@ -117,6 +118,7 @@ module "api" {
     { bucket_read_policy_arn = module.s3.iam-policy-s3-nwp-read.arn },
     { bucket_read_policy_arn = module.s3.iam-policy-s3-sat-read.arn }
   ]
+  max_ec2_count = 2
 }
 
 # 2.1
@@ -365,7 +367,7 @@ module "pv" {
 module "gsp-consumer" {
   source = "../../modules/services/ecs_task"
 
-  ecs-task_name = "gsp"
+  ecs-task_name = "pvlive"
   ecs-task_type = "consumer"
   ecs-task_execution_role_arn = module.ecs.ecs_task_execution_role_arn
   ecs-task_size = {
@@ -390,7 +392,7 @@ module "gsp-consumer" {
   values: ["DB_URL"]}
   ]
   container-tag         = var.gsp_version
-  container-name        = "openclimatefix/gspconsumer"
+  container-name        = "openclimatefix/pvliveconsumer"
   container-registry = "docker.io"
   container-command     = []
 }
@@ -399,7 +401,7 @@ module "gsp-consumer" {
 module "gsp-consumer-day-after-gsp" {
   source = "../../modules/services/ecs_task"
 
-  ecs-task_name = "gsp-day-after"
+  ecs-task_name = "pvlive-gsp-day-after"
   ecs-task_type = "consumer"
   ecs-task_execution_role_arn = module.ecs.ecs_task_execution_role_arn
   ecs-task_size = {
@@ -424,7 +426,7 @@ module "gsp-consumer-day-after-gsp" {
   values: ["DB_URL"]}
   ]
   container-tag         = var.gsp_version
-  container-name        = "openclimatefix/gspconsumer"
+  container-name        = "openclimatefix/pvliveconsumer"
   container-registry = "docker.io"
   container-command     = []
 }
@@ -433,7 +435,7 @@ module "gsp-consumer-day-after-gsp" {
 module "gsp-consumer-day-after-national" {
   source = "../../modules/services/ecs_task"
 
-  ecs-task_name = "national-day-after"
+  ecs-task_name = "pvlive-national-day-after"
   ecs-task_type = "consumer"
   ecs-task_execution_role_arn = module.ecs.ecs_task_execution_role_arn
   ecs-task_size = {
@@ -458,7 +460,7 @@ module "gsp-consumer-day-after-national" {
   values: ["DB_URL"]}
   ]
   container-tag         = var.gsp_version
-  container-name        = "openclimatefix/gspconsumer"
+  container-name        = "openclimatefix/pvliveconsumer"
   container-registry = "docker.io"
   container-command     = []
 }
@@ -529,7 +531,7 @@ source = "../../modules/services/ecs_task"
     { "name" : "LOGLEVEL", "value" : "INFO" },
     { "name" : "NWP_ZARR_PATH", "value":"s3://${module.s3.s3-nwp-bucket.id}/data-metoffice/latest.zarr"},
     { "name" : "SENTRY_DSN",  "value": var.sentry_dsn},
-    { "name": "ML_MODEL_BUCKET", "value": "s3://${module.forecasting_models_bucket.bucket_id}/"}
+    { "name": "ML_MODEL_BUCKET", "value": module.forecasting_models_bucket.bucket_id}
   ]
 
   container-secret_vars = [
@@ -918,3 +920,9 @@ module "pvsite_database_clean_up" {
                     ]
   container-command = []
 }
+
+
+# 7.1 Open Data PVnet - Public s3 bucket
+module "open_data_pvnet_s3" {
+  source = "../../modules/storage/open-data-pvnet-s3"
+}
diff --git a/terraform/nowcasting/production/main.tf b/terraform/nowcasting/production/main.tf
@@ -111,7 +111,7 @@ module "api" {
   container-tag  = var.api_version
   container-registry = "openclimatefix"
   eb-app_name    = "nowcasting-api"
-  eb-instance_type = "t3.small"
+  eb-instance_type = "t3.medium"
   s3_bucket = [
     { bucket_read_policy_arn = module.s3.iam-policy-s3-nwp-read.arn },
     { bucket_read_policy_arn = module.s3.iam-policy-s3-sat-read.arn }
@@ -120,7 +120,7 @@ module "api" {
 
 # 2.1
 module "database" {
-  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/database-pair?ref=6e24edf"
+  source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/storage/database-pair?ref=7b0c0e0"
 
   region               = var.region
   environment          = local.environment