CCM-12616 mesh poll retrieve

.github/workflows/stage-2-test.yaml

@@ -72,9 +72,10 @@ jobs:
       - uses: actions/setup-node@v6
         with:
           node-version: 24.10.0
-      - uses: actions/setup-python@v6
+      - name: "Setup Python"
+        uses: actions/setup-python@v6
         with:
-          python-version: '3.14'
+          python-version: ${{ inputs.python_version }}
       - name: "Run unit test suite"
         run: |
           make test-unit
@@ -94,14 +95,21 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: python-coverage-reports
-          path: "src/**/coverage.xml"
+          path: |
+            src/**/coverage.xml
+            utils/**/coverage.xml
+            lambdas/**/coverage.xml
   test-lint:
     name: "Linting"
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v5
+      - name: "Setup Python"
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ inputs.python_version }}
       - uses: actions/setup-node@v6
         with:
           node-version: 24.10.0
@@ -156,7 +164,7 @@ jobs:
         uses: actions/download-artifact@v5
         with:
           name: python-coverage-reports
-          path: src/
+          path: .
       - name: "Perform static analysis"
         uses: ./.github/actions/perform-static-analysis
         with:

.gitignore

@@ -12,6 +12,15 @@ version.json
 
 # Please, add your custom content below!
 
+# Don't track ephemeral local build files
+target/
+
+# Python build artifacts
+*.egg-info/
+.eggs/
+build/
+dist/
+
 # dependencies
 node_modules
 .node-version
@@ -25,6 +34,7 @@ dist
 output
 /schemas
 .env
+**/__pycache__
 
 # Python
 __pycache__/

.tool-versions

@@ -8,8 +8,6 @@ terraform 1.10.1
 terraform-docs 0.19.0
 trivy 0.61.0
 vale 3.6.0
-
-
 # ==============================================================================
 # The section below is reserved for Docker image versions.
 

Makefile

@@ -9,15 +9,26 @@ include scripts/init.mk
 
 quick-start: config clean build serve-docs # Quick start target to setup, build and serve docs @Pipeline
 
-dependencies: # Install dependencies needed to build and test the project @Pipeline
-	# TODO: Implement installation of your project dependencies
+dependencies:: # Install dependencies needed to build and test the project @Pipeline
+	$(MAKE) -C src/cloudevents install
+	$(MAKE) -C src/eventcatalogasyncapiimporter install
+	$(MAKE) -C lambdas/mesh-poll install
+	$(MAKE) -C lambdas/mesh-download install
+	$(MAKE) -C utils/metric-publishers install
+	$(MAKE) -C utils/event-publisher-py install
+	$(MAKE) -C utils/py-mock-mesh install
+	npm install --workspaces
+	$(MAKE) generate
 
-generate: # Generate any autogenerated output @Pipeline
-	npm run generate-dependencies
+dependencies-docs:: # Install documentation dependencies @Pipeline
+	$(MAKE) -C docs install
 
-build: # Build the project artefact @Pipeline
+build: dependencies-docs # Build the project artefact @Pipeline
 	$(MAKE) -C docs build
 
+generate: # Generate any autogenerated output @Pipeline
+	npm run generate-dependencies
+
 debug:
 	$(MAKE) -C docs debug
 
@@ -32,16 +43,16 @@ clean:: # Clean-up project resources (main) @Operations
 	$(MAKE) -C src/cloudevents clean && \
 	$(MAKE) -C src/eventcatalogasyncapiimporter clean && \
 	$(MAKE) -C src/eventcatalogasyncapiimporter clean-output && \
+	$(MAKE) -C lambdas/mesh-poll clean && \
+	$(MAKE) -C lambdas/mesh-download clean && \
+	$(MAKE) -C utils/metric-publishers clean && \
+	$(MAKE) -C utils/event-publisher-py clean && \
+	$(MAKE) -C utils/py-mock-mesh clean && \
 	$(MAKE) -C src/python-schema-generator clean && \
 	rm -f .version
 	npm run clean
 
-config:: _install-dependencies version # Configure development environment (main) @Configuration
-	$(MAKE) -C docs install
-	$(MAKE) -C src/cloudevents install
-	$(MAKE) -C src/eventcatalogasyncapiimporter install
-	npm install
-	$(MAKE) generate
+config:: _install-dependencies version dependencies # Configure development environment (main) @Configuration
 
 serve-docs:
 	$(MAKE) -C docs s

infrastructure/terraform/components/dl/README.md

@@ -17,14 +17,15 @@ No requirements.
 | <a name="input_component"></a> [component](#input\_component) | The variable encapsulating the name of this component | `string` | `"dl"` | no |
 | <a name="input_default_tags"></a> [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no |
 | <a name="input_enable_dynamodb_delete_protection"></a> [enable\_dynamodb\_delete\_protection](#input\_enable\_dynamodb\_delete\_protection) | Enable DynamoDB Delete Protection on all Tables | `bool` | `true` | no |
+| <a name="input_enable_mock_mesh"></a> [enable\_mock\_mesh](#input\_enable\_mock\_mesh) | Enable mock mesh access (dev only). Grants lambda permission to read mock-mesh prefix in non-pii bucket. | `bool` | `false` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |
 | <a name="input_force_destroy"></a> [force\_destroy](#input\_force\_destroy) | Flag to force deletion of S3 buckets | `bool` | `false` | no |
 | <a name="input_force_lambda_code_deploy"></a> [force\_lambda\_code\_deploy](#input\_force\_lambda\_code\_deploy) | If the lambda package in s3 has the same commit id tag as the terraform build branch, the lambda will not update automatically. Set to True if making changes to Lambda code from on the same commit for example during development | `bool` | `false` | no |
 | <a name="input_group"></a> [group](#input\_group) | The group variables are being inherited from (often synonmous with account short-name) | `string` | n/a | yes |
 | <a name="input_kms_deletion_window"></a> [kms\_deletion\_window](#input\_kms\_deletion\_window) | When a kms key is deleted, how long should it wait in the pending deletion state? | `string` | `"30"` | no |
 | <a name="input_log_level"></a> [log\_level](#input\_log\_level) | The log level to be used in lambda functions within the component. Any log with a lower severity than the configured value will not be logged: https://docs.python.org/3/library/logging.html#levels | `string` | `"INFO"` | no |
 | <a name="input_log_retention_in_days"></a> [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no |
-| <a name="input_mesh_poll_schedule"></a> [mesh\_poll\_schedule](#input\_mesh\_poll\_schedule) | Schedule to poll MESH for messages | `string` | `"cron(0,30 8-16 ? * MON-FRI *)"` | no |
+| <a name="input_mesh_poll_schedule"></a> [mesh\_poll\_schedule](#input\_mesh\_poll\_schedule) | Schedule to poll MESH for messages | `string` | `"rate(5 minutes)"` | no |
 | <a name="input_parent_acct_environment"></a> [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no |
 | <a name="input_project"></a> [project](#input\_project) | The name of the tfscaffold project | `string` | n/a | yes |
 | <a name="input_queue_batch_size"></a> [queue\_batch\_size](#input\_queue\_batch\_size) | maximum number of queue items to process | `number` | `10` | no |
@@ -39,11 +40,15 @@ No requirements.
 | <a name="module_kms"></a> [kms](#module\_kms) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-kms.zip | n/a |
 | <a name="module_lambda_apim_key_generation"></a> [lambda\_apim\_key\_generation](#module\_lambda\_apim\_key\_generation) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-lambda.zip | n/a |
 | <a name="module_lambda_lambda_apim_refresh_token"></a> [lambda\_lambda\_apim\_refresh\_token](#module\_lambda\_lambda\_apim\_refresh\_token) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-lambda.zip | n/a |
+| <a name="module_mesh_download"></a> [mesh\_download](#module\_mesh\_download) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-lambda.zip | n/a |
 | <a name="module_mesh_poll"></a> [mesh\_poll](#module\_mesh\_poll) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-lambda.zip | n/a |
 | <a name="module_s3bucket_cf_logs"></a> [s3bucket\_cf\_logs](#module\_s3bucket\_cf\_logs) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a |
 | <a name="module_s3bucket_letters"></a> [s3bucket\_letters](#module\_s3bucket\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a |
+| <a name="module_s3bucket_non_pii_data"></a> [s3bucket\_non\_pii\_data](#module\_s3bucket\_non\_pii\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a |
+| <a name="module_s3bucket_pii_data"></a> [s3bucket\_pii\_data](#module\_s3bucket\_pii\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a |
 | <a name="module_s3bucket_static_assets"></a> [s3bucket\_static\_assets](#module\_s3bucket\_static\_assets) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-s3bucket.zip | n/a |
 | <a name="module_sqs_event_publisher_errors"></a> [sqs\_event\_publisher\_errors](#module\_sqs\_event\_publisher\_errors) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |
+| <a name="module_sqs_mesh_download"></a> [sqs\_mesh\_download](#module\_sqs\_mesh\_download) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |
 | <a name="module_sqs_ttl"></a> [sqs\_ttl](#module\_sqs\_ttl) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |
 | <a name="module_sqs_ttl_handle_expiry_errors"></a> [sqs\_ttl\_handle\_expiry\_errors](#module\_sqs\_ttl\_handle\_expiry\_errors) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |
 | <a name="module_ttl_create"></a> [ttl\_create](#module\_ttl\_create) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-lambda.zip | n/a |

infrastructure/terraform/components/dl/cloudwatch_event_rule_mesh_inbox_message_received.tf

@@ -0,0 +1,24 @@
+resource "aws_cloudwatch_event_rule" "mesh_inbox_message_received" {
+  name           = "${local.csi}-mesh-inbox-message-received"
+  description    = "Route MESHInboxMessageReceived events from mesh-poll lambda to mesh-download queue"
+  event_bus_name = aws_cloudwatch_event_bus.main.name
+
+  event_pattern = jsonencode({
+    "detail" : {
+      "type" : [
+        "uk.nhs.notify.digital.letters.mesh.inbox.message.received.v1"
+      ],
+      "dataschemaversion" : [{
+        "prefix" : "1."
+      }]
+    }
+  })
+}
+
+# EventBridge target to send events to SQS queue
+resource "aws_cloudwatch_event_target" "mesh_download_sqs" {
+  rule           = aws_cloudwatch_event_rule.mesh_inbox_message_received.name
+  target_id      = "mesh-inbox-message-received-sqs-target"
+  arn            = module.sqs_mesh_download.sqs_queue_arn
+  event_bus_name = aws_cloudwatch_event_bus.main.name
+}

infrastructure/terraform/components/dl/lambda_event_source_mapping_mesh_download.tf

@@ -0,0 +1,10 @@
+resource "aws_lambda_event_source_mapping" "mesh_download" {
+  event_source_arn                   = module.sqs_mesh_download.sqs_queue_arn
+  function_name                      = module.mesh_download.function_name
+  batch_size                         = var.queue_batch_size
+  maximum_batching_window_in_seconds = var.queue_batch_window_seconds
+
+  function_response_types = [
+    "ReportBatchItemFailures"
+  ]
+}

infrastructure/terraform/components/dl/locals.tf

@@ -5,6 +5,7 @@ locals {
   apim_api_key_ssm_parameter_name      = "/${var.component}/${var.environment}/apim/api_key"
   apim_private_key_ssm_parameter_name  = "/${var.component}/${var.environment}/apim/private_key"
   apim_keystore_s3_bucket              = "nhs-${var.aws_account_id}-${var.region}-${var.environment}-${var.component}-static-assets"
+  ssm_mesh_prefix                      = "/${var.component}/${var.environment}/mesh"
   root_domain_name                     = "${var.environment}.${local.acct.route53_zone_names["digital-letters"]}"
   root_domain_id                       = local.acct.route53_zone_ids["digital-letters"]
   ttl_shard_count = 3

infrastructure/terraform/components/dl/module_lambda_mesh_download.tf

@@ -0,0 +1,164 @@
+module "mesh_download" {
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-lambda.zip"
+
+  function_name = "mesh-download"
+  description   = "A lambda function for downloading MESH messages and storing in S3"
+
+  aws_account_id = var.aws_account_id
+  component      = local.component
+  environment    = var.environment
+  project        = var.project
+  region         = var.region
+  group          = var.group
+
+  log_retention_in_days = var.log_retention_in_days
+  kms_key_arn           = module.kms.key_arn
+
+  iam_policy_document = {
+    body = data.aws_iam_policy_document.mesh_download_lambda.json
+  }
+
+  function_s3_bucket      = local.acct.s3_buckets["lambda_function_artefacts"]["id"]
+  function_code_base_path = local.aws_lambda_functions_dir_path
+  function_code_dir       = "mesh-download/target/dist"
+  function_include_common = true
+  function_module_name    = "mesh_download"
+  handler_function_name   = "handler.handler"
+  runtime                 = "python3.14"
+  memory                  = 256
+  timeout                 = 60
+  log_level               = var.log_level
+
+  force_lambda_code_deploy = var.force_lambda_code_deploy
+  enable_lambda_insights   = false
+
+  send_to_firehose          = true
+  log_destination_arn       = local.log_destination_arn
+  log_subscription_role_arn = local.acct.log_subscription_role_arn
+
+  lambda_env_vars = {
+    SSM_PREFIX                          = "/dl/${var.environment}/mesh"
+    EVENT_PUBLISHER_EVENT_BUS_ARN       = aws_cloudwatch_event_bus.main.arn
+    EVENT_PUBLISHER_DLQ_URL             = module.sqs_event_publisher_errors.sqs_queue_url
+    ENVIRONMENT                         = var.environment
+    PII_BUCKET                          = module.s3bucket_pii_data.bucket
+    CERTIFICATE_EXPIRY_METRIC_NAME      = "mesh-download-client-certificate-near-expiry"
+    CERTIFICATE_EXPIRY_METRIC_NAMESPACE = "dl-mesh-download"
+    DOWNLOAD_METRIC_NAME                = "mesh-download-successful-downloads"
+    DOWNLOAD_METRIC_NAMESPACE           = "dl-mesh-download"
+    USE_MESH_MOCK                       = var.enable_mock_mesh ? "true" : "false"
+  }
+
+}
+
+data "aws_iam_policy_document" "mesh_download_lambda" {
+  # Mock S3 ListBucket only when enabled
+  dynamic "statement" {
+    for_each = var.enable_mock_mesh ? [1] : []
+    content {
+      sid    = "MockMeshListBucket"
+      effect = "Allow"
+
+      actions = [
+        "s3:ListBucket"
+      ]
+
+      resources = [
+        module.s3bucket_non_pii_data.arn
+      ]
+
+      condition {
+        test     = "StringLike"
+        variable = "s3:prefix"
+        values   = ["mock-mesh/*"]
+      }
+    }
+  }
+
+  # Mock S3 GetObject only when enabled
+  dynamic "statement" {
+    for_each = var.enable_mock_mesh ? [1] : []
+    content {
+      sid    = "MockMeshGetObject"
+      effect = "Allow"
+
+      actions = [
+        "s3:GetObject"
+      ]
+
+      resources = [
+        "${module.s3bucket_non_pii_data.arn}/mock-mesh/*"
+      ]
+    }
+  }
+
+  statement {
+    sid    = "KMSPermissions"
+    effect = "Allow"
+
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey",
+    ]
+
+    resources = [
+      module.kms.key_arn,
+    ]
+  }
+
+  statement {
+    sid    = "S3BucketPermissions"
+    effect = "Allow"
+
+    actions = [
+      "s3:PutObject",
+      "s3:GetObject",
+    ]
+
+    resources = [
+      "${module.s3bucket_pii_data.arn}/*",
+    ]
+  }
+
+  statement {
+    sid    = "SQSPermissions"
+    effect = "Allow"
+
+    actions = [
+      "sqs:ReceiveMessage",
+      "sqs:DeleteMessage",
+      "sqs:GetQueueAttributes",
+    ]
+
+    resources = [
+      module.sqs_mesh_download.sqs_queue_arn,
+    ]
+  }
+
+  statement {
+    sid    = "EventBridgePermissions"
+    effect = "Allow"
+
+    actions = [
+      "events:PutEvents",
+    ]
+
+    resources = [
+      aws_cloudwatch_event_bus.main.arn,
+    ]
+  }
+
+  statement {
+    sid    = "DLQPermissions"
+    effect = "Allow"
+
+    actions = [
+      "sqs:SendMessage",
+      "sqs:SendMessageBatch",
+    ]
+
+    resources = [
+      module.sqs_event_publisher_errors.sqs_queue_arn,
+    ]
+  }
+}