# https://github.com/awslabs/cid-data-collection-framework/blob/main/data-exports/deploy/data-exports-aggregation.yaml
AWSTemplateFormatVersion: "2010-09-09"
Description: AWS Billing Data Export Aggregation v0.6.0 - AWS Solution SO9011
Metadata:

  AWS::CloudFormation::Interface:
    ParameterGroups:
      -
        Label:
          default: "Common Parameters Configuration"
        Parameters:
          - DestinationAccountId
          - ResourcePrefix
          - ManageCUR2
          - ManageFOCUS
          - ManageCOH
          - ManageCarbon
      -
        Label:
          default: "Parameters needed in Destination (Data Collection) Account only"
        Parameters:
          - SourceAccountIds
      -
        Label:
          default: "Technical Parameters. Please do not change."
        Parameters:
          - EnableSCAD
          - RolePath
          - TimeGranularity
          - SecondaryDestinationBucket

    ParameterLabels:
      ManageCOH:
        default: "Cost Optimization Recommendations"
      ManageCUR2:
        default: "CUR 2.0"
      ManageFOCUS:
        default: "FOCUS"
      ManageCarbon:
        default: "Carbon Emissions"
      DestinationAccountId:
        default: "Destination (Data Collection) Account Id"
      ResourcePrefix:
        default: "Resource Prefix"
      SourceAccountIds:
        default: "Source Account Ids (Comma separated list)"
      EnableSCAD:
        default: "Enable Split Cost Allocation Data (SCAD) in CUR 2.0"
      TimeGranularity:
        default: "CUR 2.0 Granularity. Do not change."
      LakeFormationEnabled:
        default: "I have LakeFormation permission model in place for this account & my CFN deployment credentials have administrative rights on LakeFormation"
      SecondaryDestinationBucket:
        default: "Secondary Destination Bucket Name. Keep it Empty."


Parameters:

  ##
  # Common params
  ##
  DestinationAccountId:
    Type: String
    Description: "AWS Account Id where DataExport will be replicated to (Where you deploy CID Quicksight Dashboards or Data Collection)"
    AllowedPattern: '\d{12}'
  ResourcePrefix:
    Type: String
    Default: "cid"
    Description: "Prefix used for all named resources, including S3 Bucket. Must be the same in destination and source stacks"
    MaxLength:  37 # = 63 - len('-123456789012-data-exports')
    AllowedPattern: "^[a-z0-9]+[a-z0-9-]{1,61}[a-z0-9]+$"
  ManageCUR2:
    Type: String
    Description: ""
    AllowedValues: ['yes', 'no']
    Default: "no"
  ManageFOCUS:
    Type: String
    Description: "NOTE: you can have only one export of this type"
    AllowedValues: ['yes', 'no']
    Default: "no"
  ManageCOH:
    Type: String
    Description:  "NOTE: you must have Cost Optimization Hub Enabled"
    AllowedValues: ['yes', 'no']
    Default: "no"
  ManageCarbon:
    Type: String
    Description: ""
    AllowedValues: ["yes", "no"]
    Default: "no"

  ##
  # Destination specific params
  ##
  SourceAccountIds:
    Type: String
    AllowedPattern: "^((\\d{12})\\,?)*$"
    Default: ''
    Description: "Ex:  12345678912,98745612312,.... If you install all in the same account (Source=Destination) please put Destination Account Id first in the list. "

  ##
  # Technical params
  ##
  EnableSCAD:
    Type: String
    Description: Whether to enable Split Cost Allocation Data (Scad). Set this to 'No', if you experience performance issues due to dataset size.
    AllowedValues: ['yes', 'no']
    Default: "yes"
  RolePath:
    Type: String
    Description: Path for roles where PermissionBoundaries can limit location
    Default: '/'
  TimeGranularity:
    Type: String
    Description: Changing of this parameter will require redeployment of this Stack, purging of data in Destination and then additional Backfill request. HOURLY is a recommended option unless your AWS invoice is more then $50M (in this case contact your TAM when installing).
    Default: "HOURLY"
    AllowedValues: ["HOURLY", "DAILY", "MONTHLY"]
  LakeFormationEnabled:
    Type: String
    Description: Choose 'yes' if Lake Formation permission model is in place for the account. If you are not sure, leave it as 'no'. You need to install cid-lakeformation-prerequisite.yaml first.
    Default: "no"
    AllowedValues: ["yes", "no"]
  SecondaryDestinationBucket:
    Type: String
    Description: Leave Empty. A Bucket name for optional supplementary data replication. If not empty, the export data will be replicated to this bucket in addition to the default replication destination. Keep it empty if unsure.
    Default: ''
  AddScheduleForBlockingWrite:
    Type: String
    Description: 'Set yes to schedule Disabling/Enabling schedule to stop CUR replication for a short period of time to secure QuickSight Dataset refresh.'
    Default: "no"
    AllowedValues: ["yes", "no"]
  DisableWriteCronSchedule:
    Type: String
    Description: 'Cron expression for disabling replication (UTC)'
    Default: '0 1 * * ? *'
    AllowedPattern: '^[0-9*,\-/\s?]+$'
    ConstraintDescription: Must be a valid cron expression
  EnableWriteCronSchedule:
    Type: String
    Description: 'Cron expression for enabling replication (UTC)'
    Default: '0 3 * * ? *'
    AllowedPattern: '^[0-9*,\-/\s?]+$'
    ConstraintDescription: Must be a valid cron expression

Conditions:
  EmptySourceAccountIds: !Equals [ !Ref SourceAccountIds, '']
  IsDestinationAccount: !Equals [!Ref DestinationAccountId, !Ref 'AWS::AccountId']
  NonEmptySecondaryDestinationBucket: !Not [ !Equals [ !Ref SecondaryDestinationBucket, ''] ]
  IsSourceAccount:
    # it is Source account if it is not a destination or if it is a destination and it is listed in Source Accounts (as the list one).
    # Unfortunately, there no 'Fn::Contains' in Conditions, so we need to request user setting Dest account as the first.
    Fn::Or:
      - !Not [!Condition IsDestinationAccount]
      - !Equals [!Ref 'AWS::AccountId', !Select [0, !Split [',', !Sub '${SourceAccountIds},']]]
  RegionSupportsDataExportsViaCFN: # CFN supports DataExports only in us-east-1 and cn-northwest-1. Other regions must use lambda.
    Fn::Or:
      - !Equals [!Ref 'AWS::Region', 'us-east-1']
      - !Equals [!Ref 'AWS::Region', 'cn-northwest-1']
  ManageCUR2: !Equals [!Ref ManageCUR2, 'yes']
  ManageFOCUS: !Equals [!Ref ManageFOCUS, 'yes']
  ManageCOH: !Equals [!Ref ManageCOH, 'yes']
  ManageCarbon: !Equals [!Ref ManageCarbon, "yes"]
  EnableSCAD: !Equals [!Ref EnableSCAD, 'yes']
  DeployDataExport:
    Fn::Or:
      - !Condition ManageCUR2
      - !Condition ManageFOCUS
      - !Condition ManageCOH
      - !Condition ManageCarbon
  DeploySourceS3:
    Fn::And:
      - !Condition IsSourceAccount
      - !Condition DeployDataExport
  DeployCOHServiceRole:  !And [!Condition IsSourceAccount, !Condition ManageCOH]
  DeployCUR2ViaCFN:      !And [!Condition IsSourceAccount, !Condition ManageCUR2,        !Condition RegionSupportsDataExportsViaCFN]
  DeployFOCUSViaCFN:     !And [!Condition IsSourceAccount, !Condition ManageFOCUS,       !Condition RegionSupportsDataExportsViaCFN]
  DeployCOHViaCFN:       !And [!Condition IsSourceAccount, !Condition ManageCOH,         !Condition RegionSupportsDataExportsViaCFN]
  DeployCarbonViaCFN:    !And [!Condition IsSourceAccount, !Condition ManageCarbon,      !Condition RegionSupportsDataExportsViaCFN]
  DeployCUR2ViaLambda:   !And [!Condition IsSourceAccount, !Condition ManageCUR2,   !Not [!Condition RegionSupportsDataExportsViaCFN]]
  DeployFOCUSViaLambda:  !And [!Condition IsSourceAccount, !Condition ManageFOCUS,  !Not [!Condition RegionSupportsDataExportsViaCFN]]
  DeployCOHViaLambda:    !And [!Condition IsSourceAccount, !Condition ManageCOH,    !Not [!Condition RegionSupportsDataExportsViaCFN]]
  DeployCarbonViaLambda: !And [!Condition IsSourceAccount, !Condition ManageCarbon, !Not [!Condition RegionSupportsDataExportsViaCFN]]
  DeployCUR2Table:       !And [!Condition IsDestinationAccount, !Condition ManageCUR2]
  DeployFOCUSTable:      !And [!Condition IsDestinationAccount, !Condition ManageFOCUS]
  DeployCOHTable:        !And [!Condition IsDestinationAccount, !Condition ManageCOH]
  DeployCarbonTable:     !And [!Condition IsDestinationAccount, !Condition ManageCarbon]
  DeployAnyExportViaLambda: !Or [!Condition DeployCUR2ViaLambda, !Condition DeployFOCUSViaLambda, !Condition DeployCOHViaLambda, !Condition DeployCarbonViaLambda]
  DeployAnyTable: !Or [!Condition DeployFOCUSTable, !Condition DeployCUR2Table, !Condition DeployCOHTable, !Condition DeployCarbonTable]

  NeedLakeFormationEnabledDB: !And [ !Equals [ !Ref LakeFormationEnabled, "yes"], !Condition DeployAnyTable]
  NeedLakeFormationEnabledCOH:    !And [!Condition NeedLakeFormationEnabledDB, !Condition ManageCOH]
  NeedLakeFormationEnabledCUR2:   !And [!Condition NeedLakeFormationEnabledDB, !Condition ManageCUR2]
  NeedLakeFormationEnabledFOCUS:  !And [!Condition NeedLakeFormationEnabledDB, !Condition ManageFOCUS]
  NeedLakeFormationEnabledCarbon: !And [!Condition NeedLakeFormationEnabledDB, !Condition ManageCarbon]
  DeployWriteBlocker:
    Fn::And:
      - !Condition IsDestinationAccount
      - !Equals [!Ref AddScheduleForBlockingWrite, "yes"]

Mappings:
  DataExports:
    #Mappings for storing values for different Data Exports tables
    CUR2:
      DefaultQuery: >-
        SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment 
        FROM COST_AND_USAGE_REPORT
      SCADQuery: >-
        SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment, split_line_item_actual_usage, split_line_item_net_split_cost, split_line_item_net_unused_cost, split_line_item_parent_resource_id, split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, split_line_item_split_usage_ratio, split_line_item_unused_cost    
        FROM COST_AND_USAGE_REPORT
    FOCUS:
      DefaultQuery: >-
        SELECT AvailabilityZone, BilledCost, BillingAccountId, BillingAccountName, BillingCurrency, BillingPeriodEnd, BillingPeriodStart, ChargeCategory, ChargeClass, ChargeDescription, ChargeFrequency, ChargePeriodEnd, ChargePeriodStart, CommitmentDiscountCategory, CommitmentDiscountId, CommitmentDiscountName, CommitmentDiscountType, CommitmentDiscountStatus, ConsumedQuantity, ConsumedUnit, ContractedCost, ContractedUnitPrice, EffectiveCost, InvoiceIssuerName, ListCost, ListUnitPrice, PricingCategory, PricingQuantity, PricingUnit, ProviderName, PublisherName, RegionId, RegionName, ResourceId, ResourceName, ResourceType, ServiceCategory, ServiceName, SkuId, SkuPriceId, SubAccountId, SubAccountName, Tags, x_CostCategories, x_Discounts, x_Operation, x_ServiceCode, x_UsageType
        FROM FOCUS_1_0_AWS
    COH:
      DefaultQuery: >-
        SELECT account_id, action_type, currency_code, current_resource_details, current_resource_summary, current_resource_type, estimated_monthly_cost_after_discount, estimated_monthly_cost_before_discount, estimated_monthly_savings_after_discount, estimated_monthly_savings_before_discount, estimated_savings_percentage_after_discount, estimated_savings_percentage_before_discount, implementation_effort, last_refresh_timestamp, recommendation_id, recommendation_lookback_period_in_days, recommendation_source, recommended_resource_details, recommended_resource_summary, recommended_resource_type, region, resource_arn, restart_needed, rollback_possible, tags
        FROM COST_OPTIMIZATION_RECOMMENDATIONS
    Carbon:
      DefaultQuery: >-
        SELECT last_refresh_timestamp, location, model_version, payer_account_id, product_code, region_code, total_mbm_emissions_unit, total_mbm_emissions_value, total_lbm_emissions_unit, total_lbm_emissions_value, usage_account_id, usage_period_end, usage_period_start
        FROM CARBON_EMISSIONS

Resources:

###########################################################################
# Destination Account Resources
###########################################################################


  DestinationS3:
    Type: AWS::S3::Bucket
    Condition: IsDestinationAccount
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      BucketName: !Sub "${ResourcePrefix}-${AWS::AccountId}-data-exports"

      ## Uncomment following lines to enable bucket logging if needed. Please be careful with the cost of logging.
      # LoggingConfiguration:
      #   DestinationBucketName: REPLACE_WITH_YOUR_LOGGING_BUCKET
      #   LogFilePrefix: REPLACE_WITH_YOUR_PREFIX

      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256 ## Use AWS managed KMS
              ## If you need Customer managed KMS key, yoy can do that using following parameters:
              # SSEAlgorithm: aws:kms
              # KMSMasterKeyID: "REPLACE_WITH_YOUR_KEY_ARN"
      AccessControl: BucketOwnerFullControl
      OwnershipControls:
        Rules:
          - ObjectOwnership: BucketOwnerEnforced
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
      VersioningConfiguration:
        Status: Enabled
      LifecycleConfiguration:
        Rules:
          - Id: Object&Version Expiration
            Status: Enabled
            NoncurrentVersionExpirationInDays: 1
          - Id: DeleteIncompleteMultipartUploadsAndExpiredDeleteMarkers
            Status: Enabled
            AbortIncompleteMultipartUpload:
              DaysAfterInitiation: 7
            ExpiredObjectDeleteMarker: true
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: 'W35'
            reason: "Data buckets would generate too much logs"
      cfn-lint:
        config:
          ignore_checks:
            - W3045 # Need to use AccessControl for replication

  DestinationS3BucketPolicy:
    Type: 'AWS::S3::BucketPolicy'
    Condition: IsDestinationAccount
    DeletionPolicy: Retain
    UpdateReplacePolicy: Delete
    Properties:
      Bucket:
        Ref: DestinationS3
      PolicyDocument:
        Id: AllowReplication
        Version: "2012-10-17"
        Statement:
          - Sid: AllowTLS12Only
            Effect: Deny
            Principal: "*"
            Action: s3:*
            Resource:
              - !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}'
              - !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}/*'
            Condition:
              NumericLessThan:
                s3:TlsVersion: 1.2
          - Sid: AllowOnlyHTTPS
            Effect: Deny
            Principal: "*"
            Action: s3:*
            Resource:
              - !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}'
              - !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}/*'
            Condition:
              Bool:
                aws:SecureTransport: false
          - Sid: AllowReplicationWrite
            Effect: Allow
            Principal:
              AWS:
                Fn::If:
                  - EmptySourceAccountIds
                  - !Ref AWS::AccountId
                  - !Split [',', !Ref SourceAccountIds]
            Action:
              - s3:ReplicateDelete
              - s3:ReplicateObject
            Resource: !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}/*'
          - Sid: AllowReplicationRead
            Effect: Allow
            Principal:
              AWS:
                Fn::If:
                  - EmptySourceAccountIds
                  - !Ref AWS::AccountId
                  - !Split [',', !Ref SourceAccountIds]
            Action:
              - s3:ListBucket
              - s3:ListBucketVersions
              - s3:GetBucketVersioning
              - s3:PutBucketVersioning
            Resource: !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}'

  SourceS3:
    Type: AWS::S3::Bucket
    Condition: DeploySourceS3
    DeletionPolicy: Delete
    UpdateReplacePolicy: Delete
    Properties:
      BucketName: !Sub ${ResourcePrefix}-${AWS::AccountId}-data-local

      ## Uncomment following lines to enable bucket logging if needed. Please be careful with the cost of logging.
      # LoggingConfiguration:
      #   DestinationBucketName: REPLACE_WITH_YOUR_LOGGING_BUCKET
      #   LogFilePrefix: REPLACE_WITH_YOUR_PREFIX

      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256 ## Use AWS managed KMS
              ## If you need Customer managed KMS key, yoy can do that using following parameters:
              # SSEAlgorithm: aws:kms
              # KMSMasterKeyID: "REPLACE_WITH_YOUR_KEY_ARN"
      AccessControl: BucketOwnerFullControl
      OwnershipControls:
        Rules:
          - ObjectOwnership: BucketOwnerEnforced
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
      VersioningConfiguration:
        Status: Enabled
      ReplicationConfiguration:
        Role: !GetAtt ReplicationRole.Arn
        Rules:
          - Destination:
              Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports"
              StorageClass: STANDARD
            Id: ReplicateCUR2Data
            Filter:
              Prefix: !Sub "cur2/${AWS::AccountId}/${ResourcePrefix}-cur2/data/"  # Hardcoded export name
            Status: Enabled
            Priority: 1
            DeleteMarkerReplication:
              Status: Enabled
          - Destination:
              Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports"
              StorageClass: STANDARD
            Id: ReplicateFOCUSData
            Filter:
              Prefix: !Sub "focus/${AWS::AccountId}/${ResourcePrefix}-focus/data/" # Hardcoded export name
            Status: Enabled
            Priority: 2
            DeleteMarkerReplication:
              Status: Enabled
          - Destination:
              Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports"
              StorageClass: STANDARD
            Id: ReplicateCOHData
            Filter:
              Prefix: !Sub "coh/${AWS::AccountId}/${ResourcePrefix}-coh/data/" # Hardcoded export name
            Status: Enabled
            Priority: 3
            DeleteMarkerReplication:
              Status: Enabled
          - Destination:
              Bucket: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports"
              StorageClass: STANDARD
            Id: ReplicateCarbonData
            Filter:
              Prefix: !Sub "carbon/${AWS::AccountId}/${ResourcePrefix}-carbon/data/" # Hardcoded export name
            Status: Enabled
            Priority: 4
            DeleteMarkerReplication:
              Status: Enabled
          - Fn::If:
            - NonEmptySecondaryDestinationBucket
            - Destination:
                Bucket: !Sub "arn:${AWS::Partition}:s3:::${SecondaryDestinationBucket}"
                StorageClass: STANDARD
              Id: ReplicateCUR2DataToSecondaryBucket
              Filter:
                Prefix: !Sub "cur2/${AWS::AccountId}/${ResourcePrefix}-cur2/data/"  # Hardcoded export name
              Status: Enabled
              Priority: 101
              DeleteMarkerReplication:
                Status: Enabled
            - !Ref 'AWS::NoValue'
          - Fn::If:
            - NonEmptySecondaryDestinationBucket
            - Destination:
                Bucket: !Sub "arn:${AWS::Partition}:s3:::${SecondaryDestinationBucket}"
                StorageClass: STANDARD
              Id: ReplicateFOCUSDataToSecondaryBucket
              Filter:
                Prefix: !Sub "focus/${AWS::AccountId}/${ResourcePrefix}-focus/data/" # Hardcoded export name
              Status: Enabled
              Priority: 102
              DeleteMarkerReplication:
                Status: Enabled
            - !Ref 'AWS::NoValue'
          - Fn::If:
            - NonEmptySecondaryDestinationBucket
            - Destination:
                Bucket: !Sub "arn:${AWS::Partition}:s3:::${SecondaryDestinationBucket}"
                StorageClass: STANDARD
              Id: ReplicateCOHDataToSecondaryBucket
              Filter:
                Prefix: !Sub "coh/${AWS::AccountId}/${ResourcePrefix}-coh/data/" # Hardcoded export name
              Status: Enabled
              Priority: 103
              DeleteMarkerReplication:
                Status: Enabled
            - !Ref 'AWS::NoValue'
          - Fn::If:
            - NonEmptySecondaryDestinationBucket
            - Destination:
                Bucket: !Sub "arn:${AWS::Partition}:s3:::${SecondaryDestinationBucket}"
                StorageClass: STANDARD
              Id: ReplicateCarbonDataToSecondaryBucket
              Filter:
                Prefix: !Sub "carbon/${AWS::AccountId}/${ResourcePrefix}-carbon/data/" # Hardcoded export name
              Status: Enabled
              Priority: 104
              DeleteMarkerReplication:
                Status: Enabled
            - !Ref "AWS::NoValue"
      LifecycleConfiguration:
        Rules:
          - Id: Object&Version Expiration
            Status: Enabled
            NoncurrentVersionExpirationInDays: 1
            ExpirationInDays: 7
          - Id: DeleteIncompleteMultipartUploadsAndExpiredDeleteMarkers
            Status: Enabled
            AbortIncompleteMultipartUpload:
              DaysAfterInitiation: 7
            ExpiredObjectDeleteMarker: true
      Tags: # Hacky way to manage conditional dependencies
        - Key: IgnoreConditionalDependency
          Value: !If [IsDestinationAccount, !Ref DestinationS3, '']
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: 'W35'
            reason: "Data buckets would generate too much logs"
      cfn-lint:
        config:
          ignore_checks:
            - W3045 # Need to use AccessControl for replication

  SourceS3BucketPolicy:
    Type: 'AWS::S3::BucketPolicy'
    Condition: DeploySourceS3
    DeletionPolicy: Delete
    UpdateReplacePolicy: Delete
    Properties:
      Bucket: !Ref SourceS3
      PolicyDocument:
        Id: AllowBillingReadAndWrite
        Version: "2012-10-17"
        Statement:
          - Sid: AllowTLS12Only
            Effect: Deny
            Principal: "*"
            Action: s3:*
            Resource:
              - !Sub 'arn:${AWS::Partition}:s3:::${SourceS3}'
              - !Sub 'arn:${AWS::Partition}:s3:::${SourceS3}/*'
            Condition:
              NumericLessThan:
                s3:TlsVersion: 1.2
          - Sid: AllowOnlyHTTPS
            Effect: Deny
            Principal: "*"
            Action: s3:*
            Resource:
              - !Sub 'arn:${AWS::Partition}:s3:::${SourceS3}'
              - !Sub 'arn:${AWS::Partition}:s3:::${SourceS3}/*'
            Condition:
              Bool:
                aws:SecureTransport: false
          - Sid: AllowBillingReadAndWrite
            Effect: Allow
            Principal:
              Service: bcm-data-exports.amazonaws.com
            Action:
              - s3:GetBucketAcl
              - s3:GetBucketPolicy
              - s3:PutObject
            Resource:
              - !Sub 'arn:${AWS::Partition}:s3:::${SourceS3}'
              - !Sub 'arn:${AWS::Partition}:s3:::${SourceS3}/*'
            Condition:
              StringEquals:
                aws:SourceAccount: !Ref AWS::AccountId

  ReplicationRole:
    Condition: DeploySourceS3
    Type: AWS::IAM::Role
    Properties:
      Path: !Sub /${ResourcePrefix}/
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          -
            Effect: Allow
            Principal:
              Service:
                - "s3.amazonaws.com"
            Action:
              - "sts:AssumeRole"
      Policies:
        - PolicyName: ReplicationPolicy
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - s3:GetReplicationConfiguration
                  - s3:ListBucket
                Resource: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${AWS::AccountId}-data-local"
              - Effect: Allow
                Action:
                  - s3:GetObjectVersionForReplication
                  - s3:GetObjectVersionAcl
                  - s3:GetObjectVersionTagging
                Resource: !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${AWS::AccountId}-data-local/*"
              - Effect: Allow
                Action:
                  - s3:ReplicateObject
                  - s3:ReplicateDelete
                  - s3:ReplicateTags
                Resource:
                  - !Sub "arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports/*/${AWS::AccountId}/*"
                  - !If [NonEmptySecondaryDestinationBucket, !Sub "arn:${AWS::Partition}:s3:::${SecondaryDestinationBucket}/*/${AWS::AccountId}/*", !Ref 'AWS::NoValue']

  BucketWriteBlockerLambdaRole:
    Type: AWS::IAM::Role
    Condition: DeployWriteBlocker
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: BucketPolicyAccess
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - s3:GetBucketPolicy
                  - s3:PutBucketPolicy
                Resource: !GetAtt DestinationS3.Arn

  BucketWriteBlockerPolicyLambda:
    Type: AWS::Lambda::Function
    Condition: DeployWriteBlocker
    Properties:
      Runtime: python3.12
      Handler: index.lambda_handler
      FunctionName: !Sub ${ResourcePrefix}-CID-WriteBlocker
      Role: !GetAtt BucketWriteBlockerLambdaRole.Arn
      Code:
        ZipFile: |
          import os
          import json
          import boto3

          POLICY_SID = os.environ['POLICY_SID']
          BUCKET_NAME = os.environ['BUCKET_NAME']
          s3 = boto3.client('s3')

          def lambda_handler(event, context):
              action = event.get('action', 'enable')  # 'enable' or 'disable'
              try:
                  policy = json.loads(s3.get_bucket_policy(Bucket=BUCKET_NAME)['Policy'])
                  # Find and modify the policy statement
                  for statement in policy['Statement']:
                      if statement.get('Sid') == POLICY_SID:
                          statement['Effect'] = 'Allow' if action == 'enable' else 'Deny'
                          break
                  else:
                      raise Exception(f'{POLICY_SID} statement not found in policy')
                  s3.put_bucket_policy(Bucket=BUCKET_NAME, Policy=json.dumps(policy))
                  return {'statusCode': 200, 'body': f'Successfully {action}d {POLICY_SID} statement'}
              except Exception as e:
                  return {'statusCode': 500, 'body': str(e) }
      Environment:
        Variables:
          BUCKET_NAME: !Ref DestinationS3
          POLICY_SID: 'AllowReplicationWrite'
      MemorySize: 128
      Timeout: 60
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: 'W89'
            reason: "This Lambda does not require VPC"
          - id: 'W92'
            reason: "One Time execution. No need for ReservedConcurrentExecutions"
  DisableRuleSchedule:
    Type: AWS::Events::Rule
    Condition: DeployWriteBlocker
    Properties:
      Description: !Sub "Schedule for disabling replication using cron: ${DisableWriteCronSchedule}"
      ScheduleExpression: !Sub "cron(${DisableWriteCronSchedule})"
      State: ENABLED
      Targets:
        - Arn: !GetAtt BucketWriteBlockerPolicyLambda.Arn
          Id: "DisableReplicationTarget"
          Input: '{"action": "disable"}'

  EnableRuleSchedule:
    Type: AWS::Events::Rule
    Condition: DeployWriteBlocker
    Properties:
      Description: !Sub "Schedule for enabling replication using cron: ${EnableWriteCronSchedule}"
      ScheduleExpression: !Sub "cron(${EnableWriteCronSchedule})"
      State: ENABLED
      Targets:
        - Arn: !GetAtt BucketWriteBlockerPolicyLambda.Arn
          Id: "EnableReplicationTarget"
          Input: '{"action": "enable"}'

  LambdaPermissionDisable:
    Type: AWS::Lambda::Permission
    Condition: DeployWriteBlocker
    Properties:
      FunctionName: !Ref BucketWriteBlockerPolicyLambda
      Action: lambda:InvokeFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt DisableRuleSchedule.Arn

  LambdaPermissionEnable:
    Type: AWS::Lambda::Permission
    Condition: DeployWriteBlocker
    Properties:
      FunctionName: !Ref BucketWriteBlockerPolicyLambda
      Action: lambda:InvokeFunction
      Principal: events.amazonaws.com
      SourceArn: !GetAtt EnableRuleSchedule.Arn

  # CUR2

  ## Deploy Data Export natively via CFN resource in regions that support native CFN
  LocalCUR2viaCFN:
    Type: AWS::BCMDataExports::Export
    Condition: DeployCUR2ViaCFN
    DependsOn:
      - SourceS3BucketPolicy
    Properties:
      Export:
        DataQuery:
          QueryStatement: !If [EnableSCAD, !FindInMap [DataExports, CUR2, SCADQuery], !FindInMap [DataExports, CUR2, DefaultQuery]]
          TableConfigurations:
            COST_AND_USAGE_REPORT:
              TIME_GRANULARITY: !Ref TimeGranularity
              INCLUDE_RESOURCES: "TRUE"
              INCLUDE_MANUAL_DISCOUNT_COMPATIBILITY: "FALSE"
              INCLUDE_SPLIT_COST_ALLOCATION_DATA: !If [EnableSCAD, "TRUE", "FALSE"]
        Description: "CUR 2.0 export for aggregation in CID"
        DestinationConfigurations:
          S3Destination:
            S3Bucket: !Ref SourceS3
            S3Prefix: !Sub "cur2/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name)
            S3Region: !Ref AWS::Region
            S3OutputConfigurations:
              Overwrite: "OVERWRITE_REPORT"
              Format: "PARQUET"
              Compression: "PARQUET"
              OutputType: "CUSTOM"
        Name: !Sub '${ResourcePrefix}-cur2'
        RefreshCadence:
          Frequency: "SYNCHRONOUS"

  # Deploy Data Export via Lambda due to missing CFN resource definition AWS::BCMDataExports::Export outside us-east-1 or cn-northwest-1
  LocalCUR2viaCustomResource:
    Type: Custom::CUR2Creator
    Condition: DeployCUR2ViaLambda
    Properties:
      ServiceToken: !GetAtt CidDataExportCreatorLambda.Arn
      BucketPolicyWait: !Ref SourceS3BucketPolicy
      Export:
        DataQuery:
          QueryStatement: !If [EnableSCAD, !FindInMap [DataExports, CUR2, SCADQuery], !FindInMap [DataExports, CUR2, DefaultQuery]]
          TableConfigurations:
            COST_AND_USAGE_REPORT:
              TIME_GRANULARITY: !Ref TimeGranularity
              INCLUDE_RESOURCES: "TRUE"
              INCLUDE_MANUAL_DISCOUNT_COMPATIBILITY: "FALSE"
              INCLUDE_SPLIT_COST_ALLOCATION_DATA: !If [EnableSCAD, "TRUE", "FALSE"]
        Description: "CUR 2.0 export for aggregation in CID"
        DestinationConfigurations:
          S3Destination:
            S3Bucket: !Ref SourceS3
            S3Prefix: !Sub "cur2/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name)
            S3Region: !Ref AWS::Region
            S3OutputConfigurations:
              Overwrite: "OVERWRITE_REPORT"
              Format: "PARQUET"
              Compression: "PARQUET"
              OutputType: "CUSTOM"
        Name: !Sub '${ResourcePrefix}-cur2'
        RefreshCadence:
          Frequency: "SYNCHRONOUS"

  # FOCUS

  ## Deploy Data Export natively via CFN resource in regions that support native CFN
  LocalFOCUSviaCFN:
    Type: AWS::BCMDataExports::Export
    Condition: DeployFOCUSViaCFN
    DependsOn:
      - SourceS3BucketPolicy
    Properties:
      Export:
        DataQuery:
          QueryStatement: !FindInMap [DataExports, FOCUS, DefaultQuery]
        Description: "FOCUS export for aggregation in CID"
        DestinationConfigurations:
          S3Destination:
            S3Bucket: !Ref SourceS3
            S3Prefix: !Sub "focus/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name)
            S3Region: !Ref AWS::Region
            S3OutputConfigurations:
              Overwrite: "OVERWRITE_REPORT"
              Format: "PARQUET"
              Compression: "PARQUET"
              OutputType: "CUSTOM"
        Name: !Sub '${ResourcePrefix}-focus'
        RefreshCadence:
          Frequency: "SYNCHRONOUS"

  # Deploy Data Export via Lambda due to missing CFN resource definition AWS::BCMDataExports::Export outside us-east-1 or cn-northwest-1
  LocalFOCUSviaCustomResource:
    Type: Custom::CUR2Creator
    Condition: DeployFOCUSViaLambda
    Properties:
      ServiceToken: !GetAtt CidDataExportCreatorLambda.Arn
      BucketPolicyWait: !Ref SourceS3BucketPolicy
      Export:
        DataQuery:
          QueryStatement: !FindInMap [DataExports, FOCUS, DefaultQuery]
        Description: "FOCUS export for aggregation in CID"
        DestinationConfigurations:
          S3Destination:
            S3Bucket: !Ref SourceS3
            S3Prefix: !Sub "focus/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name)
            S3Region: !Ref AWS::Region
            S3OutputConfigurations:
              Overwrite: "OVERWRITE_REPORT"
              Format: "PARQUET"
              Compression: "PARQUET"
              OutputType: "CUSTOM"
        Name: !Sub '${ResourcePrefix}-focus'
        RefreshCadence:
          Frequency: "SYNCHRONOUS"

  # COH

  ## Deploy Data Export natively via CFN resource in regions that support native CFN
  LocalCOHviaCFN:
    Type: AWS::BCMDataExports::Export
    Condition: DeployCOHViaCFN
    DependsOn:
      - SourceS3BucketPolicy
      - CreateServiceLinkedRoleCustomResource
    Properties:
      Export:
        DataQuery:
          QueryStatement: !FindInMap [DataExports, COH, DefaultQuery]
          TableConfigurations:
            COST_OPTIMIZATION_RECOMMENDATIONS:
              FILTER: '{}'
              INCLUDE_ALL_RECOMMENDATIONS: 'TRUE'
        Description: "Cost Optimization Hub Recommendations export for aggregation in CID"
        DestinationConfigurations:
          S3Destination:
            S3Bucket: !Ref SourceS3
            S3Prefix: !Sub "coh/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name)
            S3Region: !Ref AWS::Region
            S3OutputConfigurations:
              Overwrite: "OVERWRITE_REPORT"
              Format: "PARQUET"
              Compression: "PARQUET"
              OutputType: "CUSTOM"
        Name: !Sub '${ResourcePrefix}-coh'
        RefreshCadence:
          Frequency: "SYNCHRONOUS"

  # Deploy Data Export via Lambda due to missing CFN resource definition AWS::BCMDataExports::Export outside us-east-1 or cn-northwest-1
  LocalCOHviaCustomResource:
    Type: Custom::CUR2Creator
    DependsOn:
      - CreateServiceLinkedRoleCustomResource
    Condition: DeployCOHViaLambda
    Properties:
      ServiceToken: !GetAtt CidDataExportCreatorLambda.Arn
      BucketPolicyWait: !Ref SourceS3BucketPolicy
      Export:
        DataQuery:
          QueryStatement: !FindInMap [DataExports, COH, DefaultQuery]
          TableConfigurations:
            COST_OPTIMIZATION_RECOMMENDATIONS:
              FILTER: '{}'
              INCLUDE_ALL_RECOMMENDATIONS: 'TRUE'
        Description: "Cost Optimization Hub Recommendations export for aggregation in CID"
        DestinationConfigurations:
          S3Destination:
            S3Bucket: !Ref SourceS3
            S3Prefix: !Sub "coh/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name)
            S3Region: !Ref AWS::Region
            S3OutputConfigurations:
              Overwrite: "OVERWRITE_REPORT"
              Format: "PARQUET"
              Compression: "PARQUET"
              OutputType: "CUSTOM"
        Name: !Sub '${ResourcePrefix}-coh'
        RefreshCadence:
          Frequency: "SYNCHRONOUS"

  # COH export requires a service linked role to be created BUT this role can be already created. Thus we need to create it via custom resource
  LambdaServiceLinkedRoleExecutionRole:
    Type: 'AWS::IAM::Role'
    Condition: DeployCOHServiceRole
    Properties:
      RoleName: !Sub "${AWS::StackName}-LambdaExecutionRole"
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: !Sub "${AWS::StackName}-LambdaPolicy"
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                  - logs:PutLogEvents
                Resource:
                  - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CreateServiceLinkedRoleFunction"
                  - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CreateServiceLinkedRoleFunction:*"
                  - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CreateServiceLinkedRoleFunction:*:*"
              - Effect: Allow
                Action:
                  - iam:GetRole
                  - iam:CreateServiceLinkedRole
                Resource: !Sub 'arn:${AWS::Partition}:iam::${AWS::AccountId}:role/aws-service-role/bcm-data-exports.amazonaws.com/AWSServiceRoleForBCMDataExports'
              - Effect: Allow
                Action:
                  - cost-optimization-hub:ListEnrollmentStatuses
                Resource: '*' # Cannot restrict this

    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: 'W28'
            reason: "Need an explicit name for reference"
          - id: 'W11'
            reason: "Some COH resources cannot be restricted"


  CreateServiceLinkedRoleFunction:
    Type: 'AWS::Lambda::Function'
    Condition: DeployCOHServiceRole
    Properties:
      FunctionName: !Sub "${ResourcePrefix}-CreateServiceLinkedRoleFunction"
      Handler: index.handler
      MemorySize: 128
      Runtime: python3.12
      Timeout: 90
      Role: !GetAtt LambdaServiceLinkedRoleExecutionRole.Arn
      Code:
        ZipFile: |
          import json
          import logging
          import time

          import boto3  # type: ignore
          from botocore.exceptions import ClientError  # type: ignore
          from botocore.config import Config  # type: ignore
          import cfnresponse  # type: ignore

          CUSTOM_RESOURCE_PHYSICAL_ID = "CreateServiceLinkedRoleBcmDataExports"

          logger = logging.getLogger()
          logger.setLevel(logging.INFO)

          config = Config(retries={"max_attempts": 5, "mode": "standard"})
          iam_client = boto3.client("iam", region_name="us-east-1", config=config)
          coh_client = boto3.client("cost-optimization-hub", region_name="us-east-1", config=config)  # noqa: E501


          def handler(event, context):
              logger.info(f"Event: {json.dumps(event, default=str)}")

              try:
                  if event["RequestType"] == "Delete":
                      cfnresponse.send(event, context, cfnresponse.SUCCESS,
                                      {}, physicalResourceId=CUSTOM_RESOURCE_PHYSICAL_ID)
                      return

                  account_id = context.invoked_function_arn.split(":")[4]
                  validate_coh_enrollment(account_id)
                  create_service_linked_role(
                      service_name="bcm-data-exports.amazonaws.com",
                      description="Service-linked role for bcm-data-exports.amazonaws.com"
                  )

                  cfnresponse.send(event, context, cfnresponse.SUCCESS,
                                  {}, physicalResourceId=CUSTOM_RESOURCE_PHYSICAL_ID)

              except Exception as e:
                  logger.exception(e)
                  cfnresponse.send(event, context, cfnresponse.FAILED,
                                  {}, physicalResourceId=CUSTOM_RESOURCE_PHYSICAL_ID, reason=str(e))


          def validate_coh_enrollment(account_id: str):
              """
              Returns `None` if account is enrolled in Cost Optimization Hub; otherwise, raises `AssertionError`
              """
              logger.info("Checking if account is enrolled in COH...")

              paginator = coh_client.get_paginator("list_enrollment_statuses")
              page_iterator = paginator.paginate(accountId=account_id)
              items = [item
                      for page in page_iterator
                      for item in page["items"]
                      if item["status"] == "Active"]

              if items:
                  logger.info("Account is enrolled in COH")
                  return

              raise AssertionError(
                  "Account is NOT enrolled in COH. Please try again after enabling Cost Optimization Hub."
              )


          def create_service_linked_role(service_name: str, description: str):
              try:
                  logger.info(f"Creating a service-linked role for {service_name}...")
                  role_name = iam_client.create_service_linked_role(
                      AWSServiceName=service_name,
                      Description=description
                  )["Role"]["RoleName"]

                  logger.info(f"Waiting for the service-linked role to be available...")
                  waiter = iam_client.get_waiter("role_exists")
                  waiter.wait(
                      RoleName=role_name,
                      WaiterConfig={"Delay": 1, "MaxAttempts": 30}
                  )
                  time.sleep(10)  # Additional wait time, just in case

                  logger.info(
                      f"Successfully created a service-linked role for {service_name}: {role_name}")
              except ClientError as e:
                  if e.response["Error"]["Code"] == "InvalidInput":
                      logger.info(
                          f"Service-linked role for {service_name} already exists")
                  else:
                      raise
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: 'W89'
            reason: "This Lambda does not require VPC"
          - id: 'W92'
            reason: "One Time execution. No need for ReservedConcurrentExecutions"

  CreateServiceLinkedRoleCustomResource:
    Condition: DeployCOHServiceRole
    Type: 'AWS::CloudFormation::CustomResource'
    Properties:
      ServiceToken: !GetAtt CreateServiceLinkedRoleFunction.Arn
      ServiceTimeout: "90"

  # Carbon Emissions

  ## Deploy Data Export natively via CFN resource in regions that support native CFN
  LocalCarbonViaCFN:
    Type: AWS::BCMDataExports::Export
    Condition: DeployCarbonViaCFN
    DependsOn:
      - SourceS3BucketPolicy
    Properties:
      Export:
        DataQuery:
          QueryStatement: !FindInMap [DataExports, Carbon, DefaultQuery]
        Description: "Carbon emissions export for aggregation in CID"
        DestinationConfigurations:
          S3Destination:
            S3Bucket: !Ref SourceS3
            S3Prefix: !Sub "carbon/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name)
            S3Region: !Ref AWS::Region
            S3OutputConfigurations:
              Overwrite: "OVERWRITE_REPORT"
              Format: "PARQUET"
              Compression: "PARQUET"
              OutputType: "CUSTOM"
        Name: !Sub '${ResourcePrefix}-carbon'
        RefreshCadence:
          Frequency: "SYNCHRONOUS"

  # Deploy Data Export via Lambda due to missing CFN resource definition AWS::BCMDataExports::Export outside us-east-1 or cn-northwest-1
  LocalCarbonViaCustomResource:
    Type: Custom::CUR2Creator
    Condition: DeployCarbonViaLambda
    Properties:
      ServiceToken: !GetAtt CidDataExportCreatorLambda.Arn
      BucketPolicyWait: !Ref SourceS3BucketPolicy
      Export:
        DataQuery:
          QueryStatement: !FindInMap [DataExports, Carbon, DefaultQuery]
        Description: "Carbon emissions export for aggregation in CID"
        DestinationConfigurations:
          S3Destination:
            S3Bucket: !Ref SourceS3
            S3Prefix: !Sub "carbon/${AWS::AccountId}" # Explicitly hardcode name of folder (it will be the same as table name)
            S3Region: !Ref AWS::Region
            S3OutputConfigurations:
              Overwrite: "OVERWRITE_REPORT"
              Format: "PARQUET"
              Compression: "PARQUET"
              OutputType: "CUSTOM"
        Name: !Sub '${ResourcePrefix}-carbon'
        RefreshCadence:
          Frequency: "SYNCHRONOUS"

###########################################################################
# Lambda DataExport Creator: used to create DataExport from outside us-east-1 or cn-northwest-1
###########################################################################

  CidDataExportCreatorLambdaRole:
    Type: AWS::IAM::Role
    Condition: DeployAnyExportViaLambda
    Properties:
      Path: !Sub /${ResourcePrefix}/
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        - PolicyName: "ExecutionDefault"
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
            - Effect: Allow
              Action:
              - logs:CreateLogStream
              - logs:PutLogEvents
              - logs:CreateLogGroup
              Resource:
              - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator"
              - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator:*"
              - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-DataExportCreator:*:*"
        - PolicyName: "AllowDataExports"
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
            - Effect: Allow
              Action:
              - bcm-data-exports:CreateExport
              - bcm-data-exports:UpdateExport
              - bcm-data-exports:DeleteExport
              Resource: !Sub "arn:${AWS::Partition}:bcm-data-exports:*:${AWS::AccountId}:*"
            - Effect: Allow
              Action:
              - bcm-data-exports:ListExports
              Resource: !Sub "arn:${AWS::Partition}:bcm-data-exports:*:${AWS::AccountId}:*"
            - Effect: Allow
              Action:
              - cur:PutReportDefinition  #need this permission for bcm-data-exports to work
              Resource: !Sub "arn:${AWS::Partition}:cur:*:${AWS::AccountId}:*"
            - Effect: Allow
              Action:
              - cost-optimization-hub:GetRecommendation  #need this permission for bcm-data-exports to work
              - cost-optimization-hub:ListRecommendations
              Resource: "arn:aws:cost-optimization-hub:*"
            - Effect: Allow
              Action: sustainability:GetCarbonFootprintSummary # needed ad carbon emissions data export creation time
              Resource: "*"

  CidDataExportCreatorLambda:
    Type: AWS::Lambda::Function
    Condition: DeployAnyExportViaLambda
    Properties:
      Runtime: python3.12
      FunctionName: !Sub ${ResourcePrefix}-DataExportCreator
      Handler: index.lambda_handler
      MemorySize: 128
      Role: !GetAtt CidDataExportCreatorLambdaRole.Arn
      Timeout: 15
      Code:
        ZipFile: |
          import os
          import json
          import uuid

          import boto3
          import cfnresponse

          # DataExports only exist in us-east-1 and cn-northwest-1 regions
          region = 'us-east-1' if not os.environ['AWS_REGION'].startswith('cn-') else 'cn-northwest-1'

          client = boto3.client('bcm-data-exports', region_name=region)

          def lambda_handler(event, context):

            print(json.dumps(event))
            reason = ""

            try:
              export = event['ResourceProperties']['Export']
              old_export = event.get('OldResourceProperties', {}).get('Export')
              if event['RequestType'] == 'Create':
                  res = client.create_export(Export=export)
                  print('created:', json.dumps(res))
              elif event['RequestType'] == 'Update':
                  old_export_name = old_export['Name']
                  old_export_arn = list(client.get_paginator('list_exports').paginate().search(f"Exports[? ExportName=='{old_export_name}']"))[0]['ExportArn']
                  if old_export['Name'] != old_export['Name']:
                      res = client.create_export(Export=export)
                      print('created:', json.dumps(res))
                      try:
                          res = client.delete_export(ExportArn=old_export_arn)
                          print('deleted:', json.dumps(res))
                      except Exception as exc:
                          print('Exception on deletion:', str(exc))
                  elif export['DataQuery'] != old_export['DataQuery']: # Data export do not support update of queries
                      print(export['DataQuery'], '!=', old_export['DataQuery'])
                      res = client.delete_export(ExportArn=old_export_arn)
                      print('deleted:', json.dumps(res))
                      res = client.create_export(Export=export)
                      print('created:', json.dumps(res))
                  else:
                      res = client.update_export(ExportArn=old_export_arn, Export=export)
                      print('updated:', json.dumps(res))
              elif event['RequestType'] == 'Delete':
                  export_name = export['Name']
                  try:
                      export_arn = list(client.get_paginator('list_exports').paginate().search(f"Exports[? ExportName=='{export_name}']"))[0]['ExportArn']
                      res = client.delete_export(ExportArn=export_arn)
                      print('deleted:', json.dumps(res))
                  except client.exceptions.ResourceNotFoundException:
                      print('Not Found. Nothing to delete')
                  except:
                      print('Exception on deletion:', str(exc))
              else:
                  raise Exception("Unknown operation: " + event['RequestType'])

            except Exception as e:
                reason = str(e)
                print(e)
            finally:
                physicalResourceId = event.get('ResourceProperties',{}).get('Export', {}).get('Name', None) or str(uuid.uuid1())
                if reason:
                    print("FAILURE")
                    cfnresponse.send(event, context, cfnresponse.FAILED, {}, physicalResourceId, reason=reason)
                else:
                    print("SUCCESS")
                    cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, physicalResourceId)
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: 'W89'
            reason: "This Lambda does not require VPC"
          - id: 'W92'
            reason: "One Time execution. No need for ReservedConcurrentExecutions"

  CIDDatabase:
    Type: AWS::Glue::Database
    Condition: DeployAnyTable
    Properties:
      DatabaseInput:
        Name: !Join [ '_', !Split [ '-', !Sub '${ResourcePrefix}_data_export' ] ] # replace '-' to '_'
      CatalogId: !Sub "${AWS::AccountId}"

###########################################################################
# CUR2
###########################################################################

  CURTable:  # Initial creation of table. it will be updated by crawler later
    Type: AWS::Glue::Table
    Condition: DeployCUR2Table
    Properties:
      CatalogId: !Sub "${AWS::AccountId}"
      DatabaseName: !Ref CIDDatabase
      TableInput:
        Name: cur2
        Owner: owner
        Retention: 0
        TableType: EXTERNAL_TABLE
        Parameters:
          compressionType: none
          classification: parquet
          UPDATED_BY_CRAWLER: !Ref CURCrawler
        StorageDescriptor:
          BucketColumns: []
          Compressed: false
          Location: !Sub "s3://${DestinationS3}/cur2/"
          NumberOfBuckets: -1
          InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
          OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat
          SerdeInfo:
            Parameters:
              serialization.format: '1'
            SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
          StoredAsSubDirectories: false
          Columns: # All fields required for CID
            - {"Name": "bill_bill_type", "Type": "string" }
            - {"Name": "bill_billing_entity", "Type": "string" }
            - {"Name": "bill_billing_period_end_date", "Type": "timestamp" }
            - {"Name": "bill_billing_period_start_date", "Type": "timestamp" }
            - {"Name": "bill_invoice_id", "Type": "string" }
            - {"Name": "bill_payer_account_id", "Type": "string" }
            - {"Name": "bill_payer_account_name", "Type": "string" }
            - {"Name": "cost_category", "Type": "map<string,string>" }
            - {"Name": "discount", "Type": "map<string,double>" }
            - {"Name": "identity_line_item_id", "Type": "string" }
            - {"Name": "identity_time_interval", "Type": "string" }
            - {"Name": "line_item_availability_zone", "Type": "string" }
            - {"Name": "line_item_legal_entity", "Type": "string" }
            - {"Name": "line_item_line_item_description", "Type": "string" }
            - {"Name": "line_item_line_item_type", "Type": "string" }
            - {"Name": "line_item_operation", "Type": "string" }
            - {"Name": "line_item_product_code", "Type": "string" }
            - {"Name": "line_item_resource_id", "Type": "string" }
            - {"Name": "line_item_unblended_cost", "Type": "double" }
            - {"Name": "line_item_usage_account_id", "Type": "string" }
            - {"Name": "line_item_usage_account_name", "Type": "string" }
            - {"Name": "line_item_usage_amount", "Type": "double" }
            - {"Name": "line_item_usage_end_date", "Type": "timestamp" }
            - {"Name": "line_item_usage_start_date", "Type": "timestamp" }
            - {"Name": "line_item_usage_type", "Type": "string" }
            - {"Name": "pricing_lease_contract_length", "Type": "string" }
            - {"Name": "pricing_offering_class", "Type": "string" }
            - {"Name": "pricing_public_on_demand_cost", "Type": "double" }
            - {"Name": "pricing_purchase_option", "Type": "string" }
            - {"Name": "pricing_term", "Type": "string" }
            - {"Name": "pricing_unit", "Type": "string" }
            - {"Name": "product", "Type": "map<string,string>" }
            - {"Name": "product_from_location", "Type": "string" }
            - {"Name": "product_instance_type", "Type": "string" }
            - {"Name": "product_product_family", "Type": "string" }
            - {"Name": "product_servicecode", "Type": "string" }
            - {"Name": "product_to_location", "Type": "string" }
            - {"Name": "reservation_amortized_upfront_fee_for_billing_period", "Type": "double" }
            - {"Name": "reservation_effective_cost", "Type": "double" }
            - {"Name": "reservation_end_time", "Type": "string" }
            - {"Name": "reservation_reservation_a_r_n", "Type": "string" }
            - {"Name": "reservation_start_time", "Type": "string" }
            - {"Name": "reservation_unused_amortized_upfront_fee_for_billing_period", "Type": "double" }
            - {"Name": "reservation_unused_recurring_fee", "Type": "double" }
            - {"Name": "resource_tags", "Type": "map<string,string>" }
            - {"Name": "savings_plan_amortized_upfront_commitment_for_billing_period", "Type": "double" }
            - {"Name": "savings_plan_end_time", "Type": "string" }
            - {"Name": "savings_plan_offering_type", "Type": "string" }
            - {"Name": "savings_plan_payment_option", "Type": "string" }
            - {"Name": "savings_plan_purchase_term", "Type": "string" }
            - {"Name": "savings_plan_savings_plan_a_r_n", "Type": "string" }
            - {"Name": "savings_plan_savings_plan_effective_cost", "Type": "double" }
            - {"Name": "savings_plan_start_time", "Type": "string" }
            - {"Name": "savings_plan_total_commitment_to_date", "Type": "double" }
            - {"Name": "savings_plan_used_commitment", "Type": "double" }
            - {"Name": "split_line_item_parent_resource_id", "Type": "string" }
            - {"Name": "split_line_item_reserved_usage", "Type": "double" }
            - {"Name": "split_line_item_actual_usage", "Type": "double" }
            - {"Name": "split_line_item_split_usage", "Type": "double" }
            - {"Name": "split_line_item_split_usage_ratio", "Type": "double" }
            - {"Name": "split_line_item_split_cost", "Type": "double" }
            - {"Name": "split_line_item_unused_cost", "Type": "double" }
            - {"Name": "split_line_item_net_split_cost", "Type": "double" }
            - {"Name": "split_line_item_net_unused_cost", "Type": "double" }
            - {"Name": "split_line_item_public_on_demand_split_cost", "Type": "double" }
            - {"Name": "split_line_item_public_on_demand_unused_cost", "Type": "double" }
        PartitionKeys:
          - {"Name": "source_account_id", "Type": "string" }
          - {"Name": "report_name", "Type": "string" }
          - {"Name": "data", "Type": "string" }
          - {"Name": "billing_period", "Type": "string" }


  CURCrawler:
    Type: AWS::Glue::Crawler
    Condition: DeployCUR2Table
    Properties:
      Name:  !Sub "${ResourcePrefix}-DataExportCUR2Crawler"
      Description: A recurring crawler that keeps your CUR table in Athena up-to-date.
      Role: !GetAtt CidDataExportCrawlerRole.Arn
      DatabaseName: !Ref CIDDatabase
      Targets:
        S3Targets:
          - Path: !Sub "s3://${DestinationS3}/cur2/"
            Exclusions:
              - '**.json'
              - '**.yml'
              - '**.sql'
              - '**.csv'
              - '**.csv.metadata'
              - '**.gz'
              - '**.zip'
              - '**/cost_and_usage_data_status/*'
              - 'aws-programmatic-access-test-object'
      SchemaChangePolicy:
        DeleteBehavior: LOG
      RecrawlPolicy:
        RecrawlBehavior: CRAWL_EVERYTHING
      Schedule:
        ScheduleExpression: cron(0 2 * * ? *)
      Configuration: |
        {
          "Version":1.0,
          "Grouping": {
            "TableGroupingPolicy": "CombineCompatibleSchemas"
          },
          "CrawlerOutput":{
            "Tables":{
              "AddOrUpdateBehavior":"MergeNewColumns"
            }
          }
        }

###########################################################################
# FOCUS
###########################################################################

  FOCUSTable:  # Initial creation of table. it will be updated by crawler later
    Type: AWS::Glue::Table
    Condition: DeployFOCUSTable
    Properties:
      CatalogId: !Sub "${AWS::AccountId}"
      DatabaseName: !Ref CIDDatabase
      TableInput:
        Name: focus
        Owner: owner
        Retention: 0
        TableType: EXTERNAL_TABLE
        Parameters:
          compressionType: none
          classification: parquet
          UPDATED_BY_CRAWLER: !Ref FOCUSCrawler
        StorageDescriptor:
          BucketColumns: []
          Compressed: false
          Location: !Sub "s3://${DestinationS3}/focus/"
          NumberOfBuckets: -1
          InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
          OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat
          SerdeInfo:
            Parameters:
              serialization.format: '1'
            SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
          StoredAsSubDirectories: false
          Columns:
            - {"Name": "availabilityzone", "Type": "string" }
            - {"Name": "billedcost", "Type": "double" }
            - {"Name": "billingaccountid", "Type": "string" }
            - {"Name": "billingaccountname", "Type": "string" }
            - {"Name": "billingcurrency", "Type": "string" }
            - {"Name": "billingperiodend", "Type": "timestamp" }
            - {"Name": "billingperiodstart", "Type": "timestamp" }
            - {"Name": "chargecategory", "Type": "string" }
            - {"Name": "chargeclass", "Type": "string" }
            - {"Name": "chargedescription", "Type": "string" }
            - {"Name": "chargefrequency", "Type": "string" }
            - {"Name": "chargeperiodend", "Type": "timestamp" }
            - {"Name": "chargeperiodstart", "Type": "timestamp" }
            - {"Name": "commitmentdiscountcategory", "Type": "string" }
            - {"Name": "commitmentdiscountid", "Type": "string" }
            - {"Name": "commitmentdiscountname", "Type": "string" }
            - {"Name": "commitmentdiscounttype", "Type": "string" }
            - {"Name": "commitmentdiscountstatus", "Type": "string" }
            - {"Name": "consumedquantity", "Type": "double" }
            - {"Name": "consumedunit", "Type": "string" }
            - {"Name": "contractedcost", "Type": "double" }
            - {"Name": "contractedunitprice", "Type": "double" }
            - {"Name": "effectivecost", "Type": "double" }
            - {"Name": "invoiceissuername", "Type": "string" }
            - {"Name": "listcost", "Type": "double" }
            - {"Name": "listunitprice", "Type": "double" }
            - {"Name": "pricingcategory", "Type": "string" }
            - {"Name": "pricingquantity", "Type": "double" }
            - {"Name": "pricingunit", "Type": "string" }
            - {"Name": "providername", "Type": "string" }
            - {"Name": "publishername", "Type": "string" }
            - {"Name": "regionid", "Type": "string" }
            - {"Name": "regionname", "Type": "string" }
            - {"Name": "resourceid", "Type": "string" }
            - {"Name": "resourcename", "Type": "string" }
            - {"Name": "resourcetype", "Type": "string" }
            - {"Name": "servicecategory", "Type": "string" }
            - {"Name": "servicename", "Type": "string" }
            - {"Name": "skuid", "Type": "string" }
            - {"Name": "skupriceid", "Type": "string" }
            - {"Name": "subaccountid", "Type": "string" }
            - {"Name": "subaccountname", "Type": "string" }
            - {"Name": "tags", "Type": "map<string,string>" }
        PartitionKeys:
          - {"Name": "source_account_id", "Type": "string" }
          - {"Name": "report_name", "Type": "string" }
          - {"Name": "data", "Type": "string" }
          - {"Name": "billing_period", "Type": "string" }


  FOCUSCrawler:
    Type: AWS::Glue::Crawler
    Condition: DeployFOCUSTable
    Properties:
      Name:  !Sub "${ResourcePrefix}-DataExportFOCUSCrawler"
      Description: A recurring crawler that keeps your FOCUS table in Athena up-to-date.
      Role: !GetAtt CidDataExportCrawlerRole.Arn
      DatabaseName: !Ref CIDDatabase
      Targets:
        S3Targets:
          - Path: !Sub "s3://${DestinationS3}/focus/"
            Exclusions:
              - '**.json'
              - '**.yml'
              - '**.sql'
              - '**.csv'
              - '**.csv.metadata'
              - '**.gz'
              - '**.zip'
              - '**/cost_and_usage_data_status/*'
              - 'aws-programmatic-access-test-object'
      SchemaChangePolicy:
        DeleteBehavior: LOG
      RecrawlPolicy:
        RecrawlBehavior: CRAWL_EVERYTHING
      Schedule:
        ScheduleExpression: cron(0 2 * * ? *)
      Configuration: |
        {
          "Version":1.0,
          "Grouping": {
            "TableGroupingPolicy": "CombineCompatibleSchemas"
          },
          "CrawlerOutput":{
            "Tables":{
              "AddOrUpdateBehavior":"MergeNewColumns"
            }
          }
        }

###########################################################################
# COH
###########################################################################

  COHTable:  # Initial creation of table. it will be updated by crawler later
    Type: AWS::Glue::Table
    Condition: DeployCOHTable
    Properties:
      CatalogId: !Sub "${AWS::AccountId}"
      DatabaseName: !Ref CIDDatabase
      TableInput:
        Name: coh
        Owner: owner
        Retention: 0
        TableType: EXTERNAL_TABLE
        Parameters:
          compressionType: none
          classification: parquet
          UPDATED_BY_CRAWLER: !Ref COHCrawler
        StorageDescriptor:
          BucketColumns: []
          Compressed: false
          Location: !Sub "s3://${DestinationS3}/coh/"
          NumberOfBuckets: -1
          InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
          OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat
          SerdeInfo:
            Parameters:
              serialization.format: '1'
            SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
          StoredAsSubDirectories: false
          Columns: # All fields required for CID
            - {"Name": "account_id", "Type": "string"}
            - {"Name": "action_type", "Type": "string"}
            - {"Name": "currency_code", "Type": "string"}
            - {"Name": "current_resource_details", "Type": "string"}
            - {"Name": "current_resource_summary", "Type": "string"}
            - {"Name": "current_resource_type", "Type": "string"}
            - {"Name": "estimated_monthly_cost_after_discount", "Type": "double"}
            - {"Name": "estimated_monthly_cost_before_discount", "Type": "double"}
            - {"Name": "estimated_monthly_savings_after_discount", "Type": "double"}
            - {"Name": "estimated_monthly_savings_before_discount", "Type": "double"}
            - {"Name": "estimated_savings_percentage_after_discount", "Type": "double"}
            - {"Name": "estimated_savings_percentage_before_discount", "Type": "double"}
            - {"Name": "implementation_effort", "Type": "string"}
            - {"Name": "last_refresh_timestamp", "Type": "string"}
            - {"Name": "recommendation_id", "Type": "string"}
            - {"Name": "recommendation_lookback_period_in_days", "Type": "int"}
            - {"Name": "recommendation_source", "Type": "string"}
            - {"Name": "recommended_resource_details", "Type": "string"}
            - {"Name": "recommended_resource_summary", "Type": "string"}
            - {"Name": "recommended_resource_type", "Type": "string"}
            - {"Name": "region", "Type": "string"}
            - {"Name": "resource_arn", "Type": "string"}
            - {"Name": "restart_needed", "Type": "boolean"}
            - {"Name": "rollback_possible", "Type": "boolean"}
            - {"Name": "tags", "Type": "map<string,string>"}
        PartitionKeys:
          - {"Name": "source_account_id", "Type": "string" }
          - {"Name": "report_name", "Type": "string" }
          - {"Name": "data", "Type": "string" }
          - {"Name": "date", "Type": "string" }


  COHCrawler:
    Type: AWS::Glue::Crawler
    Condition: DeployCOHTable
    Properties:
      Name:  !Sub "${ResourcePrefix}-DataExportCOHCrawler"
      Description: A recurring crawler that keeps your COH table in Athena up-to-date.
      Role: !GetAtt CidDataExportCrawlerRole.Arn
      DatabaseName: !Ref CIDDatabase
      Targets:
        S3Targets:
          - Path: !Sub "s3://${DestinationS3}/coh/"
            Exclusions:
              - '**.json'
              - '**.yml'
              - '**.sql'
              - '**.csv'
              - '**.csv.metadata'
              - '**.gz'
              - '**.zip'
              - '**/cost_and_usage_data_status/*'
              - 'aws-programmatic-access-test-object'
      SchemaChangePolicy:
        DeleteBehavior: LOG
      RecrawlPolicy:
        RecrawlBehavior: CRAWL_EVERYTHING
      Schedule:
        ScheduleExpression: cron(0 2 * * ? *)
      Configuration: |
        {
          "Version":1.0,
          "Grouping": {
            "TableGroupingPolicy": "CombineCompatibleSchemas"
          },
          "CrawlerOutput":{
            "Tables":{
              "AddOrUpdateBehavior":"MergeNewColumns"
            }
          }
        }

###########################################################################
# Carbon emissions
###########################################################################

  CarbonTable:  # Initial creation of table. it will be updated by crawler later
    Type: AWS::Glue::Table
    Condition: DeployCarbonTable
    Properties:
      CatalogId: !Sub "${AWS::AccountId}"
      DatabaseName: !Ref CIDDatabase
      TableInput:
        Name: carbon
        Owner: owner
        Retention: 0
        TableType: EXTERNAL_TABLE
        Parameters:
          compressionType: none
          classification: parquet
          UPDATED_BY_CRAWLER: !Ref CarbonCrawler
        StorageDescriptor:
          BucketColumns: []
          Compressed: false
          Location: !Sub "s3://${DestinationS3}/carbon/"
          NumberOfBuckets: -1
          InputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
          OutputFormat: org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat
          SerdeInfo:
            Parameters:
              serialization.format: '1'
            SerializationLibrary: org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe
          StoredAsSubDirectories: false
          Columns: # All fields required for CID
            - {"Name": "payer_account_id", "Type": "string"}
            - {"Name": "usage_account_id", "Type": "string"}
            - {"Name": "total_mbm_emissions_value", "Type": "double"}
            - {"Name": "total_mbm_emissions_unit", "Type": "string"}
            - {"Name": "total_lbm_emissions_value", "Type": "double"}
            - {"Name": "total_lbm_emissions_unit", "Type": "string"}
            - {"Name": "model_version", "Type": "string"}
            - {"Name": "product_code", "Type": "string"}
            - {"Name": "usage_period_start", "Type": "timestamp"}
            - {"Name": "usage_period_end", "Type": "timestamp"}
            - {"Name": "last_refresh_timestamp", "Type": "timestamp"}
            - {"Name": "region_code", "Type": "string"}
            - {"Name": "location", "Type": "string"}
        PartitionKeys:
          - {"Name": "source_account_id", "Type": "string" }
          - {"Name": "report_name", "Type": "string" }
          - {"Name": "data", "Type": "string" }
          - {"Name": "carbon_model_version", "Type": "string" }
          - {"Name": "usage_period", "Type": "string" }

  CarbonCrawler:
    Type: AWS::Glue::Crawler
    Condition: DeployCarbonTable
    Properties:
      Name:  !Sub "${ResourcePrefix}-DataExportCarbonCrawler"
      Description: A recurring crawler that keeps your Carbon emissions table in Athena up-to-date.
      Role: !GetAtt CidDataExportCrawlerRole.Arn
      DatabaseName: !Ref CIDDatabase
      Targets:
        S3Targets:
          - Path: !Sub "s3://${DestinationS3}/carbon/"
            Exclusions:
              - '**.json'
              - '**.yml'
              - '**.sql'
              - '**.csv'
              - '**.csv.metadata'
              - '**.gz'
              - '**.zip'
              - '**/cost_and_usage_data_status/*'
              - 'aws-programmatic-access-test-object'
      SchemaChangePolicy:
        DeleteBehavior: LOG
      RecrawlPolicy:
        RecrawlBehavior: CRAWL_EVERYTHING
      Schedule:
        ScheduleExpression: cron(0 2 * * ? *)
      Configuration: |
        {
          "Version":1.0,
          "Grouping": {
            "TableGroupingPolicy": "CombineCompatibleSchemas"
          },
          "CrawlerOutput":{
            "Tables":{
              "AddOrUpdateBehavior":"MergeNewColumns"
            }
          }
        }

  LakeFormationTagsForDatabase:
    Type: AWS::LakeFormation::TagAssociation
    Condition: NeedLakeFormationEnabledDB
    Properties:
      Resource:
        Database:
          CatalogId: !Ref "AWS::AccountId"
          Name: !Ref CIDDatabase
      LFTags:
        - TagKey: !ImportValue cid-LakeFormation-TagKey # you need to install cid-lakeformation-prerequisite.yaml
          TagValues:
            - !ImportValue cid-LakeFormation-TagValue
          CatalogId: !Ref "AWS::AccountId"

  LakeFormationTagsForCUR2:
    Type: AWS::LakeFormation::TagAssociation
    Condition: NeedLakeFormationEnabledCUR2
    Properties:
      Resource:
        Table:
          CatalogId: !Ref "AWS::AccountId"
          DatabaseName: !Ref CIDDatabase
          Name: !Ref CURTable
      LFTags:
        - TagKey: !ImportValue cid-LakeFormation-TagKey  # you need to install cid-lakeformation-prerequisite.yaml
          TagValues:
            - !ImportValue cid-LakeFormation-TagValue
          CatalogId: !Ref "AWS::AccountId"

  LakeFormationTagsForCOH:
    Type: AWS::LakeFormation::TagAssociation
    Condition: NeedLakeFormationEnabledCOH
    Properties:
      Resource:
        Table:
          CatalogId: !Ref "AWS::AccountId"
          DatabaseName: !Ref CIDDatabase
          Name: !Ref COHTable
      LFTags:
        - TagKey: !ImportValue cid-LakeFormation-TagKey  # you need to install cid-lakeformation-prerequisite.yaml
          TagValues:
            - !ImportValue cid-LakeFormation-TagValue
          CatalogId: !Ref "AWS::AccountId"

  LakeFormationTagsForFOCUS:
    Type: AWS::LakeFormation::TagAssociation
    Condition: NeedLakeFormationEnabledFOCUS
    Properties:
      Resource:
        Table:
          CatalogId: !Ref "AWS::AccountId"
          DatabaseName: !Ref CIDDatabase
          Name: !Ref FOCUSTable
      LFTags:
        - TagKey: !ImportValue cid-LakeFormation-TagKey  # you need to install cid-lakeformation-prerequisite.yaml
          TagValues:
            - !ImportValue cid-LakeFormation-TagValue
          CatalogId: !Ref "AWS::AccountId"

  LakeFormationTagsForCarbon:
    Type: AWS::LakeFormation::TagAssociation
    Condition: NeedLakeFormationEnabledCarbon
    Properties:
      Resource:
        Table:
          CatalogId: !Ref "AWS::AccountId"
          DatabaseName: !Ref CIDDatabase
          Name: !Ref CarbonTable
      LFTags:
        - TagKey: !ImportValue cid-LakeFormation-TagKey  # you need to install cid-lakeformation-prerequisite.yaml
          TagValues:
            - !ImportValue cid-LakeFormation-TagValue
          CatalogId: !Ref "AWS::AccountId"

###########################################################################
# Generic Resources
###########################################################################

  CidDataExportCrawlerRole:
    Type: AWS::IAM::Role
    Condition: DeployAnyTable
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - glue.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Path: !Ref RolePath
      Policies:
        - PolicyName: CrawlerPolicy
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - 's3:GetObject'
                Resource: !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}/*'
              - Effect: Allow
                Action:
                  - 's3:ListBucket'
                Resource: !Sub 'arn:${AWS::Partition}:s3:::${DestinationS3}'
              - Effect: Allow
                Action:
                  - glue:GetDatabase
                  - glue:GetDatabases
                  - glue:UpdateDatabase
                  - glue:CreateTable
                  - glue:UpdateTable
                  - glue:GetTable
                  - glue:GetTables
                  - glue:BatchCreatePartition
                  - glue:CreatePartition
                  - glue:DeletePartition
                  - glue:BatchDeletePartition
                  - glue:UpdatePartition
                  - glue:GetPartition
                  - glue:GetPartitions
                  - glue:BatchGetPartition
                  - glue:ImportCatalogToGlue
                Resource:
                  - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog
                  - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/${CIDDatabase}
                  - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/${CIDDatabase}/*
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                Resource:
                  - !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws-glue/crawlers:*'
              - Effect: Allow
                Action:
                  - logs:PutLogEvents
                Resource:
                  - !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws-glue/crawlers:log-stream:*'

###########################################################################
# Analytics: used by CID team to track adoption, by retrieving AWS AccountId
###########################################################################

  CidLambdaAnalyticsRole: #Execution role for the custom resource CidLambdaAnalyticsExecutor
    Type: AWS::IAM::Role
    Properties:
      Path: !Sub /${ResourcePrefix}/
      #RoleName: CID-Analytics
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - sts:AssumeRole
      Policies:
        - PolicyName: "ExecutionDefault"
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
            - Effect: Allow
              Action:
              - logs:CreateLogStream
              - logs:PutLogEvents
              - logs:CreateLogGroup
              Resource:
              - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports"
              - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports:*"
              - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/${ResourcePrefix}-CID-Analytics-DataExports:*:*"

  CidLambdaAnalytics:
    Type: AWS::Lambda::Function
    Properties:
      Runtime: python3.12
      FunctionName: !Sub ${ResourcePrefix}-CID-Analytics-DataExports
      Handler: index.lambda_handler
      MemorySize: 128
      Role: !GetAtt CidLambdaAnalyticsRole.Arn
      Timeout: 15
      Environment:
        Variables:
          API_ENDPOINT: https://okakvoavfg.execute-api.eu-west-1.amazonaws.com/
      Code:
        ZipFile: |
          import os
          import json
          import uuid
          import urllib3

          import boto3
          import cfnresponse

          http = urllib3.PoolManager()
          endpoint = os.environ["API_ENDPOINT"]
          account_id = boto3.client("sts").get_caller_identity()["Account"]

          def execute_request(action, product_id, via_key):
              try:
                  payload = {'dashboard_id': product_id, 'account_id': account_id, via_key: 'CFN'}
                  encoded_data = json.dumps(payload).encode('utf-8')
                  r = http.request(action, endpoint, body=encoded_data, headers={'Content-Type': 'application/json'})
                  if r.status != 200:
                      return f"This will not fail the deployment. There has been an issue logging action {action} for product {product_id} and account {account_id}, server did not respond with a 200 response, status: {r.status}, response data {r.data.decode('utf-8')}. This issue will be ignored"
              except urllib3.exceptions.HTTPError as e:
                  return f"Issue logging action {action} for product {product_id} and account {account_id}, due to a urllib3 exception {str(e)}. This issue will be ignored"
              return None

          def register_deployment(action, products):
              message = f"Successfully logged  {action} for {products}"
              for product_id in products:
                  if action == 'Create':
                      message = execute_request('PUT', product_id, 'created_via')
                  elif action == 'Update':
                      message = execute_request('PATCH', product_id, 'updated_via')
                  elif action == 'Delete':
                      message = execute_request('DELETE', product_id, 'deleted_via')
                  if not message:
                      message = f"Successfully logged {action} for {products} "
              #Do not stop deployment if we're not able to successfully record this deployment, still return true
              return (True, message)

          def lambda_handler(event, context):
              if event['RequestType'] in ['Create', 'Update', 'Delete']:
                  res, reason = register_deployment(event['RequestType'], event['ResourceProperties']['DeploymentType'])
              else:
                  res, reason = False, "Unknown operation: " + event['RequestType']

              physicalResourceId = event.get('ResourceProperties', {}).get('Export', {}).get('Name') or str(uuid.uuid1())
              if res:
                  cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, physicalResourceId )
              else:
                  cfnresponse.send(event, context, cfnresponse.FAILED, {}, physicalResourceId, reason=reason)
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: 'W89'
            reason: "This Lambda does not require VPC"
          - id: 'W92'
            reason: "One Time execution. No need for ReservedConcurrentExecutions"

  CidLambdaAnalyticsExecutorForDataExports:
    Type: Custom::CidLambdaAnalyticsExecutorForDataExports
    Properties:
      ServiceToken: !GetAtt CidLambdaAnalytics.Arn
      DeploymentType:
        - !If [IsDestinationAccount,  "cid-dataexport-destination", !Ref 'AWS::NoValue']
        - !If [IsSourceAccount,       "cid-dataexport-source",      !Ref 'AWS::NoValue']

  DataExportsReadAccess:
    Type: AWS::IAM::ManagedPolicy
    Condition: DeployAnyTable
    Properties:
      ManagedPolicyName: !Sub ${ResourcePrefix}DataExportsReadAccess
      Description: 'Policy for QuickSight to allow DataExports access'
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Sid: AllowGlue
            Effect: Allow
            Action:
              - glue:GetPartition
              - glue:GetPartitions
              - glue:GetDatabase
              - glue:GetDatabases
              - glue:GetTable
              - glue:GetTables
            Resource:
              - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog
              - Fn::Join:
                - ''
                - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/
                  - !Join [ '_', !Split [ '-', !Sub '${ResourcePrefix}_data_export' ] ] # replace '-' to '_'
                  - '/*'
              - Fn::Join:
                - ''
                - - !Sub arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/
                  - !Join [ '_', !Split [ '-', !Sub '${ResourcePrefix}_data_export' ] ] # replace '-' to '_'
          - Sid: AllowListBucket
            Effect: Allow
            Action: s3:ListBucket
            Resource:
              - !Sub arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports
          - Sid: AllowReadBucket
            Effect: Allow
            Action:
              - s3:GetObject
              - s3:GetObjectVersion
            Resource:
              - !Sub arn:${AWS::Partition}:s3:::${ResourcePrefix}-${DestinationAccountId}-data-exports/*
    Metadata:
      cfn_nag:
        rules_to_suppress:
          - id: 'W28'
            reason: "Need an explicit name for reference"

Outputs:
  AggregateBucketName:
    Condition: IsDestinationAccount
    Description: Bucket with aggregate Data Exports
    Value: !Sub ${ResourcePrefix}-${DestinationAccountId}-data-exports
    Export: { Name: 'cid-DataExports-Bucket'}
  Database:
    Condition: DeployAnyTable
    Description: Database for Data Exports
    Value:  !Join [ '_', !Split [ '-', !Sub '${ResourcePrefix}_data_export' ] ] # replace '-' to '_'
    Export: { Name: 'cid-DataExports-Database'}
  LocalAccountBucket:
    Condition: DeploySourceS3
    Description: Local Bucket Name which replicate objects to centralized bucket
    Value: !Sub ${ResourcePrefix}-${AWS::AccountId}-data-local
  ReadAccessPolicyARN:
    Condition: DeployAnyTable
    Description: Policy to allow read access DataExports in S3 and Athena. Attach it to QuickSight role.
    Value: !Ref  DataExportsReadAccess
    Export: { Name: 'cid-DataExports-ReadAccessPolicyARN'}
