Skip to main content

Posts about aws

EFS and ECS

efs.yml

AWSTemplateFormatVersion: "2010-09-09"

Parameters:
  VPC:
    Type: AWS::EC2::VPC::Id
  KeyWord:
    Type: String
  SecurityGroupWeb:
    Type: AWS::EC2::SecurityGroup::Id
  Subnet1:
    #Type: List<AWS::EC2::Subnet::Id>
    Type: AWS::EC2::Subnet::Id
  Subnet2:
    Type: AWS::EC2::Subnet::Id

Resources:
  SecurityGroupEFS:
    Type: 'AWS::EC2::SecurityGroup'
    Properties:
      VpcId: !Ref VPC
      GroupName: !Sub ${KeyWord}_SecurityGroupEFS
      GroupDescription: 'efs-filesystem securitygroup'
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 2049
          ToPort: 2049
          SourceSecurityGroupId: !Ref SecurityGroupWeb
      Tags:
        - Key: Name
          Value: !Sub ${KeyWord}_SecurityGroupEFS

  ElasticFileSystem:
    Type: 'AWS::EFS::FileSystem'
    Properties:
      BackupPolicy:
        Status: DISABLED
      Encrypted: true
      FileSystemTags:
        - Key: Name
          Value: !Sub ${KeyWord}_test_efs
      FileSystemPolicy:
        Version: "2012-10-17"
        Statement:
          - Effect: "Allow"
            Action:
              - "elasticfilesystem:ClientMount"
              - "elasticfilesystem:ClientWrite"
            Principal:
              # AWS: !GetAtt RoleECS.Arn
              AWS: '*'
      PerformanceMode: 'generalPurpose'
      ThroughputMode: 'bursting'

  MountTargetResource1:
    Type: AWS::EFS::MountTarget
    Properties:
      FileSystemId: !Ref ElasticFileSystem
      SubnetId: !Ref Subnet1
      SecurityGroups:
        - !Ref SecurityGroupEFS

  MountTargetResource2:
    Type: AWS::EFS::MountTarget
    Properties:
      FileSystemId: !Ref ElasticFileSystem
      SubnetId: !Ref Subnet2
      SecurityGroups:
        - !Ref SecurityGroupEFS

  AccessPointResource:
    Type: 'AWS::EFS::AccessPoint'
    Properties:
      FileSystemId: !Ref ElasticFileSystem
      PosixUser:
        Uid: "1000"
        Gid: "1000"
      RootDirectory:
        CreationInfo:
          OwnerGid: "1000"
          OwnerUid: "1000"
          Permissions: "0755"
        Path: "/efs/accesspoint1"

Outputs:
  SecurityGroupEFS:
    Value: !Ref SecurityGroupEFS
  ElasticFileSystem:
    Value: !Ref ElasticFileSystem
  AccessPointResource:
    Value: !Ref AccessPointResource

ecs.yml

AWSTemplateFormatVersion: '2010-09-09'
Parameters:
  KeyWord:
    Type: String
  ElasticFileSystem:
    Type: String
  SecurityGroup1:
    Type: String
  Subnet1:
    Type: String
  Subnet2:
    Type: String
  AccessPointResource:
    Type: String

Resources:
  RoleECS:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: 'Allow'
            Principal:
              Service: 'ecs-tasks.amazonaws.com'
            Action: 'sts:AssumeRole'

  ECSCluster:
    Type: 'AWS::ECS::Cluster'
    Properties:
      ClusterName: !Sub ecs-cluster-${KeyWord}
      CapacityProviders:
        - FARGATE
      Tags:
        - Key: Name
          Value: !Sub ${KeyWord}_ecs_cluster

  ECSTaskDefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      ContainerDefinitions:
        - Name: !Sub ${KeyWord}_taskdefinition
          Image: "nginx"
          PortMappings:
            - ContainerPort: 80
          MountPoints:
            - SourceVolume: 'efs-filesystem'
              ContainerPath: '/usr/share/nginx/html'
      RequiresCompatibilities:
        - FARGATE
      Cpu: 256
      Memory: 512
      NetworkMode: awsvpc
      TaskRoleArn: !GetAtt RoleECS.Arn
      Volumes:
        - Name: 'efs-filesystem'
          EFSVolumeConfiguration:
            AuthorizationConfig:
              AccessPointId: !Ref AccessPointResource
              IAM: 'ENABLED'
            FilesystemId: !Ref ElasticFileSystem
            TransitEncryption: 'ENABLED'
      Tags:
        - Key: Name
          Value: !Sub ${KeyWord}_ecs_taskdefinition

  ECSService:
    Type: AWS::ECS::Service
    Properties:
      Cluster: !Ref ECSCluster
      DesiredCount: 0
      TaskDefinition: !Ref ECSTaskDefinition
      LaunchType: FARGATE
      NetworkConfiguration:
        AwsvpcConfiguration:
          AssignPublicIp: ENABLED
          SecurityGroups:
            - !Ref SecurityGroup1
          Subnets:
            - !Ref Subnet1
            - !Ref Subnet2
      Tags:
        - Key: Name
          Value: !Sub ${KeyWord}_ecs_service

Outputs:
  RoleECS:
    Value: !Ref RoleECS
  ECSCluster:
    Value: !Ref ECSCluster
  ECSServiceName:
    Value: !GetAtt ECSService.Name

parent_efs.yml

AWSTemplateFormatVersion: "2010-09-09"

Parameters:
  KeyWord:
    Default: nestedstacktest
    Type: String

Resources:
  EFS:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: efs.yml
      Parameters:
        KeyWord : !Ref KeyWord
        VPC: !ImportValue mystack-Vpc
        Subnet1: !ImportValue mystack-PublicSubnet1
        Subnet2: !ImportValue mystack-PublicSubnet2
        SecurityGroupWeb: !ImportValue mystack-SecurityGroupPub

  ECS:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: ecs.yml
      Parameters:
        KeyWord : !Ref KeyWord
        Subnet1: !ImportValue mystack-PublicSubnet1
        Subnet2: !ImportValue mystack-PublicSubnet2
        SecurityGroup1: !ImportValue mystack-SecurityGroupPub
        AccessPointResource: !GetAtt EFS.Outputs.AccessPointResource
        ElasticFileSystem: !GetAtt EFS.Outputs.ElasticFileSystem

Outputs:
  SecurityGroupEFS:
    Value: !GetAtt EFS.Outputs.SecurityGroupEFS
  ElasticFileSystem:
    Value: !GetAtt EFS.Outputs.ElasticFileSystem
  RoleECS:
    Value: !GetAtt ECS.Outputs.RoleECS
  ECSCluster:
    Value: !GetAtt ECS.Outputs.ECSCluster
  ECSServiceName:
    Value: !GetAtt ECS.Outputs.ECSServiceName

Makefile.efs

template_file=parent_efs.yml
package_file=parent_efs_packaged.yml
bucket_name=mybucketname
stack_name=testefs

validate:
    aws cloudformation validate-template --template-body file://efs.yml
    aws cloudformation validate-template --template-body file://ecs.yml
    aws cloudformation validate-template --template-body file://parent_efs.yml

$(package_file):
    aws cloudformation package --template-file $(template_file) --s3-bucket $(bucket_name) --output-template-file $(package_file)

package: $(package_file)

build: $(package_file)
    aws cloudformation deploy --template-file $(package_file) --stack-name $(stack_name) --tags Name=testnestedstack --capabilities CAPABILITY_IAM

events:
    aws cloudformation describe-stack-events --stack-name $(stack_name)

describe:
    aws cloudformation describe-stacks --stack-name $(stack_name)

delete:
    aws cloudformation delete-stack --stack-name $(stack_name)

clean:
    rm $(package_file)

confirm filesystem and acccess points

$ aws efs describe-file-systems
$ aws efs describe-access-points --file-system-id fs-xxxx

install efs-utils on debian EC2 instance

for detail see https://docs.aws.amazon.com/ja_jp/efs/latest/ug/installing-amazon-efs-utils.html

$ sudo apt update -y && sudo apt install -y git binutils
$ git clone https://github.com/aws/efs-utils
$ cd efs-utils
$ ./build-deb.sh
$ sudo apt install ./build/amazon-efs-utils-*_all.deb

mount and test

$ sudo mount -t efs -o tls,accesspoint=fsap-xxxx fs-xxxx:/ /mnt
or sudo mount -t efs -o tls fs-xxxx:/ /mnt

$ echo hellow world /mnt/index.html

update desired count using AWS CLI

$ aws ecs list-clusters
$ aws ecs describe-clusters --cluster xxxx
$ aws ecs list-services --cluster xxxx
$ aws ecs describe-services --cluster xxxx --services testefs-ECS-xxxx
$ aws ecs update-service --cluster xxxx --service testefs-ECS-xxxx  --desired-count 1

confirm Public IP address

$ aws ecs list-tasks --cluster xxxx
$ aws ecs describe-tasks --cluster xxxx --task xxxx
$ aws ec2 describe-network-interfaces --network-interface-ids eni-xxxx

EC2 spot instance

sample launch template In some instance type and OS, we are able to stop or hybernate the EC2 spot instances, and restart or resume them when we want to do so. While stop or hybernate the instances, we must pay for EBS and Elastic IP etc...

  MyLaunchTemplate:
    Type: AWS::EC2::LaunchTemplate
    Properties:
      LaunchTemplateName: spot-instance-template
      LaunchTemplateData:
        InstanceType: "t3.nano"
        CreditSpecification:
          CpuCredits: "standard"
        InstanceInitiatedShutdownBehavior: stop
        KeyName: "mykeyname"
        NetworkInterfaces:
          - DeviceIndex: "0"
            AssociatePublicIpAddress: "true"
            Groups:
              - Ref: mysecuritygroup
            SubnetId:
              Ref: mysubnet
        InstanceMarketOptions:
          MarketType: spot
          SpotOptions:
            InstanceInterruptionBehavior: stop
            SpotInstanceType: persistent
        BlockDeviceMappings:
          - DeviceName: /dev/xvda
            Ebs:
              DeleteOnTermination: true
              VolumeType: gp3
              VolumeSize: 8

after create or update the stack, confirm the launch template you've just created

aws ec2 describe-launch-templates
aws ec2 describe-launch-template-versions --launch-template-id lt-xxxx --versions x

create spot instance with the launch template. this example override image id, instance type.

$ aws ec2 run-instances --image-id ami-xxxx --instance-type t2.medium --launch-template 'LaunchTemplateId=lt-xxxx,Version=x' --dry-run
$ aws ec2 run-instances --image-id ami-xxxx --instance-type t2.medium --launch-template 'LaunchTemplateId=lt-xxxx,Version=x'

confirm spot instance

$ aws ec2 describe-spot-instance-requests --query 'SpotInstanceRequests[].{"SpotInstanceRequestId":SpotInstanceRequestId, "InstanceId":InstanceId, "Status":Status, "State":State}'  
4 aws ec2 describe-instances --filter 'Name=spot-instance-request-id,Values=sir-xxxx' --query 'Reservations[].Instances[].{"State":State, "InstanceId":InstanceId}'

temporary stop instance and restart it

$ aws ec2 stop-instances --instance-id i-xxxx
$ aws ec2 start-instances --instance-id i-xxxx

cancel spot instance request and terminate spot instance

$ aws ec2 cancel-spot-instance-requests --spot-instance-request-ids sir-xxxx
$ aws ec2 terminate-instances --instance-id i-xxxx

FSx for Windows

cloudformation template microsoftad.yml for AWS managed AD

AWSTemplateFormatVersion: '2010-09-09'

Parameters:
  KeyWord:
    Type: String
  DirectoryName:
    Type: String
  Subnet1:
    Type: AWS::EC2::Subnet::Id
  Subnet2:
    Type: AWS::EC2::Subnet::Id
  VPC:
    Type: AWS::EC2::VPC::Id

Resources:
  MicrosoftAD: 
    Type: AWS::DirectoryService::MicrosoftAD
    Properties: 
      Name: !Ref DirectoryName
      Edition: Standard
      Password: '{{resolve:ssm-secure:test-directory-password:1}}'
      VpcSettings: 
        SubnetIds: 
          - !Ref Subnet1
          - !Ref Subnet2
        VpcId: !Ref VPC

Outputs:
  MicrosoftAD:
    Value: !Ref MicrosoftAD
  MicrosoftADDns:
    Value: !Join
      - ','
      - !GetAtt MicrosoftAD.DnsIpAddresses

cloudformation template fsx.yml for FSx for Windows

AWSTemplateFormatVersion: '2010-09-09'

Parameters:
  KeyWord:
    Type: String
  MicrosoftAD:
    Type: String
  VPC:
    Type: AWS::EC2::VPC::Id
  Subnet1:
    Type: AWS::EC2::Subnet::Id
  Subnet2:
    Type: AWS::EC2::Subnet::Id
  SecurityGroupWeb:
    Type: AWS::EC2::SecurityGroup::Id

Resources:
  SecurityGroupFSx:
    Type: 'AWS::EC2::SecurityGroup'
    Properties:
      VpcId: !Ref VPC
      GroupName: !Sub ${KeyWord}_SecurityGroupFSx
      GroupDescription: 'FSx for windows securitygroup'
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: 445
          ToPort: 445
          SourceSecurityGroupId: !Ref SecurityGroupWeb
        - IpProtocol: tcp
          FromPort: 5985
          ToPort: 5985
          SourceSecurityGroupId: !Ref SecurityGroupWeb
      Tags:
        - Key: Name
          Value: !Sub ${KeyWord}_SecurityGroupFSx

  FSxForWindows:
    Type: 'AWS::FSx::FileSystem'
    Properties:
      FileSystemType: WINDOWS
      StorageCapacity: 32
      StorageType: SSD
      SubnetIds:
        - !Ref Subnet1
        - !Ref Subnet2
      SecurityGroupIds:
        - !Ref SecurityGroupFSx
      Tags:
        - Key: Name
          Value: !Sub ${KeyWord}_FSx_for_windows
      WindowsConfiguration:
        ActiveDirectoryId: !Ref MicrosoftAD
        DeploymentType: MULTI_AZ_1
        PreferredSubnetId: !Ref Subnet1
        ThroughputCapacity: 8

Outputs:
  FSxForWindows:
    Value: !Ref FSxForWindows
  SecurityGroupFSx:
    Value: !Ref SecurityGroupFSx
  FSxForWindowsDNSName:
    Value: !GetAtt FSxForWindows.DNSName

cloudformation parent template parent_fsx.yml

AWSTemplateFormatVersion: "2010-09-09"

Parameters:
  KeyWord:
    Default: nestedstacktest
    Type: String
  DirectoryName:
    Default: mydirectory.example.com
    Type: String

Resources:
  MicrosoftAD:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: microsoftad.yml
      Parameters:
        KeyWord : !Ref KeyWord
        VPC: !ImportValue mystack-Vpc
        Subnet1: !ImportValue mystack-PublicSubnet1
        Subnet2: !ImportValue mystack-PublicSubnet2
        DirectoryName: !Ref DirectoryName

  FSx:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: fsx.yml
      Parameters:
        KeyWord : !Ref KeyWord
        VPC: !ImportValue mystack-Vpc
        Subnet1: !ImportValue mystack-PublicSubnet1
        Subnet2: !ImportValue mystack-PublicSubnet2
        SecurityGroupWeb: !ImportValue mystack-SecurityGroupPub
        MicrosoftAD: !GetAtt MicrosoftAD.Outputs.MicrosoftAD

Outputs:
  MicrosoftAD:
    Value: !GetAtt MicrosoftAD.Outputs.MicrosoftAD
  MicrosoftADDns:
    Value: !GetAtt MicrosoftAD.Outputs.MicrosoftADDns
  SecurityGroupFSx:
    Value: !GetAtt FSx.Outputs.SecurityGroupFSx
  FSxForWindows:
    Value: !GetAtt FSx.Outputs.FSxForWindows
  FSxForWindowsDNSName:
    Value: !GetAtt FSx.Outputs.FSxForWindowsDNSName

Makefile

template_file=parent_fsx.yml
package_file=parent_fsx_packaged.yml
stack_name=testfsx
bucket_name=mybucketname

validate:
    aws cloudformation validate-template --template-body file://microsoftad.yml
    aws cloudformation validate-template --template-body file://fsx.yml
    aws cloudformation validate-template --template-body file://parent_fsx.yml

$(package_file):
    aws cloudformation package --template-file $(template_file) --s3-bucket $(bucket_name) --output-template-file $(package_file)

package: $(package_file)

build: $(package_file)
    aws cloudformation deploy --template-file $(package_file) --stack-name $(stack_name) --tags Name=testnestedstack --capabilities CAPABILITY_IAM

events:
    aws cloudformation describe-stack-events --stack-name $(stack_name)

describe:
    aws cloudformation describe-stacks --stack-name $(stack_name)

delete:
    aws cloudformation delete-stack --stack-name $(stack_name)

clean:
    rm $(package_file)

mount FSx for Windows on ubuntu server

we can check address for mount point ip address

$ dig <access point dns name> +short @<dns of your directory>

check whether you can mount your file system

$ sudo apt update && sudo apt install -y cifs-utils
$ sudo mount -t cifs -o vers=3.0,sec=ntlmsspi,user=<your username>@<your domain> //10.0.0.xxx/share /mountpoint

create credential file and test it

$ sudo chmod 600 /root/smb.cred
$ sudo cat /root/smb.cred
username=<your username>
password=<your password>
domain=<your domain>
sudo mount -t cifs -o vers=3.0,sec=ntlmsspi,credentials=/root/smb.cred //10.0.0.xxx/share /mountpoint

write fstab entry and test it

$ sudo grep /mnt /etc/fstab
//10.0.0.xxx/share      /mnt    cifs    vers=3.0,sec=ntlmsspi,credentials=/root/smb.cred        0 0      
$ sudo mount /mountpoint
$ df /mountpoint
$ sudo umount /mountpoint

SSM Automation

auto stop EC2 instances using AWS System Manager and Cloudwath Events

auto stop ec2 instances with instance id

stop specified ec2 instances

jq . parameters.json 
[
  {
    "ParameterKey": "KeyWord",
    "ParameterValue": "SomeKeyword"
  },
  {
    "ParameterKey": "InstanceIds",
    "ParameterValue": "[¥"i-xxxx¥",¥"i-yyyy¥"]"
  }
]
  RoleAutoStop:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          -  Effect: Allow
             Principal:
               Service: events.amazonaws.com
             Action: sts:AssumeRole
      RoleName: !Sub ${KeyWord}_Role_AutoStop
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonSSMAutomationRole
      Tags:
        - Key: Name
          Value: !Sub ${KeyWord}_Role_AutoStop

  EvnetRuleAutoStop:
    Type: AWS::Events::Rule
    Properties:
      Description: Event rule to stop instances automatically
      Name: !Sub ${KeyWord}-AutoStopInstances
      ScheduleExpression: cron(0 11 * * ? *)
      #RoleArn: !GetAtt  RoleAutoStop.Arn
      State: ENABLED
      Targets:
        - Arn: 'arn:aws:ssm:ap-northeast-1::automation-definition/AWS-StopEC2Instance:$DEFAULT'
          Id: StopEc2
          RoleArn: !GetAtt RoleAutoStop.Arn
          Input: !Sub '{"InstanceId": ${InstanceIds}}'

auto stop ec2 instances with tag

stop ec2 instances with StopTime tag

   AllowTagGetResources:
    Type: AWS::IAM::Policy
    Properties: 
      PolicyName: !Sub ${KeyWord}_allow_tag_getresources
      PolicyDocument:
        Version: "2012-10-17"
        Statement:
          - Effect: Allow
            Action: 
              - tag:GetResources
            Resource: '*'
      Roles:
        - !Ref RoleAutoStop

  StopEC2InstancesWithTag:
    Type: AWS::SSM::Document
    Properties:
      DocumentFormat: YAML
      DocumentType: Automation
      Tags: 
        - Key: Name
          Value: !Sub ${KeyWord}_StopEC2InstancesWithTag
      Content:
        description: StopEC2Instances Using Tags:StopTime
        schemaVersion: "0.3"
        assumeRole: "{{ AutomationAssumeRole }}"
        parameters:
          StopTime:
            type: String
            default: 6pm
            description: (Required) 6pm,7pm,8pm
            allowedValues:
              - 6pm
              - 7pm
              - 8pm
          AutomationAssumeRole:
            type: String
            description: (Optional) The ARN of the role that allows Automation to perform the actions on your behalf.
            default: ""
        mainSteps:
          - name: StopEC2Instances
            action: aws:executeAwsApi
            inputs:
              Service: ssm
              Api: StartAutomationExecution
              DocumentName: AWS-StopEC2Instance
              TargetParameterName: "InstanceId"
              Targets:
                - Key: tag:StopTime
                  Values:
                    - "{{ StopTime }}"

  EvnetRuleAutoStopWithTag:
    Type: AWS::Events::Rule
    Properties:
      Description: Event rule to stop instances automatically with tag
      Name: !Sub ${KeyWord}-AutoStopInstances-with-tag
      ScheduleExpression: cron(0 10 * * ? *)
      State: ENABLED
      Targets:
        - Arn: !Sub arn:aws:ssm:ap-northeast-1::automation-definition/${StopEC2InstancesWithTag}:$DEFAULT
          Id: StopEc2
          RoleArn: !GetAtt RoleAutoStop.Arn
          Input: !Sub '{"StopTime": ["7pm"]}'

acm

request certificate

This sample will create certificate in us-east-1 region.

myregion=us-east-1
myfqdn=www.example.com
aws acm list-certificates --region ${myregion}
aws acm request-certificate --region ${myregion} --domain-name ${myfqdn} --validation-method DNS
CertificateArn=$(aws acm list-certificates --region ${myregion} --query 'CertificateSummaryList[?DomainName == `'${myfqdn}'`].CertificateArn' --output text)
aws acm describe-certificate --region ${myregion} --certificate-arn ${CertificateArn}
aws acm describe-certificate --region ${myregion} --certificate-arn ${CertificateArn} --query 'Certificate.DomainValidationOptions[].ResourceRecord'

add Route53 entry

myodmain=exapmle.com
id=$(aws route53 list-hosted-zones --query 'HostedZones[?Name == `'${mydomain}.'`].Id' --output text)
aws route53 get-hosted-zone --id ${id}
aws route53 list-resource-record-sets --hosted-zone-id ${id}

create change batch

$ jq . change-batch.json 
{
  "Changes": [
    {
      "Action": "CREATE",
      "ResourceRecordSet": {
        "Name": "xxxxxx.example.com.",
        "Type": "CNAME",
        "TTL": 3600,
        "ResourceRecords": [
          {
            "Value": "xxxx.acm-validations.aws."
          }
        ]
      }
    }
  ]
}
aws route53 change-resource-record-sets --hosted-zone-id ${id} --change-batch file://change-batch.json
aws route53 get-change --id /change/xxxxxxxx

confirm the result

aws acm describe-certificate --region ${myregion} --certificate-arn ${CertificateArn}
aws acm get-certificate --region ${myregion} --certificate-arn ${CertificateArn}
aws acm get-certificate --region ${myregion} --certificate-arn ${CertificateArn} --query 'Certificate' --output text | openssl x509 -noout -text

nested cloudformation stack

create sub stack vpc.yml

$ cat vpc.yml 
AWSTemplateFormatVersion: '2010-09-09'

Parameters:
  CidrBlock:
    Type: String
    Default: "10.0.0.0/16"
  NameTag :
    Type: String
    Default: "nested stack"

Resources:
  VPC:

    Properties:
      CidrBlock: !Ref CidrBlock
      Tags:
      - Key: Name
        Value: !Ref NameTag

  InternetGateway:
    Type: AWS::EC2::InternetGateway
    Properties:
      Tags:
      - Key: Name
        Value: !Ref NameTag

  AttachGateway:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      VpcId: 
        Ref: VPC
      InternetGatewayId:
        Ref: InternetGateway

Outputs:
  VpcId: 
    Value: !Ref VPC
  IgwId:
    Value: !Ref InternetGateway
aws cloudformation validate-template --template-body file://vpc.yml

create another sub stack subnet.yml

$ cat subnet.yml                                                                                                                                                [5/1860]
AWSTemplateFormatVersion: '2010-09-09'

Parameters:
  NameTag:
    Type: String
    Default: "subnetnesting"
  VPC:
    Type: AWS::EC2::VPC::Id
  CidrBlock1:
    Type: String
    Default: "10.0.0.0/24"
  CidrBlock2:
    Type: String
    Default: "10.0.1.0/24"

Resources:
  Subnet1:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref VPC
      CidrBlock: !Ref CidrBlock1
      AvailabilityZone:
         Fn::Select:
         - '0'
         - Fn::GetAZs:
             Ref: AWS::Region

  Subnet2:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref VPC
      CidrBlock: !Ref CidrBlock2
      AvailabilityZone:
         Fn::Select:
         - '1'
         - Fn::GetAZs:
             Ref: AWS::Region

Outputs:
  SubnetId1:
    Value: !Ref Subnet1
  SubnetAz1:
    Value: !GetAtt Subnet1.AvailabilityZone
  SubnetId2:
    Value: !Ref Subnet2
  SubnetAz2:
    Value: !GetAtt Subnet2.AvailabilityZone
aws cloudformation validate-template --template-body file://subnet.yml

create parent stack parent.yml

$ cat parent.yml 
AWSTemplateFormatVersion: "2010-09-09"

Parameters:
  NameTag:
    Default: nested stack test
    Type: String
  CidrBlock:
    Default: 10.1.0.0/16
    Type: String
  CidrBlock1:
    Default: 10.1.0.0/24
    Type: String
  CidrBlock2:
    Default: 10.1.1.0/24
    Type: String

Resources:
  VPC:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: vpc.yml
      Parameters:
        NameTag : !Ref NameTag
        CidrBlock: !Ref CidrBlock

  SUBNET:
    Type: AWS::CloudFormation::Stack
    Properties:
      TemplateURL: subnet.yml
      Parameters:
        NameTag : !Ref NameTag
        VPC: !GetAtt VPC.Outputs.VpcId
        CidrBlock1: !Ref CidrBlock1
        CidrBlock2: !Ref CidrBlock2

Outputs:
  VpcId:
    Value: !GetAtt VPC.Outputs.VpcId
  SubnetId1:
    Value: !GetAtt SUBNET.Outputs.SubnetId1
  SubnetId2:
    Value: !GetAtt SUBNET.Outputs.SubnetId2
aws cloudformation validate-template --template-body file://parent.yml

create or update stack

aws cloudformation package --template-file parent.yml --s3-bucket your-s3bucket-name --output-template-file parent_packaged.yml
aws cloudformation deploy --template-file parent_packaged.yml --stack-name nestedstack --tags Name=testnestedstack

confirm

$ aws cloudformation describe-stacks --query 'Stacks[?Tags[?Key == `Name` && Value == `testnestedstack`]]' | less

$ aws cloudformation describe-stacks --query 'Stacks[?Tags[?Key == `Name` && Value == `testnestedstack`]].StackName' | jq -r .[] | while read stackname; do
> aws cloudformation describe-stack-events --stack-name ${stackname}
> done | less

    $ aws cloudformation describe-stacks --query 'Stacks[?Tags[?Key == `Name` && Value == `testnestedstack`]].StackName' | jq -r .[] | while read stackname; do
> aws cloudformation describe-stack-resources --stack-name ${stackname}
> done | less

delete stack

aws cloudformation delete-stack --stack-name nestedstack

cognito user pool

create user pool

aws cognito-idp create-user-pool --pool-name testpool1 --user-pool-tags 'key=Name,Value=testpool1' --admin-create-user-config 'AllowAdminCreateUserOnly=true' --account-recovery-setting 'RecoveryMechanisms=[{Priority=1,Name=admin_only}]'
aws cognito-idp list-user-pools --max-results 10
aws cognito-idp describe-user-pool --user-pool-id ap-northeast-1_xxxxxxxxx

remove user pool

aws cognito-idp delete-user-pool --user-pool-id ap-northeast-1_xxxxxxxxx

create user and set user password

aws cognito-idp list-users --user-pool-id ap-northeast-1_xxxxxxxxx
aws cognito-idp admin-create-user --user-pool-id ap-northeast-1_xxxxxxxxx --username testuser001 --temporary-password temporary_password
aws cognito-idp admin-set-user-password --user-pool-id ap-northeast-1_xxxxxxxxx --username testuser001 --password parmanent_password --permanent

create role before import csv

create policy json document

$ jq . AllowCognitoCloudwatchLogs.policy
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "logs:CreateLogGroup",
        "logs:CreateLogStream",
        "logs:DescribeLogStreams",
        "logs:PutLogEvents"
      ],
      "Resource": [
        "arn:aws:logs:ap-northeast-1:xxxxxxxxxxxx:log-group:/aws/cognito/*"
      ]
    }
  ]
}

create policy

aws iam create-policy --policy-name AllowCognitoCloudwatchLogs --policy-document file://AllowCognitoCloudwatchLogs.policy
aws iam list-policies --query 'Policies[?PolicyName==`AllowCognitoCloudwatchLogs`]'
aws iam delete-policy --policy-arn arn:aws:iam::xxxxxxxxxxxx:policy/AllowCognitoCloudwatchLogs

create assume role policy document

$ jq . assumepolicy.json
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Principal": {
        "Service": "cognito-idp.amazonaws.com"
      },
      "Action": "sts:AssumeRole"
    }
  ]
}

create role

aws iam create-role --role-name Import-Cognito-Userpool --assume-role-policy-document file://assumepolicy.json
aws iam list-roles --query 'Roles[?RoleName==`Import-Cognito-Userpool`]'
aws iam attach-role-policy --role-name Import-Cognito-Userpool --policy-arn arn:aws:iam::xxxxxxxxxxxx:policy/AllowCognitoCloudwatchLogs

import csv to user pool

create csv

name,given_name,family_name,middle_name,nickname,preferred_username,profile,picture,website,email,email_verified,gender,birthdate,zoneinfo,locale,phone_number,phone_number_verified,address,updated_at,cognito:mfa_enabled,cognito:username
,,,,,,,,,dummy@example.com,true,,,,,,false,,,false,import001

import it

aws cognito-idp create-user-import-job --user-pool-id ap-northeast-1_xxxxxxxxx --job-name import_job --cloud-watch-logs-role-arn arn:aws:iam::xxxxxxxxxxxx:role/service-role/Cognito-UserImport-Role
curl -v -T "PATH_TO_CSV_FILE" -H "x-amz-server-side-encryption:aws:kms" "PRE_SIGNED_URL"
aws cognito-idp describe-user-import-job --user-pool-id ap-northeast-1_xxxxxxxxx --job-id import-xxxxxxxxxx
aws cognito-idp start-user-import-job --user-pool-id ap-northeast-1_xxxxxxxxx --job-id import-xxxxxxxxxx

remove unnecessary attributes and set user password

aws cognito-idp list-users --user-pool-id ap-northeast-1_xxxxxxxxx
aws cognito-idp admin-delete-user-attributes --user-pool-id ap-northeast-1_xxxxxxxxx --username import001 --user-attribute-names 'email'
aws cognito-idp admin-set-user-password --user-pool-id ap-northeast-1_xxxxxxxxx --username import001 --password permanent_password --permanent

disable / enable / delete user

aws cognito-idp list-users --user-pool-id ap-northeast-1_xxxxxxxxx --filter 'username="import001"'
aws cognito-idp admin-disable-user --user-pool-id ap-northeast-1_xxxxxxxxx --username import001
aws cognito-idp admin-enable-user  --user-pool-id ap-northeast-1_xxxxxxxxx --username import001
aws cognito-idp admin-delete-user  --user-pool-id ap-northeast-1_xxxxxxxxx --username import001

user pool client

aws cognito-idp list-user-pool-clients --user-pool-id ap-northeast-1_xxxxxxxxx
aws cognito-idp create-user-pool-client --user-pool-id ap-northeast-1_xxxxxxxxx --client-name test-user-pool-client
aws cognito-idp describe-user-pool-client --user-pool-id ap-northeast-1_xxxxxxxxx --client-id xxxxxxxxxxxxxxxxxxxxxxxxx
aws cognito-idp delete-user-pool-client --user-pool-id ap-northeast-1_xxxxxxxxx --client-id xxxxxxxxxxxxxxxxxxxxxxxxx

example

https://ashura156.hatenablog.com/entry/20180309/1520586674

advanced cloudformation

create an initial cloudformation stack

aws cloudformation validate-template --template-body file://template-000.yml
aws cloudformation create-stack --stack-name mystack  --template-body file://mystack-000.yml

confirm the result

aws cloudformation describe-stacks --stack-name mystack 
aws cloudformation describe-stack-resources --stack-name mystack 
aws cloudformation describe-stack-events --stack-name mystack

change set

create change set after editing template file

aws cloudformation create-change-set --stack-name mystack --template-body file://mystack-001.yml --change-set-name mystack-001 --description 'create new Internet Gateway'
aws cloudformation list-change-sets --stack-name mystack 
aws cloudformation describe-change-set --stack-name mystack --change-set-name mystack-001

execute the change set

aws cloudformation execute-change-set --stack-name mystack --change-set-name mystack-001
aws cloudformation list-change-sets --stack-name mystack

drift

detect stack drift after manual operation

aws cloudformation detect-stack-drift --stack-name mystack 
aws cloudformation describe-stack-resource-drifts --stack-name mystack

create change set after editing template file to fit to current resource

aws cloudformation create-change-set --stack-name mystack --template-body file://mystack-002.yml --change-set-name mystack-002 --description 'reflect manual operation'
aws cloudformation list-change-sets --stack-name mystack 
aws cloudformation describe-change-set --stack-name mystack --change-set-name mystack-002

execute the change set

aws cloudformation execute-change-set --stack-name mystack --change-set-name mystack-002

remove change set when the status is false

aws cloudformation delete-change-set --stack-name mystack --change-set-name mystack-002

import

create template file

$ diff -u template.yml.orig template.yml
+Parameters:
+  ImageId:
+    Type: AWS::EC2::Image::Id
+  InstanceType:
+    Type: String
+  KeyName:
+    Type: AWS::EC2::KeyPair::KeyName
+  SecurityGroupId:
+    Type: AWS::EC2::SecurityGroup::Id
+  SubnetId:
+    Type: AWS::EC2::Subnet::Id
+

+  ## EC2 instance
+  EC2Instance1:
+    Type: AWS::EC2::Instance
+    DeletionPolicy: Retain
+    Properties: 
+      InstanceType: !Ref InstanceType
+      ImageId: !Ref ImageId
+      KeyName: !Ref KeyName
+      NetworkInterfaces: 
+        - DeviceIndex: "0"
+          GroupSet:
+            - !Ref SecurityGroupId
+          SubnetId: !Ref SubnetId
+      BlockDeviceMappings: 
+        - DeviceName: "/dev/xvda"
+          Ebs: 
+            VolumeType: "gp3"
+            VolumeSize: "8"
+      CreditSpecification:
+        CPUCredits: "standard"

create parameter file

$ jq . parameters.json
[
  {
    "ParameterKey": "ImageId",
    "ParameterValue": "ami-007daaef51c7530e7"
  },
  {
    "ParameterKey": "InstanceType",
    "ParameterValue": "t4g.nano"
  },
  {
    "ParameterKey": "KeyName",
    "ParameterValue": "testkey"
  },
  {
    "ParameterKey": "SecurityGroupId",
    "ParameterValue": "sg-xxxxxxxxxxxxxxxxx"
  },
  {
    "ParameterKey": "SubnetId",
    "ParameterValue": "subnet-xxxxxxxxxxxxxxxxx"
  }
]

create a resources-to-import file

$ jq . import.json 
[
  {
    "ResourceType": "AWS::EC2::Instance",
    "LogicalResourceId": "EC2Instance1",
    "ResourceIdentifier": {
      "InstanceId": "i-xxxxxxxxxxxxxxxxx"
    }
  }
]

create change set for import and execute it

$ aws cloudformation validate-template --template-body file://template.yml
$ aws cloudformation create-change-set --stack-name mystack --change-set-name import-ec2instance --change-set-type IMPORT --resources-to-import file://import.json --template-body file://template.yml --parameters file://parameters.json
$ aws cloudformation describe-change-set --change-set-name import-ec2instance --stack-name mystack
$ aws cloudformation execute-change-set  --change-set-name import-ec2instance --stack-name mystack

confirm the result

$ aws cloudformation describe-stacks --stack-name mystack
$ aws cloudformation describe-stack-events --stack-name mystack

detect stack drift

$ aws cloudformation detect-stack-drift --stack-name mystack
$ aws cloudformation describe-stack-drift-detection-status --stack-drift-detection-id xxxxxxxxxxxxx-xxxx-xxxx-xxxxxxxxxxx
$ aws cloudformation describe-stack-resource-drifts --stack-name mystack

If any drift exist, edit stack template to fit to current resource and create and execute change set.

after that, change deletion policy to Delete and update stack if needed

$ aws cloudformation update-stack --stack-name mystack --template-body file://template.yml --parameters file://parameters.json

ami

create AMI

aws ec2 create-image --description 'backup ami of test server' --instance-id i-xxxx --name 'backup ami of test server' --no-reboot
aws ec2 deregister-image --image-id ami-xxxx 
aws ec2 describe-snapshots --owner-id xxxx --query 'Snapshots[?contains(Description, `ami-xxxx`)]'

create an instance with AMI

aws ec2 run-instances \
--image-id ami-xxxx \
--instance-type t4g.nano \
--key-name testkey \
--security-group-ids sg-xxxx \
--subnet-id subnet-xxxx \
--credit-specification 'CpuCredits=standard' \
--tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=copied_instance}]' \
--associate-public-ip-address

delete AMI

aws ec2 deregister-image --image-id ami-xxxx 
aws ec2 describe-snapshots --owner-id xxxx --query 'Snapshots[?contains(Description, `ami-xxxx`)].SnapshotId'
aws ec2 delete-snapshot --snapshot-id snap-xxxx

ebs

create an EBS volume and attach to an EC2 instance

aws ec2 create-volume --availability-zone ap-northeast-1a --volume-type gp3 --size 1 --encrypted --tag-specifications 'ResourceType=volume,Tags=[{Key=Name,Value=test_volume}]'
aws ec2 describe-volumes --volume-id vol-xxxx
aws ec2 attach-volume --device /dev/xvdb --instance-id i-xxxx --volume-id vol-xxxx

make partition and make filesystem

$ lsblk
$ sudo parted /dev/nvme1n1 print
$ sudo parted /dev/nvme1n1 mklabel gpt
$ sudo parted /dev/nvme1n1 mkpart home ext4 1MB 100%

$ sudo mkfs -t ext4 /dev/nvme1n1p1
$ sudo tune2fs -L homefs /dev/nvme1n1p1
$ ls -l /dev/disk/by-label/

edit /etc/fstab and reboot

$ sudo mount /dev/nvme1n1p1 /mnt
$ sudo cp -pri /home/ubuntu /mnt
$ sudo cp -pri /etc/fstab /etc/fstab.000
$ sudo vi /etc/fstab
$ diff /etc/fstab /etc/fstab.000
3d2

aws ec2 reboot-instances --instance-ids i-xxxx --dry-run

extend disk size

aws ec2 modify-volume --volume-id vol-xxxx --size 2

extend partition and filesystem

$ lsblk
$ sudo growpart /dev/nvme1n1 1

$ df -hT /home
$ sudo resize2fs /dev/nvme1n1p1

create snapshot

aws ec2 describe-snapshots --owner-id xxxx
aws ec2 create-snapshot --volume-id vol-xxxx --description 'test snapshot of homefs' --tag-specifications 'ResourceType=snapshot,Tags=[{Key=Name,Value=homefs}]'
aws ec2 describe-snapshots --owner-id xxxx --filters 'Name=volume-id,Values=vol-xxxx'

create new EBS volume from snapshot

aws ec2 create-volume --availability-zone ap-northeast-1a --snapshot-id snap-xxxx --volume-type gp3 --encrypted --tag-specifications 'ResourceType=volume,Tags=[{Key=Name,Value=test_volume}]'
aws ec2 describe-volumes --volume-id vol-yyyy

attach new volume and detach old volume

aws ec2 attach-volume --device /dev/xvdc --instance-id i-xxxx --volume-id vol-yyyy
aws ec2 stop-instances --instance-ids i-xxxx
aws ec2 detach-volume --no-force --instance-id i-xxxx --volume-id vol-xxxx
aws ec2 start-instances --instance-ids i-xxxx

delete volume

aws ec2 delete-volume --volume-id vol-yyyy
aws ec2 describe-volumes

delete snapshot

aws ec2 describe-snapshots --owner-id xxxx
aws ec2 delete-snapshot --snapshot-id snap-xxxx