create S3 bucket and DynamoDB table

At first make tf file to build S3 bucket for backend to store state file and DynamoDB table for lock control

resource "aws_s3_bucket" "terraform_state" {
  bucket = "mybucketname"
  versioning {
    enabled = true
  }

  server_side_encryption_configuration {
    rule {
      apply_server_side_encryption_by_default {
        sse_algorithm = "AES256"
      }
    }
  }

  # lifecycle {
  #   prevent_destroy = true
  # }

  tags = {
    Name = "terraform_backend"
  }
}

resource "aws_s3_bucket_public_access_block" "terraform_state" {
  bucket = aws_s3_bucket.terraform_state.bucket

  block_public_acls = true
  block_public_policy = true
  ignore_public_acls = true
  restrict_public_buckets = true
}

resource "aws_dynamodb_table" "terraform_state_lock" {
  name = "terraform_state_lock"
  read_capacity = 1
  write_capacity = 1
  hash_key = "LockID"

  attribute {
    name = "LockID"
    type = "S"
  }
}
variable "region" {
  default = "ap-northeast-1"
}

provider "aws" {
  region = var.region
  version = "~> 2.61.0"
}

terraform {
  required_version = ">= 0.12.26"
#  backend "s3" {
#    bucket = "mybucketname"
#    key    = "network/terraform.tfstate"
#    region = "ap-northeast-1"
#    dynamodb_table = "terraform_state_lock"
#  }
}

then initialize

terraform init 
terraform show

at last create bucket and table

terraform plan -out terraform.plan -no-color 
terraform apply "terraform.plan" -no-color 
terraform show

change backend to S3

at first edit tf file to enable S3 backend

terraform {
  required_version = ">= 0.12.26"
  backend "s3" {
    bucket = "mybucketname"
    key    = "network/terraform.tfstate"
    region = "ap-northeast-1"
    dynamodb_table = "terraform_state_lock"
  }
}

then initialize

terraform init -no-color 
aws s3api list-object-versions  --bucket mybucketname --prefix network/terraform.tfstate --query 'Versions[].{VersionId:VersionId, LastModified:LastModified}'

then you can use S3 backend

add tf file content to build aws resource

variable "cidr_block" {
  default = "10.0.0.0/16"
}

resource "aws_vpc" "terraform_test_vpc" {
  cidr_block           = var.cidr_block
  instance_tenancy     = "default"
  enable_dns_support   = true
  enable_dns_hostnames = true

  tags = {
    Name = "terraform_test"
  }
}

plan and apply as usual

terraform plan -no-color -out terraform.plan 
terraform apply "terraform.plan" 
terraform show

aws s3api list-object-versions  --bucket mybucketname --prefix network/terraform.tfstate --query 'Versions[].{VersionId:VersionId, LastModified:LastModified}'

remove all reosources other than S3 bucket and DynamoDB table

before change backend from S3 to local again, remove all other resources.

at first remove all aws resources from tf file other than S3 bucket for backend and DynamoDB table for lock control. then plan and apply

terraform plan -no-color -out terraform.plan 
terraform apply "terraform.plan" 
terraform show

change backend from S3 to local

comment out or remove backend from tf file

    terraform {
      required_version = ">= 0.12.26"
    #  backend "s3" {
    #    bucket = "mybucketname"
    #    key    = "network/terraform.tfstate"
    #    region = "ap-northeast-1"
    #    dynamodb_table = "terraform_state_lock"
    #  }
    }

then initialize.

terraform init -no-color 
ls -l terraform.tfstate

remove S3 tables and DynamoDB table

before remove them, make sure S3 bucket empty

aws s3api list-object-versions  --bucket mybucketname --prefix network/terraform.tfstate --query 'Versions[].{VersionId:VersionId, LastModified:LastModified}'
delete_objects=$(aws s3api list-object-versions --bucket mybucketname --prefix network/terraform.tfstate \
--query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}')
aws s3api delete-objects --bucket mybucketname --delete "${delete_objects}"

terraform destroy 
terraform show

Sample yaml file for stack of CloudFormation to build backend S3 bucket and DynamoDB table

AWSTemplateFormatVersion: '2010-09-09'
Parameters:
  BucketName:
    Type: String
  TableName:
    Type: String
Resources:
  BackendBucket:
    Type: AWS::S3::Bucket
    DeletionPolicy: Retain
    Properties:
      BucketName: !Ref BucketName
      PublicAccessBlockConfiguration:
        BlockPublicAcls: True
        BlockPublicPolicy: True
        IgnorePublicAcls: True
        RestrictPublicBuckets: True
      BucketEncryption:
        ServerSideEncryptionConfiguration:
        - ServerSideEncryptionByDefault:
            SSEAlgorithm: AES256
      VersioningConfiguration:
        Status: "Enabled"
      Tags:
        - "Key": "Name"
          "Value": "test"
  LockctrlTable:
    Type: AWS::DynamoDB::Table
    Properties:
      TableName: !Ref TableName
      AttributeDefinitions:
        - AttributeName: "LockID"
          AttributeType: "S"
      KeySchema:
        - AttributeName: "LockID"
          KeyType: "HASH"
      BillingMode: "PROVISIONED"
      ProvisionedThroughput:
        ReadCapacityUnits: 1
        WriteCapacityUnits: 1
      Tags:
        - Key: "Name"
          Value: "test"

Dockerfile

From centos:centos8

ARG VERSION=0.12.25
RUN yum install -y unzip python2-pip openssh-clients && \
    yum clean all && \
    curl -s https://releases.hashicorp.com/terraform/${VERSION}/terraform_${VERSION}_linux_amd64.zip -o terraform.zip && \
    unzip terraform.zip && \
    rm terraform.zip && \
    mv terraform /usr/local/bin && \
    pip2 install ansible boto boto3 awscli && \
    ln -s /usr/bin/python2.7 /usr/bin/python && \
    mkdir /tmp/terraform && \
    useradd -m docker

VOLUME ["/tmp/terraform"]
USER "docker"
WORKDIR "/tmp/terraform"
CMD ["/bin/bash"]

build an image

$ sudo docker build --build-arg VERSION=0.12.25 -t terraform:0.12.25 .
$ sudo docker tag terraform:0.12.25 terraform:latest

test the image

$ sudo docker run -v /mylocal/terraform:/tmp/terraform --rm -it terraform:latest terraform --version
$ sudo docker run -v /mylocal/terraform:/tmp/terraform --rm -it terraform:latest ansible --version
$ sudo docker run -v /mylocal/terraform:/tmp/terraform --rm -it terraform:latest bash

create variable file /mydir/credentials

AWS_ACCESS_KEY_ID=xxxx
AWS_SECRET_ACCESS_KEY=xxxx
AWS_DEFAULT_REGION=xxxx

create wrapper script infradeploy.sh

docker run -v /mylocal/terraform:/tmp/terraform --env-file /mydir/credentials --rm -it terraform:latest $*

test terraform and ansible work

$ sudo sh infradeploy.sh terraform show
$ sudo sh infradeploy.sh ./ec2.py --list
$ sudo sh infradeploy.sh ansible -i ec2.py -u admin ec2 -m ping --private-key id_rsa.mykey
$ sudo sh infradeploy.sh ansible-playbook --check -e @extravars.json playbook.yml

As described at this site we can use ec2.py and ec2.ini(optional) for dynamic inventory When I ran ec2.py I got ImportError: No module named ansible.module_utils in the case I installed ansible from ubuntu repository. It seems ansible should be installed with pip if you want to use ec2.py

playbook sample

- name: test playbook
  hosts: tag_Name_tagname
  remote_user: admin
  become: yes
  vars:
    ansible_ssh_private_key_file: "./id_rsa.mykey"
  tasks:
    - name: install some packages
      apt:
        name: ['make','screen']
        state: present
        install_recommends: no
      when: ansible_pkg_mgr == 'apt'
      tags: packages

    - name: create a directory
      file:
        path: /my/direcoty
        state: directory
        owner: admin
        mode: '0755'

$ sudo sh infradeploy.sh ansible-playbook --check --diff -i ec2.py playbook.yml
$ sudo sh infradeploy.sh ansible-playbook --diff -i ec2.py playbook.yml

install

$ wget url https://releases.hashicorp.com/terraform/x.xx.xx/terraform_x.xx.xx_linux_amd64.zip
$ unzip terraform_x.xx.xx_linux_amd64.zip
$ cp terraform /usr/local/bin

sample varible file

terraform.tfvars

# region   = "ap-northeast-2"
# az = {
#   zone0    = "ap-northeast-2a"
#   zone1    = "ap-northeast-2b"
# }
# ami     = "ami-0d79f772de48b11f7"
cidr_block   = "172.16.0.0/16"
cidr_subnet0 = "172.16.0.0/24"
cidr_subnet1 = "172.16.10.0/24"

sample tf file

test.tf

variable "region" {
  default = "ap-northeast-1"
}
variable "az" {
  default = {
    zone0 = "ap-northeast-1a"
    zone1 = "ap-northeast-1c"
  }
}
variable "ami" {
  default = "ami-03a1ce3bba6d67270"
}
variable "instance_type" {
  default = "t3.nano"
}
variable "cidr_block" {}
variable "cidr_subnet0" {}
variable "cidr_subnet1" {}
variable "operation_addr" {}
variable "key_name" {}
variable "private_key" {}

provider "aws" {
  region = var.region
}

resource "aws_vpc" "terraform_test_vpc" {
  cidr_block           = var.cidr_block
  instance_tenancy     = "default"
  enable_dns_support   = true
  enable_dns_hostnames = true

  tags = {
    Name = "terraform_test"
  }
}

resource "aws_subnet" "terraform_test_subnet0" {
  vpc_id            = aws_vpc.terraform_test_vpc.id
  cidr_block        = var.cidr_subnet0
  availability_zone = var.az.zone0

  tags = {
    Name = "terraform_test"
  }
}

resource "aws_subnet" "terraform_test_subnet1" {
  vpc_id            = aws_vpc.terraform_test_vpc.id
  cidr_block        = var.cidr_subnet1
  availability_zone = var.az.zone1

  tags = {
    Name = "terraform_test"
  }
}

resource "aws_internet_gateway" "terraform_test_igw" {
  vpc_id = aws_vpc.terraform_test_vpc.id

  tags = {
    Name = "terraform_test"
  }
}

resource "aws_route_table" "terraform_test_route" {
  vpc_id = aws_vpc.terraform_test_vpc.id
  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.terraform_test_igw.id
  }

  tags = {
    Name = "terraform_test"
  }
}

resource "aws_route_table_association" "terraform_test_subnet0" {
  subnet_id      = aws_subnet.terraform_test_subnet0.id
  route_table_id = aws_route_table.terraform_test_route.id
}

resource "aws_route_table_association" "terraform_test_subnet1" {
  subnet_id      = aws_subnet.terraform_test_subnet1.id
  route_table_id = aws_route_table.terraform_test_route.id
}

resource "aws_security_group" "sg_ssh_web" {
  name = "terraform_test_security_group"
  description = "allow inbound ssh and web traffic"
  vpc_id = aws_vpc.terraform_test_vpc.id
  ingress {
    from_port = 22
    to_port = 22
    protocol = "tcp"
    cidr_blocks = [var.operation_addr]
  }
  ingress {
    from_port = 80
    to_port = 80
    protocol = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
  egress {
    from_port = 0
    to_port = 0
    protocol = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

resource "aws_instance" "ec2_public" {
    ami           = var.ami
    instance_type = var.instance_type
    key_name      = var.key_name
    vpc_security_group_ids = [
      aws_security_group.sg_ssh_web.id,
    ]
    subnet_id = aws_subnet.terraform_test_subnet0.id
    associate_public_ip_address = "true"
    user_data = file("userdata.sh")
    connection {
      user = "admin"
      host = self.public_ip
      private_key = file(var.private_key)
    }
    provisioner "file" {
      source      = "index.html"
      destination = "~/index.html"
    }
    provisioner "remote-exec" {
      inline = [
        "sudo apt update",
        "sudo apt install -y nginx",
        "sudo mv ~/index.html /var/www/html"
      ]
    }

    tags = {
        Name = "terraform_test"
    }
}

output "public_dns" {
  value = aws_instance.ec2_public.public_dns
}

ininialize

install plugin file under the current directory

$ terraform init

validate tf file

$ terraform validate

apply

$ terraform plan -out=apply.plan -var 'operation_addr=xxx.xxx.xxx.xxx/xx' -var 'key_name=mykey' -var 'private_key=./id_rsa.mykey'
$ terraform apply -auto-approve "apply.plan"

or when destroy

$ terraform plan -destroy -out=apply.plan -var 'operation_addr=xxx.xxx.xxx.xxx/xx' -var 'key_name=mykey' -var 'private_key=./id_rsa.mykey' -target aws_instance.web1
$ terraform apply -auto-approve "destroy.plan"

confirm result

$ terraform show

destroy

$ terraform destroy -auto-approve -var 'operation_addr=xxx.xxx.xxx.xxx/xx' -var 'key_name=mykey' -var 'private_key=./id_rsa.mykey'

sample ansible playbook

apply.playbook

- hosts: localhost
  connection: local
  gather_facts: no

  tasks:
    - name: terraform plan
      terraform:
        project_path: './'
        state: planned
        plan_file: ansible.plan
        variables:
          operation_addr: "{{ operation_addr }}"
          key_name: "{{ key_name }}"
          private_key: "{{ private_key }}"
      register: result

    - name: debug result
      debug: 
        var: result

    - name: terraform apply
      terraform:
        project_path: './'
        state: present
        plan_file: ansible.plan
        variables:
          operation_addr: "{{ operation_addr }}"
          key_name: "{{ key_name }}"
          private_key: "{{ private_key }}"
      when: not ansible_check_mode
      register: result

    - name: debug result
      debug: 
        var: result
      when: not ansible_check_mode

destroy.playbook

- hosts: localhost
  connection: local
  gather_facts: no

  tasks:
    - name: terraform destroy
      terraform:
        project_path: './'
        state: absent
        plan_file: ansible.plan
        variables:
          operation_addr: "{{ operation_addr }}"
          key_name: "{{ key_name }}"
          private_key: "{{ private_key }}"
      when: not ansible_check_mode
      register: result

    - name: debug result
      debug: 
        var: result
      when: not ansible_check_mode

apply and destroy with ansible

$ jq . extravars.json
{
  "operation_addr": "xxx.xxx.xxx.xxx/xx",
  "key_name": "mykey",
  "private_key": "id_rsa.mykey"
}
$ ansible-playbook --check -e @extravars.json apply.playbook
$ ansible-playbook -e @extravars.json apply.playbook
$ ansible-playbook --check -e @extravars.json destroy.playbook
$ ansible-playbook -e @extravars.json destroy.playbook