show all messages

journalctl

show all messages from boot

journalctl -b

show messages with explanations

journalctl -x

show kernel ring buffer

journalctl -k

show messages of limitted time span

journalctl --since "2020-06-11"
journalctl --since "20 min ago"
journalctl --since "2020-06-11 00:00:00" --untill "2020-06-11 23:59:59"

show specific messages by unit, executable or process

journalctl -u networking
journalctl /usr/bin/sudo
journalctl _PID=1

show specific messages by priority or syslog facility

journalctl -p 5
journalctl -p 0..5
journalctl SYSLOG_FACILITY=10

follow new messages

journalctl -f

don't pipe output to a pager

journalctl --no-pager

Basically journal messages exist under /run/log/journal/ which is volatile directory. Below commands move journal messages directory to directory under /var/log/journal/.

echo "Storage=persistent" | sudo tee -a /etc/systemd/journald.conf
systemctl force-reload systemd-journald

You can specify directory in which journal messages saved. It would be convinient, for example, when you read other system's journal messages which is mounted on any mount point.

journalctl -D /mnt/var/log/journal -x

show system status

systemctl status

show running or failed units

systemctl list-units
systemctl
systemctl --failed

show installed unit files

systemctl list-unit-files
ls -l /usr/lib/systemd/system /etc/systemd/system

start, stop, restart or reload a unit

systemctl start unit
systemctl stop unit
systemctl restart unit
systemctl reload unit

show status of a unit

systemctl status unit

enable or disable a unit to be started on bootstrap

systemctl enable unit
systemctl disable unit

check whether a unit is enabled or not

systemctl is-enabled unit

mask or unmask a unit to make it impossible to start it

systemctl mask unit
systemctl unmask unit

reboot or poweroff the system

systemctl reboot
systemctl poweroff

Dockerfile

From centos:centos8

ARG VERSION=0.12.25
RUN yum install -y unzip python2-pip openssh-clients && \
    yum clean all && \
    curl -s https://releases.hashicorp.com/terraform/${VERSION}/terraform_${VERSION}_linux_amd64.zip -o terraform.zip && \
    unzip terraform.zip && \
    rm terraform.zip && \
    mv terraform /usr/local/bin && \
    pip2 install ansible boto boto3 awscli && \
    ln -s /usr/bin/python2.7 /usr/bin/python && \
    mkdir /tmp/terraform && \
    useradd -m docker

VOLUME ["/tmp/terraform"]
USER "docker"
WORKDIR "/tmp/terraform"
CMD ["/bin/bash"]

build an image

$ sudo docker build --build-arg VERSION=0.12.25 -t terraform:0.12.25 .
$ sudo docker tag terraform:0.12.25 terraform:latest

test the image

$ sudo docker run -v /mylocal/terraform:/tmp/terraform --rm -it terraform:latest terraform --version
$ sudo docker run -v /mylocal/terraform:/tmp/terraform --rm -it terraform:latest ansible --version
$ sudo docker run -v /mylocal/terraform:/tmp/terraform --rm -it terraform:latest bash

create variable file /mydir/credentials

AWS_ACCESS_KEY_ID=xxxx
AWS_SECRET_ACCESS_KEY=xxxx
AWS_DEFAULT_REGION=xxxx

create wrapper script infradeploy.sh

docker run -v /mylocal/terraform:/tmp/terraform --env-file /mydir/credentials --rm -it terraform:latest $*

test terraform and ansible work

$ sudo sh infradeploy.sh terraform show
$ sudo sh infradeploy.sh ./ec2.py --list
$ sudo sh infradeploy.sh ansible -i ec2.py -u admin ec2 -m ping --private-key id_rsa.mykey
$ sudo sh infradeploy.sh ansible-playbook --check -e @extravars.json playbook.yml

As described at this site we can use ec2.py and ec2.ini(optional) for dynamic inventory When I ran ec2.py I got ImportError: No module named ansible.module_utils in the case I installed ansible from ubuntu repository. It seems ansible should be installed with pip if you want to use ec2.py

playbook sample

- name: test playbook
  hosts: tag_Name_tagname
  remote_user: admin
  become: yes
  vars:
    ansible_ssh_private_key_file: "./id_rsa.mykey"
  tasks:
    - name: install some packages
      apt:
        name: ['make','screen']
        state: present
        install_recommends: no
      when: ansible_pkg_mgr == 'apt'
      tags: packages

    - name: create a directory
      file:
        path: /my/direcoty
        state: directory
        owner: admin
        mode: '0755'

$ sudo sh infradeploy.sh ansible-playbook --check --diff -i ec2.py playbook.yml
$ sudo sh infradeploy.sh ansible-playbook --diff -i ec2.py playbook.yml

install

$ wget url https://releases.hashicorp.com/terraform/x.xx.xx/terraform_x.xx.xx_linux_amd64.zip
$ unzip terraform_x.xx.xx_linux_amd64.zip
$ cp terraform /usr/local/bin

sample varible file

terraform.tfvars

# region   = "ap-northeast-2"
# az = {
#   zone0    = "ap-northeast-2a"
#   zone1    = "ap-northeast-2b"
# }
# ami     = "ami-0d79f772de48b11f7"
cidr_block   = "172.16.0.0/16"
cidr_subnet0 = "172.16.0.0/24"
cidr_subnet1 = "172.16.10.0/24"

sample tf file

test.tf

variable "region" {
  default = "ap-northeast-1"
}
variable "az" {
  default = {
    zone0 = "ap-northeast-1a"
    zone1 = "ap-northeast-1c"
  }
}
variable "ami" {
  default = "ami-03a1ce3bba6d67270"
}
variable "instance_type" {
  default = "t3.nano"
}
variable "cidr_block" {}
variable "cidr_subnet0" {}
variable "cidr_subnet1" {}
variable "operation_addr" {}
variable "key_name" {}
variable "private_key" {}

provider "aws" {
  region = var.region
}

resource "aws_vpc" "terraform_test_vpc" {
  cidr_block           = var.cidr_block
  instance_tenancy     = "default"
  enable_dns_support   = true
  enable_dns_hostnames = true

  tags = {
    Name = "terraform_test"
  }
}

resource "aws_subnet" "terraform_test_subnet0" {
  vpc_id            = aws_vpc.terraform_test_vpc.id
  cidr_block        = var.cidr_subnet0
  availability_zone = var.az.zone0

  tags = {
    Name = "terraform_test"
  }
}

resource "aws_subnet" "terraform_test_subnet1" {
  vpc_id            = aws_vpc.terraform_test_vpc.id
  cidr_block        = var.cidr_subnet1
  availability_zone = var.az.zone1

  tags = {
    Name = "terraform_test"
  }
}

resource "aws_internet_gateway" "terraform_test_igw" {
  vpc_id = aws_vpc.terraform_test_vpc.id

  tags = {
    Name = "terraform_test"
  }
}

resource "aws_route_table" "terraform_test_route" {
  vpc_id = aws_vpc.terraform_test_vpc.id
  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.terraform_test_igw.id
  }

  tags = {
    Name = "terraform_test"
  }
}

resource "aws_route_table_association" "terraform_test_subnet0" {
  subnet_id      = aws_subnet.terraform_test_subnet0.id
  route_table_id = aws_route_table.terraform_test_route.id
}

resource "aws_route_table_association" "terraform_test_subnet1" {
  subnet_id      = aws_subnet.terraform_test_subnet1.id
  route_table_id = aws_route_table.terraform_test_route.id
}

resource "aws_security_group" "sg_ssh_web" {
  name = "terraform_test_security_group"
  description = "allow inbound ssh and web traffic"
  vpc_id = aws_vpc.terraform_test_vpc.id
  ingress {
    from_port = 22
    to_port = 22
    protocol = "tcp"
    cidr_blocks = [var.operation_addr]
  }
  ingress {
    from_port = 80
    to_port = 80
    protocol = "tcp"
    cidr_blocks = ["0.0.0.0/0"]
  }
  egress {
    from_port = 0
    to_port = 0
    protocol = "-1"
    cidr_blocks = ["0.0.0.0/0"]
  }
}

resource "aws_instance" "ec2_public" {
    ami           = var.ami
    instance_type = var.instance_type
    key_name      = var.key_name
    vpc_security_group_ids = [
      aws_security_group.sg_ssh_web.id,
    ]
    subnet_id = aws_subnet.terraform_test_subnet0.id
    associate_public_ip_address = "true"
    user_data = file("userdata.sh")
    connection {
      user = "admin"
      host = self.public_ip
      private_key = file(var.private_key)
    }
    provisioner "file" {
      source      = "index.html"
      destination = "~/index.html"
    }
    provisioner "remote-exec" {
      inline = [
        "sudo apt update",
        "sudo apt install -y nginx",
        "sudo mv ~/index.html /var/www/html"
      ]
    }

    tags = {
        Name = "terraform_test"
    }
}

output "public_dns" {
  value = aws_instance.ec2_public.public_dns
}

ininialize

install plugin file under the current directory

$ terraform init

validate tf file

$ terraform validate

apply

$ terraform plan -out=apply.plan -var 'operation_addr=xxx.xxx.xxx.xxx/xx' -var 'key_name=mykey' -var 'private_key=./id_rsa.mykey'
$ terraform apply -auto-approve "apply.plan"

or when destroy

$ terraform plan -destroy -out=apply.plan -var 'operation_addr=xxx.xxx.xxx.xxx/xx' -var 'key_name=mykey' -var 'private_key=./id_rsa.mykey' -target aws_instance.web1
$ terraform apply -auto-approve "destroy.plan"

confirm result

$ terraform show

destroy

$ terraform destroy -auto-approve -var 'operation_addr=xxx.xxx.xxx.xxx/xx' -var 'key_name=mykey' -var 'private_key=./id_rsa.mykey'

sample ansible playbook

apply.playbook

- hosts: localhost
  connection: local
  gather_facts: no

  tasks:
    - name: terraform plan
      terraform:
        project_path: './'
        state: planned
        plan_file: ansible.plan
        variables:
          operation_addr: "{{ operation_addr }}"
          key_name: "{{ key_name }}"
          private_key: "{{ private_key }}"
      register: result

    - name: debug result
      debug: 
        var: result

    - name: terraform apply
      terraform:
        project_path: './'
        state: present
        plan_file: ansible.plan
        variables:
          operation_addr: "{{ operation_addr }}"
          key_name: "{{ key_name }}"
          private_key: "{{ private_key }}"
      when: not ansible_check_mode
      register: result

    - name: debug result
      debug: 
        var: result
      when: not ansible_check_mode

destroy.playbook

- hosts: localhost
  connection: local
  gather_facts: no

  tasks:
    - name: terraform destroy
      terraform:
        project_path: './'
        state: absent
        plan_file: ansible.plan
        variables:
          operation_addr: "{{ operation_addr }}"
          key_name: "{{ key_name }}"
          private_key: "{{ private_key }}"
      when: not ansible_check_mode
      register: result

    - name: debug result
      debug: 
        var: result
      when: not ansible_check_mode

apply and destroy with ansible

$ jq . extravars.json
{
  "operation_addr": "xxx.xxx.xxx.xxx/xx",
  "key_name": "mykey",
  "private_key": "id_rsa.mykey"
}
$ ansible-playbook --check -e @extravars.json apply.playbook
$ ansible-playbook -e @extravars.json apply.playbook
$ ansible-playbook --check -e @extravars.json destroy.playbook
$ ansible-playbook -e @extravars.json destroy.playbook

remove all carriage returns

cat file | tr -d '\n'

url encoding and decoding

echo 'http://example.com' | nkf -WwMQ | sed 's/=$//g' | tr = % | tr -d '\n'

echo http%3A%2F%2Fexample%2Ecom | nkf -w --url-input

handle files which name may include spaces

find dirname -print0 | xargs -0 ls

convert character code from sjis to UTF8

iconv -f SJIS -t UTF8 oldfile > newfile

set positional parameter

$ set -a -b foo bar
$ echo $# $@
2 foo bar

$ set -- -a -b foo bar
$ echo $# $@
4 -a -b foo bar

w | awk '$1~/'$(whoami)'/{print}' | while read line; do
  set ${line}; echo USER:$1 TTY:$2 FROM:$3 LOGIN@:$4 IDLE:$5 JCPU:$6 PCPU:$7 WHAT:${*:8}
done

color text

esc=$(printf '\033')

color="${esc}[36m" # cyan
bold="${esc}[1m"
underline="${esc}[4m"
colorbold="${esc}[1;36m"
colorunderline="${esc}[4;36m"
reset="${esc}[0m"

echo "${color}color text${reset}"
echo "${bold}bold text${reset}"
echo "${underline}underline text${reset}"
echo "${colorbold}color bold text${reset}"
echo "${colorunderline}color underline text${reset}"
echo normal text

dictionary

add key:value pair to a dictionary

>>> newdict = {}
>>> newdict['foo'] = "bar"
>>> newdict
{'foo': 'bar'}

get all keys or values of a dictionary

>>> newdict.keys()
['foo']
>>> newdict.values()
['bar']

get a value for a key from a dictionary

>>> newdict['foo']
'bar'
>>> newdict.get("foo")
'bar'

add a dictionary as a value

>>> newdict0 = {}
>>> newdict0["mykey"] = newdict
>>> newdict0
{'mykey': {'foo': 'bar'}}

get a value

>>> newdict0['mykey']['foo']
'bar'

add a list as a value

>>> newdict0["mykey"] = [newdict, {'hoge': 'piyo'}]
>>> newdict0
{'mykey': [{'foo': 'bar'}, {'hoge': 'piyo'}]}

get a list

>>> for dict in newdict0.get('mykey'):
...     print dict
...
{'foo': 'bar'}
{'hoge': 'piyo'}

json

dump dictionary as json format string

>>> import json
>>> mystr = json.dumps(newdict)
>>> mystr
'{"foo": "bar"}'

load json format string to dictionary

>>> mydict = json.loads(mystr)
>>> mydict
{u'foo': u'bar'}
>>> mydict.get("foo")
u'bar'

To show non-ASCII characters, we can use ensure_ascii=False option, otherwise the output are escaped with \uXXXX sequences

$ python26
>>> import json
>>> mystr = '{"foo":"ばぁ"}'
>>> mydict = json.loads(mystr)
>>>
>>> print(json.dumps(mydict))
{"foo": "\u3070\u3041"}
>>>
>>> print(json.dumps(mydict, ensure_ascii=False))
{"foo": "ばぁ"}
>>>

If locale is not UTF-8, we will met UnicodeEncodeError when it prints UTF-8 strings to stdout. In this case, we can use encode() to avoid the error.

$ LANG=C python26
>>> import json
>>> mystr = '{"foo":"ばぁ"}'
>>> mydict = json.loads(mystr)
>>>
>>> print(json.dumps(mydict))
{"foo": "\u3070\u3041"}
>>>
>>> print(json.dumps(mydict, ensure_ascii=False))
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
UnicodeEncodeError: 'ascii' codec can't encode characters in position 14-15: ordinal not in range(128)
>>>
>>> print(json.dumps(mydict, ensure_ascii=False).encode('utf-8'))
{"foo": "ばぁ"}
>>>

write json to a file

mystr = '{"foo":"ばぁ"}
mydict = json.loads(mystr)

with open('new.json', 'w') as f:
    json.dump(mydict, f)

read json from a file

with open('new.json') as f:
    myjson = json.load(f)
    print(json.dumps(myjson, ensure_ascii=False).encode('utf-8'))

sample script

using urllib2 library

similar as above but using reauests library

find wifi device

$ /sbin/iw dev
phy#0
    Interface wls3
        ifindex 3
        wdev 0x1
        addr xx:xx:xx:xx:xx:xx
        ssid xxxxxxx
        type managed
        channel 36 (5180 MHz), width: 20 MHz (no HT), center1: 5180 MHz
        txpower 15.00 dBm

scan for available N/W

$ /sbin/iwlist scan

for WPA2

install related packages

$ sudo apt install wpasupplicant

generate psk for a ssid

$ wpa_passphrase myssid
# reading passphrase from stdin
mypassphrase < input from stdin
network={
    ssid="myssid"
    #psk="mypassphrase"
    psk=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}

add these lines to /etc/network/interface

auto wls3
iface wls3 inet dhcp
  wpa-ap-scan 1
  wpa-scan-ssid 1
  wpa-ssid myssid
  wpa-psk xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

change permission /etc/network/interface

$ chmod 600 /etc/network/interface

mpd

Music Player Daemon. server-client architecture.

mpd.conf

You can specify audio output type and device in /etc/mpd.conf. Various types are available including http streaming. This is example for 2nd alsa device.

audio_output {
    type            "alsa"
    name            "My ALSA Device"
    device          "hw:1,0"
}

mpc

command line interface of mpd

option

-h <host> # specify server

subcommands

play
pause
next
prev

consume [on|off]

ls <directory> # lists all files in directory
listall <file> # lists all files
add <file> # add file to queue
clear # empty the queue

search <type> <query> # for example: $ mpc search filename ENYA
find <type> <query> # similar to search. but match exactly
list <type> # shows a list of all tags of type
update <path> # scans for updated files in the path

lsplaylists # lists available playlists
playlist <playlist name> # lists all songs in playlist
rm <playlist> # remove playlist
save <playlist> # save playlist

load <playlist> # load <playlist> in the queue

We can load m3u or cue files in a music directory as playlist. The file path in a playlist should be relative path from the playlist. M3U can be extended M3U.

mpc load <subdir in music dir>/<any subdir>/playlistfile ; mpc playlist

MPDroid

client for android

MaximumMPD

client for iOS

KODI Add-on:MPD Client

Add-on of KODI formerly known as XBMC. If HDMI-CEC is enabled on your monitor device, you can controll MPD with your remote controller.

VLC is opensource and crossplatform multimedia player. cvlc is command line version of vlc.

VLC(or cvlc) can stream via http. See Streaming HowTo/Command Line Examples or Streaming HowTo/Advanced Streaming Using the Command Line for detail

cvlc
--sout '#standard{access=http,mux=mp3,dst=192.168.x.x:xxxx}' # http stream. mux=dummy or raw will be ok
vlc://quit

http interface

VLC(or cvlc) could be controlled with http request. See VLC HTTP requests for detail

start cvlc with http interface

cvlc
-I HTTP # start http interface
--http-password <password> # specify http password
--aout alsa --alsa-audio-device plughw:2,0 # specify audio output device
vlc://quit

You can control vlc with not only web browser but also command line tool

curl
-I # fetche the headers only
--user ":<mypassword>" # specify http passowrd (user name is null)
http://127.0.0.1:8080/requests/status.xml 
?command=pl_play
?command=pl_pause
?command=volume&val=+<int>
?command=volume&val=-<int>

alias

$ postconf -n | grep alias
$ ls -l /etc/aliases*
$ postalias hash:/etc/aliases
$ newaliases

send mail or read mail of Maildir

$ apt install mailutils
$ echo "This is body message" | mail -s "My subject" pi
$ MAILDIR=$HOME/Maildir mail