node-test:
  hosts:
    10.10.10.11: { nodename: node-test-1 }
    10.10.10.12: { nodename: node-test-2 }
    10.10.10.13: { nodename: node-test-3 }
  vars:
    node_cluster: node-test

#nodename:           # [INSTANCE] # node instance identity, use hostname if missing, optional
node_cluster: nodes   # [CLUSTER] # node cluster identity, use 'nodes' if missing, optional
nodename_overwrite: true          # overwrite node's hostname with nodename?
nodename_exchange: false          # exchange nodename among play hosts?
node_id_from_pg: true             # use postgres identity as node identity if applicable?

node_write_etc_hosts: true        # modify `/etc/hosts` on target node?
node_default_etc_hosts:           # static dns records in `/etc/hosts`
  - "${admin_ip} h.pigsty a.pigsty p.pigsty g.pigsty"
node_etc_hosts: []                # extra static dns records in `/etc/hosts`
node_dns_method: add              # how to handle dns servers: add,none,overwrite
node_dns_servers: ['${admin_ip}'] # dynamic nameserver in `/etc/resolv.conf`
node_dns_options:                 # dns resolv options in `/etc/resolv.conf`
  - options single-request-reopen timeout:1

["${admin_ip} h.pigsty a.pigsty p.pigsty g.pigsty"]

- options single-request-reopen timeout:1

node_repo_modules: local          # upstream repo to be added on node, local by default
node_repo_remove: true            # remove existing repo on node?
node_packages: [openssh-server]   # packages to be installed current nodes with latest version
#node_default_packages: []        # default packages to be installed on infra nodes, (defaults are load from node_id/vars)

- lz4,unzip,bzip2,pv,jq,git,ncdu,make,patch,bash,lsof,wget,uuid,tuned,nvme-cli,numactl,sysstat,iotop,htop,rsync,tcpdump
- python3,python3-pip,socat,lrzsz,net-tools,ipvsadm,telnet,ca-certificates,openssl,keepalived,etcd,haproxy,chrony
- zlib,yum,audit,bind-utils,readline,vim-minimal,node_exporter,grubby,openssh-server,openssh-clients

- lz4,unzip,bzip2,pv,jq,git,ncdu,make,patch,bash,lsof,wget,uuid,tuned,nvme-cli,numactl,sysstat,iotop,htop,rsync,tcpdump
- python3,python3-pip,socat,lrzsz,net-tools,ipvsadm,telnet,ca-certificates,openssl,keepalived,etcd,haproxy,chrony
- zlib1g,acl,dnsutils,libreadline-dev,vim-tiny,node-exporter,openssh-server,openssh-client

node_disable_firewall: true       # disable node firewall? true by default
node_disable_selinux: true        # disable node selinux? true by default
node_disable_numa: false          # disable node numa, reboot required
node_disable_swap: false          # disable node swap, use with caution
node_static_network: true         # preserve dns resolver settings after reboot
node_disk_prefetch: false         # setup disk prefetch on HDD to increase performance
node_kernel_modules: [ softdog, br_netfilter, ip_vs, ip_vs_rr, ip_vs_wrr, ip_vs_sh ]
node_hugepage_count: 0            # number of 2MB hugepage, take precedence over ratio
node_hugepage_ratio: 0            # node mem hugepage ratio, 0 disable it by default
node_overcommit_ratio: 0          # node mem overcommit ratio, 0 disable it by default
node_tune: oltp                   # node tuned profile: none,oltp,olap,crit,tiny
node_sysctl_params: { }           # sysctl parameters in k:v format in addition to tuned

node_kernel_modules: [ softdog, br_netfilter, ip_vs, ip_vs_rr, ip_vs_wrr, ip_vs_sh ]

node_data: /data                  # node main data directory, `/data` by default
node_admin_enabled: true          # create a admin user on target node?
node_admin_uid: 88                # uid and gid for node admin user
node_admin_username: dba          # name of node admin user, `dba` by default
node_admin_ssh_exchange: true     # exchange admin ssh key among node cluster
node_admin_pk_current: true       # add current user's ssh pk to admin authorized_keys
node_admin_pk_list: []            # ssh public keys to be added to admin user

node_aliases:
  g:   git
  d:   docker

alias g="git"
alias d="docker"

node_timezone: ''                 # setup node timezone, empty string to skip
node_ntp_enabled: true            # enable chronyd time sync service?
node_ntp_servers:                 # ntp servers in `/etc/chrony.conf`
  - pool pool.ntp.org iburst
node_crontab_overwrite: true      # overwrite or append to `/etc/crontab`?
node_crontab: [ ]                 # crontab entries in `/etc/crontab`

node_ntp_servers: [ 'pool ${admin_ip} iburst' ]

vip_enabled: false                # enable vip on this node cluster?
# vip_address:         [IDENTITY] # node vip address in ipv4 format, required if vip is enabled
# vip_vrid:            [IDENTITY] # required, integer, 1-254, should be unique among same VLAN
vip_role: backup                  # optional, `master/backup`, backup by default, use as init role
vip_preempt: false                # optional, `true/false`, false by default, enable vip preemption
vip_interface: eth0               # node vip network interface to listen, `eth0` by default
vip_dns_suffix: ''                # node vip dns name suffix, empty string by default
vip_exporter_port: 9650           # keepalived exporter listen port, 9650 by default

haproxy_enabled: true             # enable haproxy on this node?
haproxy_clean: false              # cleanup all existing haproxy config?
haproxy_reload: true              # reload haproxy after config?
haproxy_auth_enabled: true        # enable authentication for haproxy admin page
haproxy_admin_username: admin     # haproxy admin username, `admin` by default
haproxy_admin_password: pigsty    # haproxy admin password, `pigsty` by default
haproxy_exporter_port: 9101       # haproxy admin/exporter port, 9101 by default
haproxy_client_timeout: 24h       # client side connection timeout, 24h by default
haproxy_server_timeout: 24h       # server side connection timeout, 24h by default
haproxy_services: []              # list of haproxy service to be exposed on node

haproxy_services:                   # list of haproxy service

  # expose pg-test read only replicas
  - name: pg-test-ro                # [REQUIRED] service name, unique
    port: 5440                      # [REQUIRED] service port, unique
    ip: "*"                         # [OPTIONAL] service listen addr, "*" by default
    protocol: tcp                   # [OPTIONAL] service protocol, 'tcp' by default
    balance: leastconn              # [OPTIONAL] load balance algorithm, roundrobin by default (or leastconn)
    maxconn: 20000                  # [OPTIONAL] max allowed front-end connection, 20000 by default
    default: 'inter 3s fastinter 1s downinter 5s rise 3 fall 3 on-marked-down shutdown-sessions slowstart 30s maxconn 3000 maxqueue 128 weight 100'
    options:
      - option httpchk
      - option http-keep-alive
      - http-check send meth OPTIONS uri /read-only
      - http-check expect status 200
    servers:
      - { name: pg-test-1 ,ip: 10.10.10.11 , port: 5432 , options: check port 8008 , backup: true }
      - { name: pg-test-2 ,ip: 10.10.10.12 , port: 5432 , options: check port 8008 }
      - { name: pg-test-3 ,ip: 10.10.10.13 , port: 5432 , options: check port 8008 }

node_exporter_enabled: true       # setup node_exporter on this node?
node_exporter_port: 9100          # node exporter listen port, 9100 by default
node_exporter_options: '--no-collector.softnet --no-collector.nvme --collector.tcpstat --collector.processes'

promtail_enabled: true            # enable promtail logging collector?
promtail_clean: false             # purge existing promtail status file during init?
promtail_port: 9080               # promtail listen port, 9080 by default
promtail_positions: /var/log/positions.yaml # promtail position status file path

# node-id       : generate node identity
# node_name     : setup hostname
# node_hosts    : setup /etc/hosts records
# node_resolv   : setup dns resolver
# node_firewall : setup firewall & selinux
# node_ca       : add & trust ca certificate
# node_repo     : add upstream repo
# node_pkg      : install yum packages
# node_feature  : setup numa, grub, static network
# node_kernel   : enable kernel modules
# node_tune     : setup tuned profile
# node_sysctl   : setup additional sysctl parameters
# node_profile  : write /etc/profile.d/node.sh
# node_ulimit   : setup resource limits
# node_data     : setup main data dir
# node_admin    : setup admin user and ssh key
# node_timezone : setup timezone
# node_ntp      : setup ntp server/clients
# node_crontab  : add/overwrite crontab tasks
# node_vip      : setup optional l2 vrrp vip for node cluster
#   - vip_install
#   - vip_config
#   - vip_launch
#   - vip_reload
# haproxy       : setup haproxy on node to expose services
#   - haproxy_install
#   - haproxy_config
#   - haproxy_launch
#   - haproxy_reload
# monitor       : setup node_exporter & promtail for metrics & logs
#   - haproxy_register
#   - vip_dns
#   - node_exporter
#     - node_exporter_config
#     - node_exporter_launch
#   - vip_exporter
#     - vip_exporter_config
#     - vip_exporter_launch
#   - node_register
#   - promtail
#     - promtail_clean
#     - promtail_config
#     - promtail_install
#     - promtail_launch

# register       : remove register from prometheus & nginx
#   - prometheus : remove registered prometheus monitor target
#   - nginx      : remove nginx proxy record for haproxy admin
# vip            : remove node keepalived if enabled
# haproxy        : remove haproxy load balancer
# node_exporter  : remove monitoring exporter
# vip_exporter   : remove keepalived_exporter if enabled
# promtail       : remove loki log agent
# profile        : remove /etc/profile.d/node.sh

# ./node.yml -l <cls|ip|group>        # the underlying playbook
# bin/node-add <selector|ip...>       # add cluster/node to pigsty
bin/node-add node-test                # init node cluster 'node-test'
bin/node-add 10.10.10.10              # init node '10.10.10.10'

# ./node-rm.yml -l <cls|ip|group>    # the underlying playbook
# bin/node-rm <selector|ip...>       # remove node from pigsty:
bin/node-rm node-test                # remove node cluster 'node-test'
bin/node-rm 10.10.10.10              # remove node '10.10.10.10'

node.yml -t node_admin -k -K -e ansible_user=<another admin>   # input ssh/sudo password for another admin 

proxy:
  hosts:
    10.10.10.29: { nodename: proxy-1 } 
    10.10.10.30: { nodename: proxy-2 } # , vip_role: master }
  vars:
    node_cluster: proxy
    vip_enabled: true
    vip_vrid: 128
    vip_address: 10.10.10.99
    vip_interface: eth1

./node.yml -l proxy -t node_vip     # enable for the first time
./node.yml -l proxy -t vip_refresh  # refresh vip config (e.g. designated master) 

# Play
./node.yml -t node                            # init node itself (haproxy monitor not included)
./node.yml -t haproxy                         # setup haproxy on node to expose services
./node.yml -t monitor                         # setup node_exporter & promtail for metrics & logs
./node.yml -t node_vip                        # enable keepalived for node cluster L2 VIP
./node.yml -t vip_config,vip_reload           # refresh L2 VIP configuration
./node.yml -t haproxy_config,haproxy_reload   # refresh haproxy services definition on node cluster
./node.yml -t register_prometheus             # register node to Prometheus
./node.yml -t register_nginx                  # register haproxy admin page url to Nginx on infra nodes

# Task
./node.yml -t node-id        # generate node identity
./node.yml -t node_name      # setup hostname
./node.yml -t node_hosts     # setup /etc/hosts records
./node.yml -t node_resolv    # setup dns resolver
./node.yml -t node_firewall  # setup firewall & selinux
./node.yml -t node_ca        # add & trust ca certificate
./node.yml -t node_repo      # add upstream repo
./node.yml -t node_pkg       # install yum packages
./node.yml -t node_feature   # setup numa, grub, static network
./node.yml -t node_kernel    # enable kernel modules
./node.yml -t node_tune      # setup tuned profile
./node.yml -t node_sysctl    # setup additional sysctl parameters
./node.yml -t node_profile   # write /etc/profile.d/node.sh
./node.yml -t node_ulimit    # setup resource limits
./node.yml -t node_data      # setup main data dir
./node.yml -t node_admin     # setup admin user and ssh key
./node.yml -t node_timezone  # setup timezone
./node.yml -t node_ntp       # setup ntp server/clients
./node.yml -t node_crontab   # add/overwrite crontab tasks
./node.yml -t node_vip       # setup optional l2 vrrp vip for node cluster

################################################################
#                         Node Alert                           #
################################################################
- name: node-alert
  rules:

    #==============================================================#
    #                          Aliveness                           #
    #==============================================================#
    # node exporter is dead indicate node is down
    - alert: NodeDown
      expr: node_up < 1
      for: 1m
      labels: { level: 0, severity: CRIT, category: node }
      annotations:
        summary: "CRIT NodeDown {{ $labels.ins }}@{{ $labels.instance }}"
        description: |
          node_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1
          http://g.pigsty/d/node-instance?var-ins={{ $labels.ins }}          

    # haproxy the load balancer
    - alert: HaproxyDown
      expr: haproxy_up < 1
      for: 1m
      labels: { level: 0, severity: CRIT, category: node }
      annotations:
        summary: "CRIT HaproxyDown {{ $labels.ins }}@{{ $labels.instance }}"
        description: |
          haproxy_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1
          http://g.pigsty/d/node-haproxy?var-ins={{ $labels.ins }}          

    # promtail the logging agent
    - alert: PromtailDown
      expr: promtail_up < 1
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: "WARN PromtailDown {{ $labels.ins }}@{{ $labels.instance }}"
        description: |
          promtail_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1
          http://g.pigsty/d/node-instance?var-ins={{ $labels.ins }}          

    # docker the container engine
    - alert: DockerDown
      expr: docker_up < 1
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: "WARN DockerDown {{ $labels.ins }}@{{ $labels.instance }}"
        description: |
          docker_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1
          http://g.pigsty/d/node-instance?var-ins={{ $labels.ins }}          

    # keepalived daemon
    - alert: KeepalivedDown
      expr: keepalived_up < 1
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: "WARN KeepalivedDown {{ $labels.ins }}@{{ $labels.instance }}"
        description: |
          keepalived_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1
          http://g.pigsty/d/node-instance?var-ins={{ $labels.ins }}          



    #==============================================================#
    #                          Node : CPU                          #
    #==============================================================#
    # cpu usage high : 1m avg cpu usage > 70% for 3m
    - alert: NodeCpuHigh
      expr: node:ins:cpu_usage_1m > 0.70
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeCpuHigh {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:ins:cpu_usage[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 70%          

    # OPTIONAL: one core high
    # OPTIONAL: throttled
    # OPTIONAL: frequency
    # OPTIONAL: steal

    #==============================================================#
    #                       Node : Schedule                        #
    #==============================================================#
    # node load high : 1m avg standard load > 100% for 3m
    - alert: NodeLoadHigh
      expr: node:ins:stdload1 > 1
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeLoadHigh {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:ins:stdload1[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 100%          


    #==============================================================#
    #                        Node : Memory                         #
    #==============================================================#
    # available memory < 10%
    - alert: NodeOutOfMem
      expr: node:ins:mem_avail < 0.10
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeOutOfMem {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:ins:mem_avail[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} < 10%          

    # commit ratio > 90%
    #- alert: NodeMemCommitRatioHigh
    #  expr: node:ins:mem_commit_ratio > 0.90
    #  for: 1m
    #  labels: { level: 1, severity: WARN, category: node }
    #  annotations:
    #    summary: 'WARN NodeMemCommitRatioHigh {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
    #    description: |
    #      node:ins:mem_commit_ratio[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 90%

    # OPTIONAL: EDAC Errors

    #==============================================================#
    #                        Node : Swap                           #
    #==============================================================#
    # swap usage > 1%
    - alert: NodeMemSwapped
      expr: node:ins:swap_usage > 0.01
      for: 5m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeMemSwapped {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:ins:swap_usage[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 1%          

    #==============================================================#
    #                     Node : File System                       #
    #==============================================================#

    # filesystem usage > 90%
    - alert: NodeFsSpaceFull
      expr: node:fs:space_usage > 0.90
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeFsSpaceFull {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:fs:space_usage[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 90%          

    # inode usage > 90%
    - alert: NodeFsFilesFull
      expr: node:fs:inode_usage > 0.90
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeFsFilesFull {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:fs:inode_usage[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 90%          

    # file descriptor usage > 90%
    - alert: NodeFdFull
      expr: node:ins:fd_usage > 0.90
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeFdFull {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          node:ins:fd_usage[ins={{ $labels.ins }}] = {{ $value  | printf "%.2f" }} > 90%          

    # OPTIONAL: space predict 1d
    # OPTIONAL: filesystem read-only
    # OPTIONAL: fast release on disk space

    #==============================================================#
    #                          Node : Disk                         #
    #==============================================================#
    # read latency > 32ms (typical on pci-e ssd: 100µs)
    - alert: NodeDiskSlow
      expr: node:dev:disk_read_rt_1m{device="dfa"} > 0.032 or node:dev:disk_write_rt_1m{device="dfa"} > 0.032
      for: 1m
      labels: { level: 2, severity: INFO, category: node }
      annotations:
        summary: 'INFO NodeReadSlow {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.6f" }}'
        description: |
          node:dev:disk_read_rt_1m[ins={{ $labels.ins }}] = {{ $value  | printf "%.6f" }} > 32ms          

    # OPTIONAL: raid card failure
    # OPTIONAL: read/write traffic high
    # OPTIONAL: read/write latency high

    #==============================================================#
    #                        Node : Network                        #
    #==============================================================#
    # OPTIONAL: unusual network traffic
    # OPTIONAL: interface saturation high

    #==============================================================#
    #                        Node : Protocol                       #
    #==============================================================#

    # rate(node:ins:tcp_error[1m]) > 1
    - alert: NodeTcpErrHigh
      expr: rate(node:ins:tcp_error[1m]) > 1
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeTcpErrHigh {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.2f" }}'
        description: |
          rate(node:ins:tcp_error{ins={{ $labels.ins }}}[1m]) = {{ $value  | printf "%.2f" }} > 1          

    # node:ins:tcp_retrans_ratio1m > 1e-4
    - alert: NodeTcpRetransHigh
      expr: node:ins:tcp_retrans_ratio1m > 1e-2
      for: 1m
      labels: { level: 2, severity: INFO, category: node }
      annotations:
        summary: 'INFO NodeTcpRetransHigh {{ $labels.ins }}@{{ $labels.instance }} {{ $value  | printf "%.6f" }}'
        description: |
          node:ins:tcp_retrans_ratio1m[ins={{ $labels.ins }}] = {{ $value  | printf "%.6f" }} > 1%          

    # OPTIONAL: tcp conn high
    # OPTIONAL: udp traffic high
    # OPTIONAL: conn track

    #==============================================================#
    #                          Node : Time                         #
    #==============================================================#

    - alert: NodeTimeDrift
      expr: node_timex_sync_status != 1
      for: 1m
      labels: { level: 1, severity: WARN, category: node }
      annotations:
        summary: 'WARN NodeTimeDrift {{ $labels.ins }}@{{ $labels.instance }}'
        description: |
          node_timex_status[ins={{ $labels.ins }}]) = {{ $value | printf "%.6f" }} != 0 or
          node_timex_sync_status[ins={{ $labels.ins }}]) = {{ $value | printf "%.6f" }} != 1          


    # time drift > 64ms
    # - alert: NodeTimeDrift
    #   expr: node:ins:time_drift > 0.064
    #   for: 1m
    #   labels: { level: 1, severity: WARN, category: node }
    #   annotations:
    #     summary: 'WARN NodeTimeDrift {{ $labels.ins }}@{{ $labels.instance }}'
    #     description: |
    #       abs(node_timex_offset_seconds)[ins={{ $labels.ins }}]) = {{ $value | printf "%.6f" }} > 64ms