526 Log server and clients (ansible_init)

Use case

Use the iocage template ansible-init created in 524 iocage template ansible-init. Configure the repository ansible-conf-init to pull the jails’ configuration from the repositories ansible-conf-syslogng-server and ansible-conf-syslogng-client. Create jails from the template. Use class=log-server and class=log-client to select the configuration. Run ansible-pull asynchronously.

Tree

shell> tree .
.
├── ansible.cfg
├── group_vars
│   └── all
│       ├── project-hosts.yml
│       └── project.yml
├── hosts
│   └── 06_iocage2.yml
├── iocage.ini
├── pb-logclient-test.yml
├── pb-logserver-test.yml
└── templates
    └── project-hosts.yml.j2

Synopsis

Requirements

Note

Note

vbotka.freebsd.iocage_template is the role iocage_template in the collection vbotka.freebsd.
vbotka.freebsd_iocage_template is the role freebsd_iocage_template in the namespace vbotka.
vbotka.freebsd_postinstall is the role freebsd_postinstall in the namespace vbotka.

ansible.cfg

[defaults]
callback_result_format = yaml
deprecation_warnings = False
display_skipped_hosts = False
gathering = explicit
interpreter_python = auto_silent
log_path = /var/log/ansible.log

[connection]
pipelining = true

Inventory iocage.ini

iocage_06

[iocage]
iocage_06

[iocage:vars]
ansible_user=admin
ansible_become=true
ansible_python_interpreter=auto_silent

hosts

hosts/06_iocage2.yml
plugin: vbotka.freebsd.iocage2
host: iocage_06
user: admin
sudo: true
get_properties: true
inventory_hostname_tag: alias

compose:
  iocage_tags: dict(iocage_properties.notes | regex_findall('(\w+)=([\w\-]+)'))
  iocage_classes: iocage_properties.notes | regex_findall('(?<=class=)[\w\-]+|(?<=,)[\w\-]+')
# connection plugin vbotka.freebsd.jailexec
  ansible_connection: "'vbotka.freebsd.jailexec'"
  ansible_ssh_user: "'admin'"
  ansible_jail_host: dict(iocage_properties.notes | regex_findall('(\w+)=([\w\-]+)')).vmm | d('none')
  ansible_jail_name: iocage_jid
  ansible_jail_privilege_escalation: "'sudo'"
# ansible options
  ansible_python_interpreter: "'auto_silent'"

groups:
  log_servers: iocage_classes is contains('log-server')
  log_clients: iocage_classes is contains('log-client')

keyed_groups:
  - prefix: state
    key: iocage_state
  - prefix: vmm
    key: iocage_tags.vmm

group_vars

group_vars/all/project-hosts.yml
project_hosts:
  log_server: 172.16.99.10
  iocage_06:
    local_repos: 10.30.10.69
    defaultrouter: 172.16.99.1
    default_pkg_repo: 172.16.99.1
    local_pkg_repo: 172.16.99.1
    log_server: 172.16.99.10
    repos: 172.16.99.21
    repos_devel: 172.16.99.22
    pkg_repo: 172.16.99.23
group_vars/all/project.yml
project:
  log-server-01:
    notes: alias=log_server_01
    vmm: iocage_06
    template: ansible-init
    class: [log-server]
    custom_facts: class.fact.j2
    ai_vars_templates:
      - project-hosts.yml
    properties:
      defaultrouter: "{{ project_hosts[inventory_hostname]['defaultrouter'] }}"
      ip4_addr: "vnet0|{{ project_hosts[inventory_hostname]['log_server'] }}/24"
      vnet: 1
      boot: 1
  www-01:
    notes: alias=www_01
    vmm: iocage_06
    template: ansible-init
    class: [log-client]
    custom_facts: class.fact.j2
    ai_vars_templates:
      - project-hosts.yml
    properties:
      bpf: 1
      dhcp: 1
      vnet: 1
      boot: 1
  www-02:
    notes: alias=www_02
    vmm: iocage_06
    template: ansible-init
    class: [log-client]
    custom_facts: class.fact.j2
    ai_vars_templates:
      - project-hosts.yml
    properties:
      bpf: 1
      dhcp: 1
      vnet: 1
      boot: 1

vmm: "{{ (project | vbotka.freebsd.project).vmm }}"

templates

templates/project-hosts.yml.j2
project_hosts:
  {{ project_hosts[inventory_hostname] }}

Playbook output - Create project jails from iocage templates

(env) > ansible-playbook -i iocage.ini -i hosts \
                         vbotka.freebsd.pb_iocage_project_create_from_templates.yml
PLAY [Create and start project jails from iocage templates.] *******************

TASK [Setup: Get activated pool.] **********************************************
ok: [iocage_06]

TASK [Create: Create jails.] ***************************************************
ok: [iocage_06] => (item=www-02)
ok: [iocage_06] => (item=log-server-01)
ok: [iocage_06] => (item=www-01)

TASK [Facts: Create custom facts dirs.] ****************************************
changed: [iocage_06] => (item=log-server-01)
changed: [iocage_06] => (item=www-01)
changed: [iocage_06] => (item=www-02)

TASK [Facts: Create custom facts files.] ***************************************
changed: [iocage_06] => (item=log-server-01)
changed: [iocage_06] => (item=www-01)
changed: [iocage_06] => (item=www-02)

TASK [Ai-vars-templates: Create ansible init vars dirs.] ***********************
changed: [iocage_06] => (item=log-server-01)
changed: [iocage_06] => (item=www-01)
changed: [iocage_06] => (item=www-02)

TASK [Ai-vars-templates: Create ansible init vars from templates.] *************
changed: [iocage_06] => (item=log-server-01 project-hosts.yml)
changed: [iocage_06] => (item=www-01 project-hosts.yml)
changed: [iocage_06] => (item=www-02 project-hosts.yml)

TASK [Ai-vars-class: Create ansible init vars dirs.] ***************************
ok: [iocage_06] => (item=log-server-01)
ok: [iocage_06] => (item=www-01)
ok: [iocage_06] => (item=www-02)

TASK [Ai-vars-jail: Create ansible init vars dirs.] ****************************
ok: [iocage_06] => (item=log-server-01)
ok: [iocage_06] => (item=www-01)
ok: [iocage_06] => (item=www-02)

TASK [Properties: Set properties.] *********************************************
ok: [iocage_06] => (item=log-server-01)
ok: [iocage_06] => (item=www-01)
ok: [iocage_06] => (item=www-02)

TASK [Start: Start jails.] *****************************************************
ok: [iocage_06]

PLAY RECAP *********************************************************************
iocage_06                  : ok=10   changed=4    unreachable=0    failed=0    skipped=19   rescued=0    ignored=0   

Graph

(env) > ansible-inventory -i hosts --graph
@all:
  |--@ungrouped:
  |--@state_up:
  |  |--repos
  |  |--repos_devel
  |  |--pkg_repo
  |  |--log_server_01
  |  |--www_02
  |  |--www_01
  |--@vmm_iocage_06:
  |  |--repos
  |  |--repos_devel
  |  |--pkg_repo
  |  |--log_server_01
  |  |--www_02
  |  |--www_01
  |--@log_servers:
  |  |--log_server_01
  |--@log_clients:
  |  |--www_02
  |  |--www_01

Jails

shell> ssh admin@iocage_06 sudo iocage list -l
+-----+---------------+------+-------+------+-----------------+-----------------------+-----+------------------+----------+
| JID |     NAME      | BOOT | STATE | TYPE |     RELEASE     |          IP4          | IP6 |     TEMPLATE     | BASEJAIL |
+=====+===============+======+=======+======+=================+=======================+=====+==================+==========+
| 29  | log-server-01 | on   | up    | jail | 15.1-RELEASE-p1 | vnet0|172.16.99.10/24 | -   | ansible-init     | no       |
+-----+---------------+------+-------+------+-----------------+-----------------------+-----+------------------+----------+
| 2   | pkg-repo      | on   | up    | jail | 15.1-RELEASE-p1 | vnet0|172.16.99.23/24 | -   | ansible-pkg-repo | no       |
+-----+---------------+------+-------+------+-----------------+-----------------------+-----+------------------+----------+
| 23  | repos         | on   | up    | jail | 15.1-RELEASE-p1 | vnet0|172.16.99.21/24 | -   | ansible-repos    | no       |
+-----+---------------+------+-------+------+-----------------+-----------------------+-----+------------------+----------+
| 24  | repos-devel   | on   | up    | jail | 15.1-RELEASE-p1 | vnet0|172.16.99.22/24 | -   | ansible-repos    | no       |
+-----+---------------+------+-------+------+-----------------+-----------------------+-----+------------------+----------+
| 30  | www-01        | on   | up    | jail | 15.1-RELEASE-p1 | epair0b|172.16.99.159 | -   | ansible-init     | no       |
+-----+---------------+------+-------+------+-----------------+-----------------------+-----+------------------+----------+
| 31  | www-02        | on   | up    | jail | 15.1-RELEASE-p1 | epair0b|172.16.99.147 | -   | ansible-init     | no       |
+-----+---------------+------+-------+------+-----------------+-----------------------+-----+------------------+----------+

Playbook pb-logserver-test.yml

---
- name: Test Log Server.
  hosts: log_servers
    
  tasks:

    - name: Run commands.
      register: out
      ansible.builtin.shell: "{{ item }}"
      loop:
        - rm -rf /var/log/remote/localhost/*
        - service syslog-ng status
        - loggen -i -S -n 1 localhost 514
        - sleep 2; find /var/log/remote/localhost/ -name *.log | xargs cat

    - name: Debug.
      when: debug | d(false) | bool
      ansible.builtin.debug:
        msg: |
          {% for i in out.results %}
          shell> {{ i.item }}
          {% if i.stdout | length > 0 %}
          {{ i.stdout }}
          {% endif %}
          {% if i.stderr | length > 0 %}
          {{ i.stderr }}
          {% endif %}

          {% endfor %}

Playbook output - Test Log Server

(env) > ansible-playbook -i hosts -e debug=true pb-logserver-test.yml
PLAY [Test Log Server.] ********************************************************

TASK [Run commands.] ***********************************************************
changed: [log_server_01] => (item=rm -rf /var/log/remote/localhost/*)
changed: [log_server_01] => (item=service syslog-ng status)
changed: [log_server_01] => (item=loggen -i -S -n 1 localhost 514)
changed: [log_server_01] => (item=sleep 2; find /var/log/remote/localhost/ -name *.log | xargs cat)

TASK [Debug.] ******************************************************************
ok: [log_server_01] => 
    msg: |-
        shell> rm -rf /var/log/remote/localhost/*

        shell> service syslog-ng status
        syslog_ng is running as pid 66959.

        shell> loggen -i -S -n 1 localhost 514
        count=1, rate = 43478.26 msg/sec
        average rate = 2.00 msg/sec, count=1, time=0.500244, (average) msg size=256, bandwidth=0.50 kB/sec

        shell> sleep 2; find /var/log/remote/localhost/ -name *.log | xargs cat

PLAY RECAP *********************************************************************
log_server_01              : ok=2    changed=1    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0   

Playbook pb-logclient-test.yml

---
- name: Test Log Clients.
  hosts: log_clients

  vars:

    log_server: "{{ hostvars['log_server_01']['iocage_ip4'] }}"

  tasks:

    - name: Run loggen
      register: out
      ansible.builtin.command: "loggen -i -S -n 1 {{ log_server }} 514"

    - name: Debug.
      ansible.builtin.debug:
        var: out.stderr

- name: Display the logs.
  hosts: log_servers
    
  tasks:

    - name: Get log files paths.
      register: out
      ansible.builtin.find:
        paths: /var/log/remote
        recurse: true
        patterns: '*.log'

    - name: Get log files.
      register: out
      ansible.builtin.command: "cat {{ item }}"
      loop: "{{ out.files | map(attribute='path') }}"

    - name: Display log files.
      ansible.builtin.debug:
        msg: |
          {{ item.stdout }}

          - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      loop: "{{ out.results }}"
      loop_control:
        label: "{{ item.cmd }}"

Playbook output - Test Log Clients

(env) > ansible-playbook -i hosts pb-logclient-test.yml
PLAY [Test Log Clients.] *******************************************************

TASK [Run loggen] **************************************************************
changed: [www_02]
changed: [www_01]

TASK [Debug.] ******************************************************************
ok: [www_01] => 
    out.stderr: |-
        count=1, rate = 66666.67 msg/sec
        average rate = 2.00 msg/sec, count=1, time=0.500305, (average) msg size=256, bandwidth=0.50 kB/sec
ok: [www_02] => 
    out.stderr: |-
        count=1, rate = 66666.67 msg/sec
        average rate = 2.00 msg/sec, count=1, time=0.500163, (average) msg size=256, bandwidth=0.50 kB/sec

PLAY [Display the logs.] *******************************************************

TASK [Get log files paths.] ****************************************************
ok: [log_server_01]

TASK [Get log files.] **********************************************************
changed: [log_server_01] => (item=/var/log/remote/ip-172-16-99-147/2026_09_18.log)
changed: [log_server_01] => (item=/var/log/remote/ip-172-16-99-159/2026_09_18.log)

TASK [Display log files.] ******************************************************
ok: [log_server_01] => (item=['cat', '/var/log/remote/ip-172-16-99-147/2026_09_18.log']) => 
    msg: |-
        Sep 18 14:18:15 ip-172-16-99-147 prg00000[1234]: seq: 0000000000, thread: 0000, runid: 1789733895, stamp: 2026-09-18T14:18:15 PADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADD

        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ok: [log_server_01] => (item=['cat', '/var/log/remote/ip-172-16-99-159/2026_09_18.log']) => 
    msg: |-
        Sep 18 14:18:15 ip-172-16-99-159 prg00000[1234]: seq: 0000000000, thread: 0000, runid: 1789733895, stamp: 2026-09-18T14:18:15 PADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADDPADD

        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

PLAY RECAP *********************************************************************
log_server_01              : ok=3    changed=1    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0   
www_01                     : ok=2    changed=1    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0   
www_02                     : ok=2    changed=1    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0   

Hint

Use the lnav utility on the log server to display all log files in the /var/log/remote directory. Run the following commands on the iocage host:

shell> iocage console log-server-01
root@log-server-01:~ # lnav -r /var/log/remote/

Troubleshooting

  • Inspect the Ansible log /var/log/ansible.log in the jails:

shell> iocage console log-server-01
root@log-server-01:~ # cat /var/log/ansible.log
  • Update and pull the repositories. For example:

root@log-server-01:~ # cd /root/ansible-conf-init/
root@log-server-01:~/ansible-conf-init # git pull
Already up to date.
root@log-server-01: # cd /root/ansible-conf-syslogng-server/
root@log-server-01:~/ansible-conf-syslogng-server # git pull
Already up to date.
root@log-server-01:~ # service ansible_init start
Service ansible_init started.
Starting Ansible Pull at 2026-06-26 04:43:37
/usr/local/bin/ansible-pull -i hosts -U git://172.16.99.21/ansible-conf-init -d /root/ansible-conf-init -e ai_vars=/root/ansible-vars -e ai_pull_mode=true pb-init.yml
[WARNING]: Could not match supplied host pattern, ignoring: localhost.my.domain
[WARNING]: Could not match supplied host pattern, ignoring: log-server-01

...
TASK [Display vars.] ***********************************************************
ok: [localhost.my.domain] =>
    msg: |-
        ai_at:
        ai_async: True
        ai_cmd:      ansible-pull -i hosts -U git://172.16.99.21/ansible-conf-syslogng-server -d /root/ansible-conf-syslogng-server -e "ai_vars=/root/ansible-vars" -e "ai_pull_mode=true" pb-logserv.yml &&  echo '[INFO] ansible-pull finished.'

TASK [Execute command async=3600 poll=0] ***************************************
changed: [localhost.my.domain]

PLAY RECAP *********************************************************************
localhost.my.domain        : ok=8    changed=1    unreachable=0    failed=0    skipped=3    rescued=0    ignored=0

Note

The command service ansible_init start removes the sentinel file /firstboot.

  • Review the Ansible log /var/log/ansible.log. The warnings are harmless:

root@log-server:~ # cat /var/log/ansible.log
...
2026-08-31 08:10:36,572 p=39114 u=root n=ansible WARNING| [WARNING]: Could not match supplied host pattern, ignoring: log-server-01

2026-08-31 08:10:36,688 p=39114 u=root n=ansible INFO| PLAY [ansible-init] ************************************************************
2026-08-31 08:10:37,731 p=39114 u=root n=ansible INFO| TASK [Get hostname.] ***********************************************************
2026-08-31 08:10:37,731 p=39114 u=root n=ansible INFO| ok: [localhost.my.domain]
2026-08-31 08:10:40,361 p=39114 u=root n=ansible INFO| TASK [Get custom facts.] *******************************************************
2026-08-31 08:10:40,362 p=39114 u=root n=ansible INFO| ok: [localhost.my.domain]
2026-08-31 08:10:40,439 p=39114 u=root n=ansible INFO| TASK [Include variables from ai_vars directory.] *******************************
2026-08-31 08:10:40,441 p=39114 u=root n=ansible INFO| ok: [localhost.my.domain] => (item=/root/ansible-vars/project-hosts.yml)
2026-08-31 08:10:40,516 p=39114 u=root n=ansible INFO| TASK [Display vars.] ***********************************************************
2026-08-31 08:10:40,517 p=39114 u=root n=ansible INFO| ok: [localhost.my.domain] =>
    msg: |-
        ai_pull_mode: true
        ai_vars: /root/ansible-vars
        ai_hostname: log-server-01
        ai_class: ['log-server']
        project_hosts: {'defaultrouter': '172.16.99.1', 'log_server': '172.16.99.10', 'repos': '172.16.99.21', 'repos_devel': '172.16.99.22', 'ansible_pull_syslogng_server': '172.16.99.31'}

2026-08-31 08:10:40,705 p=39114 u=root n=ansible INFO| TASK [Set ai_cmd] **************************************************************
2026-08-31 08:10:40,706 p=39114 u=root n=ansible INFO| ok: [localhost.my.domain]
2026-08-31 08:10:40,732 p=39114 u=root n=ansible INFO| TASK [Execute ai_cmd] **********************************************************
2026-08-31 08:10:40,745 p=39114 u=root n=ansible INFO| included: /root/ansible-conf-init/tasks/execute-cmd.yml for localhost.my.domain
2026-08-31 08:10:40,778 p=39114 u=root n=ansible INFO| TASK [Display vars.] ***********************************************************
2026-08-31 08:10:40,779 p=39114 u=root n=ansible INFO| ok: [localhost.my.domain] =>
    msg: |-
        ai_at:
        ai_async: True
        ai_cmd:      ansible-pull -i hosts -U git://172.16.99.21/ansible-conf-syslogng-server -d /root/ansible-conf-syslogng-server -e "ai_vars=/root/ansible-vars" -e "ai_pull_mode=true" pb-logserv.yml &&  echo '[INFO] ansible-pull finished.'

2026-08-31 08:10:41,390 p=39114 u=root n=ansible INFO| TASK [Execute command async=3600 poll=0] ***************************************
2026-08-31 08:10:41,390 p=39114 u=root n=ansible INFO| changed: [localhost.my.domain]
2026-08-31 08:10:41,405 p=39114 u=root n=ansible INFO| PLAY RECAP *********************************************************************
2026-08-31 08:10:41,405 p=39114 u=root n=ansible INFO| localhost.my.domain        : ok=8    changed=1    unreachable=0    failed=0    skipped=3    rescued=0    ignored=0
2026-08-31 08:10:42,470 p=39178 u=root n=ansible INFO| Starting Ansible Pull at 2026-08-31 08:10:42
2026-08-31 08:10:42,470 p=39178 u=root n=ansible INFO| /usr/local/bin/ansible-pull -i hosts -U git://172.16.99.21/ansible-conf-syslogng-server -d /root/ansible-conf-syslogng-server -e ai_vars=/root/ansible-vars -e ai_pull_mode=true pb-logserv.yml
2026-08-31 08:10:43,308 p=39180 u=root n=ansible WARNING| [WARNING]: Could not match supplied host pattern, ignoring: log-server-01

2026-08-31 08:10:44,587 p=39180 u=root n=ansible INFO| localhost.my.domain | CHANGED => {
    "after": "95c7328f7be5a7af78b9525480924c072dc60642",
    "ansible_facts": {
        "discovered_interpreter_python": "/usr/local/bin/python3.12"
    },
    "before": null,
    "changed": true
}
2026-08-31 08:10:46,540 p=40141 u=root n=ansible WARNING| [WARNING]: Could not match supplied host pattern, ignoring: log-server-01

2026-08-31 08:10:47,558 p=40141 u=root n=ansible INFO| PLAY [Configure and start Log Server.] *****************************************
2026-08-31 08:10:47,620 p=40141 u=root n=ansible INFO| TASK [Include variables from ai_vars directory.] *******************************
2026-08-31 08:10:47,620 p=40141 u=root n=ansible INFO| ok: [localhost.my.domain] => (item=/root/ansible-vars/project-hosts.yml)
2026-08-31 08:10:47,694 p=40141 u=root n=ansible INFO| TASK [Display vars.] ***********************************************************
2026-08-31 08:10:47,695 p=40141 u=root n=ansible INFO| ok: [localhost.my.domain] =>
    msg: |-
        ai_pull_mode: true
        ai_vars: /root/ansible-vars

2026-08-31 08:11:14,348 p=40141 u=root n=ansible INFO| TASK [Install packages.] *******************************************************
2026-08-31 08:11:14,349 p=40141 u=root n=ansible INFO| changed: [localhost.my.domain]
2026-08-31 08:11:15,801 p=40141 u=root n=ansible INFO| TASK [vbotka.freebsd.postinstall : Rcconf: Configure syslogd_enable in /etc/rc.conf] ***
2026-08-31 08:11:15,802 p=40141 u=root n=ansible INFO| changed: [localhost.my.domain]
2026-08-31 08:11:16,011 p=40141 u=root n=ansible INFO| TASK [vbotka.freebsd.postinstall : Syslog-ng: Sanity fp_syslogng_conf is empty.] ***
2026-08-31 08:11:16,011 p=40141 u=root n=ansible INFO| ok: [localhost.my.domain]
2026-08-31 08:11:18,529 p=40141 u=root n=ansible INFO| TASK [vbotka.freebsd.postinstall : Syslog-ng: Configure /usr/local/etc/syslog-ng.conf] ***
2026-08-31 08:11:18,529 p=40141 u=root n=ansible INFO| changed: [localhost.my.domain]
2026-08-31 08:11:19,315 p=40141 u=root n=ansible INFO| TASK [vbotka.freebsd.postinstall : Rcconf: Configure syslog_ng_enable in /etc/rc.conf] ***
2026-08-31 08:11:19,315 p=40141 u=root n=ansible INFO| changed: [localhost.my.domain]
2026-08-31 08:11:20,713 p=40141 u=root n=ansible INFO| RUNNING HANDLER [vbotka.freebsd.postinstall : Start syslog-ng] *****************
2026-08-31 08:11:20,714 p=40141 u=root n=ansible INFO| changed: [localhost.my.domain]
2026-08-31 08:11:21,719 p=40141 u=root n=ansible INFO| RUNNING HANDLER [vbotka.freebsd.postinstall : Reload syslog-ng] ****************
2026-08-31 08:11:21,720 p=40141 u=root n=ansible INFO| ok: [localhost.my.domain]
2026-08-31 08:11:22,788 p=40141 u=root n=ansible INFO| RUNNING HANDLER [vbotka.freebsd.postinstall : Stop syslogd] ********************
2026-08-31 08:11:22,789 p=40141 u=root n=ansible INFO| ok: [localhost.my.domain]
2026-08-31 08:11:22,792 p=40141 u=root n=ansible INFO| PLAY RECAP *********************************************************************
2026-08-31 08:11:22,792 p=40141 u=root n=ansible INFO| localhost.my.domain        : ok=10   changed=5    unreachable=0    failed=0    skipped=19   rescued=0    ignored=0