diff options
Diffstat (limited to 'contrib/test/integration')
| -rw-r--r-- | contrib/test/integration/README.md | 21 | ||||
| -rw-r--r-- | contrib/test/integration/ansible.cfg | 359 | ||||
| -rw-r--r-- | contrib/test/integration/build/bats.yml | 17 | ||||
| -rw-r--r-- | contrib/test/integration/build/cri-o.yml | 79 | ||||
| -rw-r--r-- | contrib/test/integration/build/cri-tools.yml | 16 | ||||
| -rw-r--r-- | contrib/test/integration/build/kubernetes.yml | 63 | ||||
| -rw-r--r-- | contrib/test/integration/build/plugins.yml | 50 | ||||
| -rw-r--r-- | contrib/test/integration/build/runc.yml | 23 | ||||
| -rw-r--r-- | contrib/test/integration/callback_plugins/default.py | 156 | ||||
| -rw-r--r-- | contrib/test/integration/e2e.yml | 57 | ||||
| -rw-r--r-- | contrib/test/integration/golang.yml | 51 | ||||
| -rw-r--r-- | contrib/test/integration/main.yml | 58 | ||||
| -rw-r--r-- | contrib/test/integration/results.yml | 62 | ||||
| -rw-r--r-- | contrib/test/integration/system.yml | 117 | ||||
| -rw-r--r-- | contrib/test/integration/test.yml | 25 | ||||
| -rw-r--r-- | contrib/test/integration/vars.yml | 8 |
16 files changed, 1162 insertions, 0 deletions
diff --git a/contrib/test/integration/README.md b/contrib/test/integration/README.md new file mode 100644 index 000000000..f13b8b925 --- /dev/null +++ b/contrib/test/integration/README.md @@ -0,0 +1,21 @@ +# Fedora and RHEL Integration and End-to-End Tests + +This directory contains playbooks to set up for and run the integration and +end-to-end tests for CRI-O on RHEL and Fedora hosts. Two entrypoints exist: + + - `main.yml`: sets up the machine and runs tests + - `results.yml`: gathers test output to `/tmp/artifacts` + +When running `main.yml`, three tags are present: + + - `setup`: run all tasks to set up the system for testing + - `e2e`: build CRI-O from source and run Kubernetes node E2Es + - `integration`: build CRI-O from source and run the local integration suite + +The playbooks assume the following things about your system: + + - on RHEL, the server and extras repos are configured and certs are present + - `ansible` is installed and the host is boot-strapped to allow `ansible` to run against it + - the `$GOPATH` is set and present for all shells (*e.g.* written in `/etc/environment`) + - CRI-O is checked out to the correct state at `${GOPATH}/src/github.com/kubernetes-incubator/cri-o` + - the user running the playbook has access to passwordless `sudo`
\ No newline at end of file diff --git a/contrib/test/integration/ansible.cfg b/contrib/test/integration/ansible.cfg new file mode 100644 index 000000000..92a13a5fb --- /dev/null +++ b/contrib/test/integration/ansible.cfg @@ -0,0 +1,359 @@ +# config file for ansible -- http://ansible.com/ +# ============================================== + +# nearly all parameters can be overridden in ansible-playbook +# or with command line flags. ansible will read ANSIBLE_CONFIG, +# ansible.cfg in the current working directory, .ansible.cfg in +# the home directory or /etc/ansible/ansible.cfg, whichever it +# finds first + +[defaults] + +# some basic default values... + +#inventory = inventory +#library = /usr/share/my_modules/ +#remote_tmp = $HOME/.ansible/tmp +#local_tmp = .ansible/tmp +#forks = 5 +forks = 10 +#poll_interval = 15 +#sudo_user = root +#ask_sudo_pass = True +ask_sudo_pass = False +#ask_pass = True +ask_pass = False +#transport = smart +#remote_port = 22 +#module_lang = C +#module_set_locale = True + +# plays will gather facts by default, which contain information about +# the remote system. +# +# smart - gather by default, but don't regather if already gathered +# implicit - gather by default, turn off with gather_facts: False +# explicit - do not gather by default, must say gather_facts: True +#gathering = implicit +gathering = smart + +# by default retrieve all facts subsets +# all - gather all subsets +# network - gather min and network facts +# hardware - gather hardware facts (longest facts to retrieve) +# virtual - gather min and virtual facts +# facter - import facts from facter +# ohai - import facts from ohai +# You can combine them using comma (ex: network,virtual) +# You can negate them using ! (ex: !hardware,!facter,!ohai) +# A minimal set of facts is always gathered. +gather_subset = network + +# additional paths to search for roles in, colon separated +# N/B: This depends on how ansible is called +#roles_path = $WORKSPACE/kommandir_workspace/roles + +# uncomment this to disable SSH key host checking +#host_key_checking = False +host_key_checking = False + +# change the default callback +#stdout_callback = skippy +# enable additional callbacks +#callback_whitelist = timer, mail + +# Determine whether includes in tasks and handlers are "static" by +# default. As of 2.0, includes are dynamic by default. Setting these +# values to True will make includes behave more like they did in the +# 1.x versions. +task_includes_static = True +handler_includes_static = True + +# change this for alternative sudo implementations +#sudo_exe = sudo + +# What flags to pass to sudo +# WARNING: leaving out the defaults might create unexpected behaviours +#sudo_flags = -H -S -n + +# SSH timeout +#timeout = 10 + +# default user to use for playbooks if user is not specified +# (/usr/bin/ansible will use current user as default) +#remote_user = root +remote_user = root + +# logging is off by default unless this path is defined +# if so defined, consider logrotate +log_path = $ARTIFACTS/main.log + +# default module name for /usr/bin/ansible +#module_name = command + +# use this shell for commands executed under sudo +# you may need to change this to bin/bash in rare instances +# if sudo is constrained +# executable = /bin/sh + +# if inventory variables overlap, does the higher precedence one win +# or are hash values merged together? The default is 'replace' but +# this can also be set to 'merge'. +hash_behaviour = replace + +# by default, variables from roles will be visible in the global variable +# scope. To prevent this, the following option can be enabled, and only +# tasks and handlers within the role will see the variables there +private_role_vars = False + +# list any Jinja2 extensions to enable here: +#jinja2_extensions = jinja2.ext.do,jinja2.ext.i18n + +# if set, always use this private key file for authentication, same as +# if passing --private-key to ansible or ansible-playbook +#private_key_file = /path/to/file + +# If set, configures the path to the Vault password file as an alternative to +# specifying --vault-password-file on the command line. +#vault_password_file = /path/to/vault_password_file + +# format of string {{ ansible_managed }} available within Jinja2 +# templates indicates to users editing templates files will be replaced. +# replacing {file}, {host} and {uid} and strftime codes with proper values. +#ansible_managed = Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid} on {host} +# This short version is better used in templates as it won't flag the file as changed every run. +#ansible_managed = Ansible managed: {file} on {host} + +# by default, ansible-playbook will display "Skipping [host]" if it determines a task +# should not be run on a host. Set this to "False" if you don't want to see these "Skipping" +# messages. NOTE: the task header will still be shown regardless of whether or not the +# task is skipped. +#display_skipped_hosts = True +display_skipped_hosts = False + +# by default, if a task in a playbook does not include a name: field then +# ansible-playbook will construct a header that includes the task's action but +# not the task's args. This is a security feature because ansible cannot know +# if the *module* considers an argument to be no_log at the time that the +# header is printed. If your environment doesn't have a problem securing +# stdout from ansible-playbook (or you have manually specified no_log in your +# playbook on all of the tasks where you have secret information) then you can +# safely set this to True to get more informative messages. +display_args_to_stdout = False + +# by default (as of 1.3), Ansible will raise errors when attempting to dereference +# Jinja2 variables that are not set in templates or action lines. Uncomment this line +# to revert the behavior to pre-1.3. +#error_on_undefined_vars = False + +# by default (as of 1.6), Ansible may display warnings based on the configuration of the +# system running ansible itself. This may include warnings about 3rd party packages or +# other conditions that should be resolved if possible. +# to disable these warnings, set the following value to False: +system_warnings = False + +# by default (as of 1.4), Ansible may display deprecation warnings for language +# features that should no longer be used and will be removed in future versions. +# to disable these warnings, set the following value to False: +deprecation_warnings = False + +# (as of 1.8), Ansible can optionally warn when usage of the shell and +# command module appear to be simplified by using a default Ansible module +# instead. These warnings can be silenced by adjusting the following +# setting or adding warn=yes or warn=no to the end of the command line +# parameter string. This will for example suggest using the git module +# instead of shelling out to the git command. +command_warnings = False + + +# set plugin path directories here, separate with colons +#action_plugins = /usr/share/ansible/plugins/action +#callback_plugins = /usr/share/ansible/plugins/callback +#connection_plugins = /usr/share/ansible/plugins/connection +#lookup_plugins = /usr/share/ansible/plugins/lookup +#vars_plugins = /usr/share/ansible/plugins/vars +#filter_plugins = /usr/share/ansible/plugins/filter +#test_plugins = /usr/share/ansible/plugins/test +#strategy_plugins = /usr/share/ansible/plugins/strategy + +# Most callbacks shipped with Ansible are disabled by default +# and need to be whitelisted in your ansible.cfg file in order to function. +callback_whitelist = default + +# by default callbacks are not loaded for /bin/ansible, enable this if you +# want, for example, a notification or logging callback to also apply to +# /bin/ansible runs +#bin_ansible_callbacks = False + + +# don't like cows? that's unfortunate. +# set to 1 if you don't want cowsay support or export ANSIBLE_NOCOWS=1 +#nocows = 1 + +# set which cowsay stencil you'd like to use by default. When set to 'random', +# a random stencil will be selected for each task. The selection will be filtered +# against the `cow_whitelist` option below. +#cow_selection = default +#cow_selection = random + +# when using the 'random' option for cowsay, stencils will be restricted to this list. +# it should be formatted as a comma-separated list with no spaces between names. +# NOTE: line continuations here are for formatting purposes only, as the INI parser +# in python does not support them. +#cow_whitelist=bud-frogs,bunny,cheese,daemon,default,dragon,elephant-in-snake,elephant,eyes,\ +# hellokitty,kitty,luke-koala,meow,milk,moofasa,moose,ren,sheep,small,stegosaurus,\ +# stimpy,supermilker,three-eyes,turkey,turtle,tux,udder,vader-koala,vader,www + +# don't like colors either? +# set to 1 if you don't want colors, or export ANSIBLE_NOCOLOR=1 +nocolor = 0 + +# if set to a persistent type (not 'memory', for example 'redis') fact values +# from previous runs in Ansible will be stored. This may be useful when +# wanting to use, for example, IP information from one group of servers +# without having to talk to them in the same playbook run to get their +# current IP information. +#fact_caching = memory + +# retry files +# When a playbook fails by default a .retry file will be created in ~/ +# You can disable this feature by setting retry_files_enabled to False +# and you can change the location of the files by setting retry_files_save_path + +#retry_files_enabled = False +retry_files_enabled = False + +# squash actions +# Ansible can optimise actions that call modules with list parameters +# when looping. Instead of calling the module once per with_ item, the +# module is called once with all items at once. Currently this only works +# under limited circumstances, and only with parameters named 'name'. +squash_actions = apk,apt,dnf,package,pacman,pkgng,yum,zypper + +# prevents logging of task data, off by default +#no_log = False + +# prevents logging of tasks, but only on the targets, data is still logged on the master/controller +no_target_syslog = True + +# controls whether Ansible will raise an error or warning if a task has no +# choice but to create world readable temporary files to execute a module on +# the remote machine. This option is False by default for security. Users may +# turn this on to have behaviour more like Ansible prior to 2.1.x. See +# https://docs.ansible.com/ansible/become.html#becoming-an-unprivileged-user +# for more secure ways to fix this than enabling this option. +#allow_world_readable_tmpfiles = False + +# controls the compression level of variables sent to +# worker processes. At the default of 0, no compression +# is used. This value must be an integer from 0 to 9. +#var_compression_level = 9 + +# controls what compression method is used for new-style ansible modules when +# they are sent to the remote system. The compression types depend on having +# support compiled into both the controller's python and the client's python. +# The names should match with the python Zipfile compression types: +# * ZIP_STORED (no compression. available everywhere) +# * ZIP_DEFLATED (uses zlib, the default) +# These values may be set per host via the ansible_module_compression inventory +# variable +#module_compression = 'ZIP_DEFLATED' + +# This controls the cutoff point (in bytes) on --diff for files +# set to 0 for unlimited (RAM may suffer!). +#max_diff_size = 1048576 + +[privilege_escalation] +#become=True +#become_method=sudo +#become_user=root +become_user=root +#become_ask_pass=False + +[paramiko_connection] + +# uncomment this line to cause the paramiko connection plugin to not record new host +# keys encountered. Increases performance on new host additions. Setting works independently of the +# host key checking setting above. +#record_host_keys=False + +# by default, Ansible requests a pseudo-terminal for commands executed under sudo. Uncomment this +# line to disable this behaviour. +#pty=False + +[ssh_connection] + +# ssh arguments to use +# Leaving off ControlPersist will result in poor performance, so use +# paramiko on older platforms rather than removing it +ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o UserKnownHostsFile=/dev/null -o PreferredAuthentications=publickey -o ConnectTimeout=13 + +# The path to use for the ControlPath sockets. This defaults to +# "%(directory)s/ansible-ssh-%%h-%%p-%%r", however on some systems with +# very long hostnames or very long path names (caused by long user names or +# deeply nested home directories) this can exceed the character limit on +# file socket names (108 characters for most platforms). In that case, you +# may wish to shorten the string below. +# +# Example: +# control_path = %(directory)s/%%h-%%r +#control_path = %(directory)s/ansible-ssh-%%h-%%p-%%r + +# Enabling pipelining reduces the number of SSH operations required to +# execute a module on the remote server. This can result in a significant +# performance improvement when enabled, however when using "sudo:" you must +# first disable 'requiretty' in /etc/sudoers +# +# By default, this option is disabled to preserve compatibility with +# sudoers configurations that have requiretty (the default on many distros). +# +#pipelining = False +pipelining=True + +# if True, make ansible use scp if the connection type is ssh +# (default is sftp) +#scp_if_ssh = True + +# if False, sftp will not use batch mode to transfer files. This may cause some +# types of file transfer failures impossible to catch however, and should +# only be disabled if your sftp version has problems with batch mode +#sftp_batch_mode = False + +[accelerate] +#accelerate_port = 5099 +#accelerate_timeout = 30 +#accelerate_connect_timeout = 5.0 + +# The daemon timeout is measured in minutes. This time is measured +# from the last activity to the accelerate daemon. +#accelerate_daemon_timeout = 30 + +# If set to yes, accelerate_multi_key will allow multiple +# private keys to be uploaded to it, though each user must +# have access to the system via SSH to add a new key. The default +# is "no". +#accelerate_multi_key = yes + +[selinux] +# file systems that require special treatment when dealing with security context +# the default behaviour that copies the existing context or uses the user default +# needs to be changed to use the file system dependent context. +#special_context_filesystems=nfs,vboxsf,fuse,ramfs + +# Set this to yes to allow libvirt_lxc connections to work without SELinux. +#libvirt_lxc_noseclabel = yes + +[colors] +#highlight = white +#verbose = blue +#warn = bright purple +#error = red +#debug = dark gray +#deprecate = purple +#skip = cyan +#unreachable = red +#ok = green +#changed = yellow +#diff_add = green +#diff_remove = red +#diff_lines = cyan diff --git a/contrib/test/integration/build/bats.yml b/contrib/test/integration/build/bats.yml new file mode 100644 index 000000000..d4ea19c65 --- /dev/null +++ b/contrib/test/integration/build/bats.yml @@ -0,0 +1,17 @@ +--- + +- name: clone bats source repo + git: + repo: "https://github.com/sstephenson/bats.git" + dest: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats" + +- name: install bats + command: "./install.sh /usr/local" + args: + chdir: "{{ ansible_env.GOPATH }}/src/github.com/sstephenson/bats" + +- name: link bats + file: + src: /usr/local/bin/bats + dest: /usr/bin/bats + state: link diff --git a/contrib/test/integration/build/cri-o.yml b/contrib/test/integration/build/cri-o.yml new file mode 100644 index 000000000..fa025035c --- /dev/null +++ b/contrib/test/integration/build/cri-o.yml @@ -0,0 +1,79 @@ +--- + +- name: stat the expected cri-o directory + stat: + path: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + register: dir_stat + +- name: expect cri-o to be cloned already + fail: + msg: "Expected cri-o to be cloned at {{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o but it wasn't!" + when: not dir_stat.stat.exists + +- name: install cri-o tools + make: + target: install.tools + chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + +- name: build cri-o + make: + chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + +- name: install cri-o + make: + target: install + chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + +- name: install cri-o systemd files + make: + target: install.systemd + chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + +- name: install cri-o config + make: + target: install.config + chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + +- name: install configs + copy: + src: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o/{{ item.src }}" + dest: "{{ item.dest }}" + remote_src: yes + with_items: + - src: contrib/cni/10-crio-bridge.conf + dest: /etc/cni/net.d/10-crio-bridge.conf + - src: contrib/cni/99-loopback.conf + dest: /etc/cni/net.d/99-loopback.conf + - src: test/redhat_sigstore.yaml + dest: /etc/containers/registries.d/registry.access.redhat.com.yaml + +- name: run with overlay + replace: + regexp: 'storage_driver = ""' + replace: 'storage_driver = "overlay"' + name: /etc/crio/crio.conf + backup: yes + +- name: run with systemd cgroup manager + replace: + regexp: 'cgroup_manager = "cgroupfs"' + replace: 'cgroup_manager = "systemd"' + name: /etc/crio/crio.conf + backup: yes + +- name: add docker.io default registry + lineinfile: + dest: /etc/crio/crio.conf + line: '"docker.io"' + insertafter: 'registries = \[' + regexp: 'docker\.io' + state: present + +- name: add overlay storage opts on RHEL/CentOS + lineinfile: + dest: /etc/crio/crio.conf + line: '"overlay.override_kernel_check=1"' + insertafter: 'storage_option = \[' + regexp: 'overlay\.override_kernel_check=1' + state: present + when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' diff --git a/contrib/test/integration/build/cri-tools.yml b/contrib/test/integration/build/cri-tools.yml new file mode 100644 index 000000000..e314225ef --- /dev/null +++ b/contrib/test/integration/build/cri-tools.yml @@ -0,0 +1,16 @@ +--- + +- name: clone cri-tools source repo + git: + repo: "https://github.com/kubernetes-incubator/cri-tools.git" + dest: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-tools" + version: "16e6fe4d7199c5689db4630a9330e6a8a12cecd1" + +- name: install crictl + command: "/usr/bin/go install github.com/kubernetes-incubator/cri-tools/cmd/crictl" + +- name: link crictl + file: + src: "{{ ansible_env.GOPATH }}/bin/crictl" + dest: /usr/bin/crictl + state: link diff --git a/contrib/test/integration/build/kubernetes.yml b/contrib/test/integration/build/kubernetes.yml new file mode 100644 index 000000000..206cba448 --- /dev/null +++ b/contrib/test/integration/build/kubernetes.yml @@ -0,0 +1,63 @@ +--- + +- name: clone kubernetes source repo + git: + repo: "https://github.com/runcom/kubernetes.git" + dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + version: "cri-o-node-e2e-patched" + +- name: install etcd + command: "hack/install-etcd.sh" + args: + chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + +- name: build kubernetes + make: + chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" + +- name: Add custom cluster service file for the e2e testing + copy: + dest: /etc/systemd/system/customcluster.service + content: | + [Unit] + After=network-online.target + Wants=network-online.target + [Service] + WorkingDirectory={{ ansible_env.GOPATH }}/src/k8s.io/kubernetes + ExecStart=/usr/local/bin/createcluster.sh + User=root + [Install] + WantedBy=multi-user.target + +- name: Add create cluster background script for e2e testing + copy: + dest: /usr/local/bin/createcluster.sh + content: | + #!/bin/bash + + export PATH=/usr/local/go/bin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/bin:{{ ansible_env.GOPATH }}/bin:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/third_party/etcd:{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/local/bin/linux/amd64/ + export CONTAINER_RUNTIME=remote + export CGROUP_DRIVER=systemd + export CONTAINER_RUNTIME_ENDPOINT='/var/run/crio.sock --runtime-request-timeout=5m' + export ALLOW_SECURITY_CONTEXT="," + export ALLOW_PRIVILEGED=1 + export DNS_SERVER_IP={{ ansible_eth0.ipv4.address }} + export API_HOST={{ ansible_eth0.ipv4.address }} + export API_HOST_IP={{ ansible_eth0.ipv4.address }} + export KUBE_ENABLE_CLUSTER_DNS=true + ./hack/local-up-cluster.sh + mode: "u=rwx,g=rwx,o=x" + +- name: Set kubernetes_provider to be local + lineinfile: + dest: /etc/environment + line: 'KUBERNETES_PROVIDER=local' + regexp: 'KUBERNETES_PROVIDER=' + state: present + +- name: Set KUBECONFIG + lineinfile: + dest: /etc/environment + line: 'KUBECONFIG=/var/run/kubernetes/admin.kubeconfig' + regexp: 'KUBECONFIG=' + state: present diff --git a/contrib/test/integration/build/plugins.yml b/contrib/test/integration/build/plugins.yml new file mode 100644 index 000000000..e342a0b91 --- /dev/null +++ b/contrib/test/integration/build/plugins.yml @@ -0,0 +1,50 @@ +--- + +- name: clone plugins source repo + git: + repo: "https://github.com/containernetworking/plugins.git" + dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + version: "dcf7368eeab15e2affc6256f0bb1e84dd46a34de" + +- name: build plugins + command: "./build.sh" + args: + chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + +- name: install plugins + copy: + src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/{{ item }}" + dest: "/opt/cni/bin" + mode: "o=rwx,g=rx,o=rx" + remote_src: yes + with_items: + - bridge + - dhcp + - flannel + - host-local + - ipvlan + - loopback + - macvlan + - ptp + - sample + - tuning + - vlan + +- name: clone runcom plugins source repo + git: + repo: "https://github.com/runcom/plugins.git" + dest: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + version: "custom-bridge" + force: yes + +- name: build plugins + command: "./build.sh" + args: + chdir: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins" + +- name: install custom bridge + copy: + src: "{{ ansible_env.GOPATH }}/src/github.com/containernetworking/plugins/bin/bridge" + dest: "/opt/cni/bin/bridge-custom" + mode: "o=rwx,g=rx,o=rx" + remote_src: yes diff --git a/contrib/test/integration/build/runc.yml b/contrib/test/integration/build/runc.yml new file mode 100644 index 000000000..7bb0491d4 --- /dev/null +++ b/contrib/test/integration/build/runc.yml @@ -0,0 +1,23 @@ +--- + +- name: clone runc source repo + git: + repo: "https://github.com/opencontainers/runc.git" + dest: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + version: "84a082bfef6f932de921437815355186db37aeb1" + +- name: build runc + make: + params: BUILDTAGS="seccomp selinux" + chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + +- name: install runc + make: + target: "install" + chdir: "{{ ansible_env.GOPATH }}/src/github.com/opencontainers/runc" + +- name: link runc + file: + src: /usr/local/sbin/runc + dest: /usr/bin/runc + state: link diff --git a/contrib/test/integration/callback_plugins/default.py b/contrib/test/integration/callback_plugins/default.py new file mode 100644 index 000000000..99821f660 --- /dev/null +++ b/contrib/test/integration/callback_plugins/default.py @@ -0,0 +1,156 @@ +'''Plugin to override the default output logic.''' + +# upstream: https://gist.github.com/cliffano/9868180 + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +# For some reason this has to be done +import imp +import os + +ANSIBLE_PATH = imp.find_module('ansible')[1] +DEFAULT_PATH = os.path.join(ANSIBLE_PATH, 'plugins/callback/default.py') +DEFAULT_MODULE = imp.load_source( + 'ansible.plugins.callback.default', + DEFAULT_PATH +) + +try: + from ansible.plugins.callback import CallbackBase + BASECLASS = CallbackBase +except ImportError: # < ansible 2.1 + BASECLASS = DEFAULT_MODULE.CallbackModule + + +class CallbackModule(DEFAULT_MODULE.CallbackModule): # pylint: disable=too-few-public-methods,no-init + ''' + Override for the default callback module. + + Render std err/out outside of the rest of the result which it prints with + indentation. + ''' + CALLBACK_VERSION = 2.0 + CALLBACK_TYPE = 'stdout' + CALLBACK_NAME = 'default' + + def __init__(self, *args, **kwargs): + # pylint: disable=non-parent-init-called + BASECLASS.__init__(self, *args, **kwargs) + self.failed_task = [] + self.result_file = os.environ.get('AHT_RESULT_FILE') + + def _dump_results(self, result): + '''Return the text to output for a result.''' + result['_ansible_verbose_always'] = True + + save = {} + for key in ['stdout', 'stdout_lines', 'stderr', 'stderr_lines', 'msg']: + if key in result: + save[key] = result.pop(key) + + output = BASECLASS._dump_results(self, result) # pylint: disable=protected-access + + for key in ['stdout', 'stderr', 'msg']: + if key in save and save[key]: + output += '\n\n%s:\n---\n%s\n---' % (key.upper(), save[key]) + + for key, value in save.items(): + result[key] = value + + return output + + def v2_runner_on_unreachable(self, result): + self.failed_task = result + + if self._play.strategy == 'free' and self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + delegated_vars = result._result.get('_ansible_delegated_vars', None) + if delegated_vars: + self._display.display("fatal: [%s -> %s]: UNREACHABLE! => %s" % (result._host.get_name(), delegated_vars['ansible_host'], self._dump_results(result._result)), color=C.COLOR_UNREACHABLE) + else: + self._display.display("fatal: [%s]: UNREACHABLE! => %s" % (result._host.get_name(), self._dump_results(result._result)), color=C.COLOR_UNREACHABLE) + + def v2_runner_on_failed(self,result, ignore_errors=False): + if ignore_errors is not True: + # Sets environment variable for test failures for use in playboks. + # Handlers tasks can conditionalize themselves using this variable + # to run only on failure. + os.environ["AHT_FAILURE"] = "1" + + # Save last failure + self.failed_task = result + + if self._play.strategy == 'free' and self._last_task_banner != result._task._uuid: + self._print_task_banner(result._task) + + delegated_vars = result._result.get('_ansible_delegated_vars', None) + if 'exception' in result._result: + if self._display.verbosity < 3: + # extract just the actual error message from the exception text + error = result._result['exception'].strip().split('\n')[-1] + msg = "An exception occurred during task execution. To see the full traceback, use -vvv. The error was: %s" % error + else: + msg = "An exception occurred during task execution. The full traceback is:\n" + result._result['exception'] + + self._display.display(msg, color=C.COLOR_ERROR) + + if result._task.loop and 'results' in result._result: + self._process_items(result) + + else: + if delegated_vars: + self._display.display("fatal: [%s -> %s]: FAILED! => %s" % (result._host.get_name(), delegated_vars['ansible_host'], self._dump_results(result._result)), color=C.COLOR_ERROR) + else: + self._display.display("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), self._dump_results(result._result)), color=C.COLOR_ERROR) + + if ignore_errors: + self._display.display("...ignoring", color=C.COLOR_SKIP) + + def v2_playbook_on_stats(self, stats): + self._display.banner("PLAY RECAP") + + hosts = sorted(stats.processed.keys()) + for h in hosts: + t = stats.summarize(h) + + self._display.display(u"%s : %s %s %s %s" % ( + hostcolor(h, t), + colorize(u'ok', t['ok'], C.COLOR_OK), + colorize(u'changed', t['changed'], C.COLOR_CHANGED), + colorize(u'unreachable', t['unreachable'], C.COLOR_UNREACHABLE), + colorize(u'failed', t['failures'], C.COLOR_ERROR)), + screen_only=True + ) + + self._display.display(u"%s : %s %s %s %s" % ( + hostcolor(h, t, False), + colorize(u'ok', t['ok'], None), + colorize(u'changed', t['changed'], None), + colorize(u'unreachable', t['unreachable'], None), + colorize(u'failed', t['failures'], None)), + log_only=True + ) + + self._display.display("", screen_only=True) + # Save result to file if environment variable exists + if self.result_file is not None: + if self.failed_task: + with open(self.result_file, 'w') as f: + f.write("PLAY: %s\n%s\n%s" % (self._play, \ + self.failed_task._task, \ + self._dump_results(self.failed_task._result))) + else: + open(self.result_file, 'w').close() diff --git a/contrib/test/integration/e2e.yml b/contrib/test/integration/e2e.yml new file mode 100644 index 000000000..5c4d656ef --- /dev/null +++ b/contrib/test/integration/e2e.yml @@ -0,0 +1,57 @@ +--- + +- name: enable and start CRI-O + systemd: + name: crio + state: started + enabled: yes + daemon_reload: yes + +- name: update the server address for the custom cluster + lineinfile: + dest: /usr/local/bin/createcluster.sh + line: "export {{ item }}={{ ansible_eth0.ipv4.address }}" + regexp: "^export {{ item }}=" + state: present + with_items: + - DNS_SERVER_IP + - API_HOST + - API_HOST_IP + +- name: enable and start the custom cluster + systemd: + name: customcluster.service + state: started + enabled: yes + daemon_reload: yes + +- name: wait for the cluster to be running + command: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/_output/bin/kubectl get service kubernetes --namespace default" + register: kube_poll + until: kube_poll | succeeded + retries: 100 + delay: 30 + +- name: ensure directory exists for e2e reports + file: + path: "{{ artifacts }}" + state: directory + +- name: Buffer the e2e testing command to workaround Ansible YAML folding "feature" + set_fact: + e2e_shell_cmd: > + /usr/bin/go run hack/e2e.go + --test + --test_args="-host=https://{{ ansible_default_ipv4.address }}:6443 + --ginkgo.focus=\[Conformance\] + --report-dir={{ artifacts }}" + &> {{ artifacts }}/e2e.log + # Fix vim syntax hilighting: " + +- name: disable SELinux + command: setenforce 0 + +- name: run e2e tests + shell: "{{ e2e_shell_cmd | regex_replace('\\s+', ' ') }}" + args: + chdir: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes" diff --git a/contrib/test/integration/golang.yml b/contrib/test/integration/golang.yml new file mode 100644 index 000000000..63e556977 --- /dev/null +++ b/contrib/test/integration/golang.yml @@ -0,0 +1,51 @@ +--- + +- name: fetch Golang + unarchive: + remote_src: yes + src: https://storage.googleapis.com/golang/go1.8.4.linux-amd64.tar.gz + dest: /usr/local + +- name: link go toolchain + file: + src: "/usr/local/go/bin/{{ item }}" + dest: "/usr/bin/{{ item }}" + state: link + with_items: + - go + - gofmt + - godoc + +- name: ensure user profile exists + file: + path: "{{ ansible_user_dir }}/.profile" + state: touch + +- name: set up PATH for Go toolchain and built binaries + lineinfile: + dest: "{{ ansible_user_dir }}/.profile" + line: 'PATH={{ ansible_env.PATH }}:{{ ansible_env.GOPATH }}/bin:/usr/local/go/bin' + regexp: '^PATH=' + state: present + +- name: set up directories + file: + path: "{{ item }}" + state: directory + with_items: + - "{{ ansible_env.GOPATH }}/src/github.com/containernetworking" + - "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator" + - "{{ ansible_env.GOPATH }}/src/github.com/k8s.io" + - "{{ ansible_env.GOPATH }}/src/github.com/sstephenson" + - "{{ ansible_env.GOPATH }}/src/github.com/opencontainers" + +- name: install Go tools and dependencies + shell: /usr/bin/go get -u "github.com/{{ item }}" + with_items: + - tools/godep + - onsi/ginkgo/ginkgo + - onsi/gomega + - cloudflare/cfssl/cmd/... + - jteeuwen/go-bindata/go-bindata + - vbatts/git-validation + - cpuguy83/go-md2man diff --git a/contrib/test/integration/main.yml b/contrib/test/integration/main.yml new file mode 100644 index 000000000..ce4a206fb --- /dev/null +++ b/contrib/test/integration/main.yml @@ -0,0 +1,58 @@ +- hosts: all + remote_user: root + vars_files: + - "{{ playbook_dir }}/vars.yml" + tags: + - setup + tasks: + - name: set up the system + include: system.yml + + - name: install Golang tools + include: golang.yml + + - name: clone build and install bats + include: "build/bats.yml" + + - name: clone build and install cri-tools + include: "build/cri-tools.yml" + + - name: clone build and install kubernetes + include: "build/kubernetes.yml" + + - name: clone build and install runc + include: "build/runc.yml" + + - name: clone build and install networking plugins + include: "build/plugins.yml" + +- hosts: all + remote_user: root + vars_files: + - "{{ playbook_dir }}/vars.yml" + tags: + - integration + - e2e + tasks: + - name: clone build and install cri-o + include: "build/cri-o.yml" + +- hosts: all + remote_user: root + vars_files: + - "{{ playbook_dir }}/vars.yml" + tags: + - integration + tasks: + - name: run cri-o integration tests + include: test.yml + +- hosts: all + remote_user: root + vars_files: + - "{{ playbook_dir }}/vars.yml" + tags: + - e2e + tasks: + - name: run k8s e2e tests + include: e2e.yml diff --git a/contrib/test/integration/results.yml b/contrib/test/integration/results.yml new file mode 100644 index 000000000..c9a96abb1 --- /dev/null +++ b/contrib/test/integration/results.yml @@ -0,0 +1,62 @@ +--- +# vim-syntax: ansible + +- hosts: '{{ hosts | default("all") }}' + vars_files: + - "{{ playbook_dir }}/vars.yml" + vars: + _result_filepaths: [] # do not use + _dstfnbuff: [] # do not use + tasks: + - name: The crio_integration_filepath is required + tags: + - integration + set_fact: + _result_filepaths: "{{ _result_filepaths + [crio_integration_filepath] }}" + + - name: The crio_node_e2e_filepath is required + tags: + - e2e + set_fact: + _result_filepaths: "{{ _result_filepaths + [crio_node_e2e_filepath] }}" + + - name: Verify expectations + assert: + that: + - 'result_dest_basedir | default(False, True)' + - '_result_filepaths | default(False, True)' + - '_dstfnbuff == []' + - 'results_fetched is undefined' + + - name: Results directory exists + file: + path: "{{ result_dest_basedir }}" + state: directory + delegate_to: localhost + + - name: destination file paths are buffered for overwrite-checking and jUnit conversion + set_fact: + _dstfnbuff: > + {{ _dstfnbuff | + union( [result_dest_basedir ~ "/" ~ inventory_hostname ~ "/" ~ item | basename] ) }} + with_items: '{{ _result_filepaths }}' + + - name: Overwriting existing results assumed very very bad + fail: + msg: "Cowardly refusing to overwrite {{ item }}" + when: item | exists + delegate_to: localhost + with_items: '{{ _dstfnbuff }}' + + # fetch module doesn't support directories + - name: Retrieve results from all hosts + synchronize: + checksum: True # Don't rely on date/time being in sync + archive: False # Don't bother with permissions or times + copy_links: True # We want files, not links to files + recursive: True + mode: pull + dest: '{{ result_dest_basedir }}/{{ inventory_hostname }}/' # must end in / + src: '{{ item }}' + register: results_fetched + with_items: '{{ _result_filepaths }}' diff --git a/contrib/test/integration/system.yml b/contrib/test/integration/system.yml new file mode 100644 index 000000000..d07ae0c82 --- /dev/null +++ b/contrib/test/integration/system.yml @@ -0,0 +1,117 @@ +--- + +- name: Make sure we have all required packages + package: + name: "{{ item }}" + state: present + with_items: + - container-selinux + - curl + - device-mapper-devel + - expect + - findutils + - gcc + - git + - glib2-devel + - glibc-devel + - glibc-static + - gpgme-devel + - hostname + - iproute + - iptables + - krb5-workstation + - libassuan-devel + - libffi-devel + - libgpg-error-devel + - libguestfs-tools + - libseccomp-devel + - libvirt-client + - libvirt-python + - libxml2-devel + - libxslt-devel + - make + - mlocate + - nfs-utils + - nmap-ncat + - oci-register-machine + - oci-systemd-hook + - oci-umount + - openssl + - openssl-devel + - ostree-devel + - pkgconfig + - python + - python2-boto + - python2-crypto + - python-devel + - python-virtualenv + - PyYAML + - redhat-rpm-config + - rpcbind + - rsync + - sed + - skopeo-containers + - socat + - tar + - wget + async: 600 + poll: 10 + +- name: Add Btrfs for Fedora + package: + name: "{{ item }}" + state: present + with_items: + - btrfs-progs-devel + when: ansible_distribution in ['Fedora'] + +- name: Update all packages + package: + name: '*' + state: latest + async: 600 + poll: 10 + +- name: Setup swap to prevent kernel firing off the OOM killer + shell: | + truncate -s 8G /root/swap && \ + export SWAPDEV=$(losetup --show -f /root/swap | head -1) && \ + mkswap $SWAPDEV && \ + swapon $SWAPDEV && \ + swapon --show + +- name: ensure directories exist as needed + file: + path: "{{ item }}" + state: directory + with_items: + - /opt/cni/bin + - /etc/cni/net.d + +- name: set sysctl vm.overcommit_memory=1 for CentOS + sysctl: + name: vm.overcommit_memory + state: present + value: 1 + when: ansible_distribution == 'CentOS' + +- name: inject hostname into /etc/hosts + lineinfile: + dest: /etc/hosts + line: '{{ ansible_default_ipv4.address }} {{ ansible_nodename }}' + insertafter: 'EOF' + regexp: '{{ ansible_default_ipv4.address }}\s+{{ ansible_nodename }}' + state: present + +- name: Flush the iptables + command: iptables -F + +- name: Enable localnet routing + command: sysctl -w net.ipv4.conf.all.route_localnet=1 + +- name: Add masquerade for localhost + command: iptables -t nat -I POSTROUTING -s 127.0.0.1 ! -d 127.0.0.1 -j MASQUERADE + +- name: Update the kernel cmdline to include quota support + command: grubby --update-kernel=ALL --args="rootflags=pquota" + when: ansible_distribution in ['RedHat', 'CentOS'] diff --git a/contrib/test/integration/test.yml b/contrib/test/integration/test.yml new file mode 100644 index 000000000..418ceff78 --- /dev/null +++ b/contrib/test/integration/test.yml @@ -0,0 +1,25 @@ +--- + +- name: Make testing output verbose so it can be converted to xunit + lineinfile: + dest: "{{ ansible_env.GOPATH }}/src/k8s.io/kubernetes/hack/make-rules/test.sh" + line: ' go test -v "${goflags[@]:+${goflags[@]}}" \' + regexp: ' go test \"\$' + state: present + +- name: set extra storage options + set_fact: + extra_storage_opts: " --storage-opt overlay.override_kernel_check=1" + when: ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS' + +- name: ensure directory exists for e2e reports + file: + path: "{{ artifacts }}" + state: directory + +- name: run integration tests + shell: "CGROUP_MANAGER=cgroupfs STORAGE_OPTIONS='--storage-driver=overlay{{ extra_storage_opts | default('') }}' make localintegration >& {{ artifacts }}/testout.txt" + args: + chdir: "{{ ansible_env.GOPATH }}/src/github.com/kubernetes-incubator/cri-o" + async: 5400 + poll: 30 diff --git a/contrib/test/integration/vars.yml b/contrib/test/integration/vars.yml new file mode 100644 index 000000000..f1e5e2f73 --- /dev/null +++ b/contrib/test/integration/vars.yml @@ -0,0 +1,8 @@ +--- + +# For results.yml Paths use rsync 'source' conventions +artifacts: "/tmp/artifacts" # Base-directory for collection +crio_integration_filepath: "{{ artifacts }}/testout.txt" +crio_node_e2e_filepath: "{{ artifacts }}/junit_01.xml" +result_dest_basedir: '{{ lookup("env","WORKSPACE") | + default(playbook_dir, True) }}/artifacts' |