diff --git a/docs/misc/firewall.md b/docs/misc/firewall.md index c56923afb..d6d332145 100644 --- a/docs/misc/firewall.md +++ b/docs/misc/firewall.md @@ -51,7 +51,6 @@ Pre-configured NodePorts for [DeepOps default service specs](/services) include: - `tcp/30500`: Prometheus - `tcp/30400`: AlertManager - `tcp/30200`: Grafana -- `tcp/30700`: Kibana - `tcp/30000`: Apt mirror If a service is not listed here, it will use a random port from the range. diff --git a/docs/misc/syslog.md b/docs/misc/syslog.md index d21f8369c..3685435a4 100644 --- a/docs/misc/syslog.md +++ b/docs/misc/syslog.md @@ -10,7 +10,7 @@ Centralized logging with syslog Both the Slurm and Kubernetes cluster playbooks include a minimal implementation of centralized cluster logging using rsyslog. -Rsyslog was selected for the minimal cluster logging implementation in order to provide a light-weight solution using software already installed on the nodes. This ensures that logs are recorded in one place, making it easier to debug node-specific issues even in the case where the nodes are down or non-responsive. However, for a more full-featured logging solution with search and visualization capabilities, we recommend deploying the [ELK stack](../k8s-cluster/logging.md) or other log solution. +Rsyslog was selected for the minimal cluster logging implementation in order to provide a light-weight solution using software already installed on the nodes. This ensures that logs are recorded in one place, making it easier to debug node-specific issues even in the case where the nodes are down or non-responsive. However, for a more full-featured logging solution with search and visualization capabilities, consider a site-managed log stack; for Kubernetes clusters, see the Helm-based procedure in [logging](../k8s-cluster/logging.md). In the syslog-based implementation, the first cluster management node is selected as a syslog server and listens on `rsyslog_client_tcp_port` for connections. The remaining nodes in the cluster then forward their logs to the selected syslog server. Log files for remote nodes are stored on the syslog server in node-specific files under `/var/log/deepops-hosts`. diff --git a/playbooks/slurm-cluster/logging.yml b/playbooks/slurm-cluster/logging.yml deleted file mode 100644 index bb56a77c4..000000000 --- a/playbooks/slurm-cluster/logging.yml +++ /dev/null @@ -1,86 +0,0 @@ ---- -- hosts: slurm-master[0] - become: true - vars: - elasticsearch_network_host: 0.0.0.0 - pre_tasks: - - name: debian - ensure apt cache updated - apt: - update_cache: true - when: ansible_os_family == "Debian" - roles: - - robertdebock.java - - robertdebock.elastic_repo - - robertdebock.elasticsearch - - robertdebock.logstash - - robertdebock.kibana - -- hosts: slurm-master[0] - become: true - vars: - filebeat_port: "5000" - tasks: - - name: configure logstash to accept logs from filebeat - template: - src: "filebeat.conf" - dest: "/etc/logstash/conf.d/filebeat.conf" - owner: "root" - group: "root" - mode: "0644" - -# Mitigation for CVE-2021-44228 impacting Log4j2 -# https://discuss.elastic.co/t/apache-log4j2-remote-code-execution-rce-vulnerability-cve-2021-44228-esa-2021-31/291476 -- hosts: slurm-master[0] - become: yes - tasks: - - name: configure elasticsearch to mitigate CVE-2021-44228 - copy: - src: "cve_2021_44228.options" - dest: "/etc/elasticsearch/jvm.options.d/cve_2021_44228.options" - owner: "root" - group: "root" - mode: "0644" - notify: - - restart-elasticsearch - - name: check for relevant class in logstash - shell: unzip -l /usr/share/logstash/logstash-core/lib/jars/log4j-core-2.* | grep JndiLookup.class - register: logstash_jndi - changed_when: logstash_jndi.rc == 0 - failed_when: logstash_jndi.rc == 2 - - name: configure logstash to mitigate CVE-2021-44228 - shell: zip -q -d /usr/share/logstash/logstash-core/lib/jars/log4j-core-2.* org/apache/logging/log4j/core/lookup/JndiLookup.class - notify: - - restart-logstash - when: logstash_jndi.changed - - name: manually stop logstash as restart is not consistently working later - service: - name: logstash - state: stopped - notify: - - restart-logstash - when: logstash_jndi.changed - handlers: - - name: restart-elasticsearch - service: - name: elasticsearch - state: restarted - - name: restart-logstash - service: - name: logstash - state: restarted - -- hosts: slurm-cluster - become: true - vars: - filebeat_create_config: true - filebeat_inputs: - - input_type: log - paths: - - "/var/log/*.log" - - "/var/log/syslog" - - "/var/log/slurm/*.log" - - "/var/log/apt/*.log" - filebeat_output_logstash_hosts: - - "{{ groups['slurm-master'][0] }}:5000" - roles: - - geerlingguy.filebeat diff --git a/playbooks/slurm-cluster/templates/filebeat.conf b/playbooks/slurm-cluster/templates/filebeat.conf deleted file mode 100644 index 53860ed4f..000000000 --- a/playbooks/slurm-cluster/templates/filebeat.conf +++ /dev/null @@ -1,12 +0,0 @@ -input { - beats { - port => {{ filebeat_port }} - } -} - -output { - elasticsearch { - hosts => ["http://localhost:9200"] - index => "%{[@metadata][beat]}-%{[@metadata][version]}" - } -} diff --git a/roles/requirements.yml b/roles/requirements.yml index aea5dd635..b3cb1ef94 100644 --- a/roles/requirements.yml +++ b/roles/requirements.yml @@ -30,6 +30,10 @@ roles: - src: nvidia.enroot version: "v0.5.0" +- src: https://github.com/mrlesmithjr/ansible-maas.git + name: ansible-maas + version: '178a999c9bfc979ef32c42f4f59c034664df10d0' + - src: geerlingguy.filebeat version: "3.5.0" @@ -48,6 +52,9 @@ roles: - src: robertdebock.kibana version: "1.2.6" +- src: https://github.com/DeepOps/ansible-role-chrony + name: DeepOps.chrony + version: 'c9022153036dfdde4e2b313aecde4a46cd6f6687' - src: https://github.com/OSC/ood-ansible.git version: 'v3.0.3'