Hi
I am having an odd situation, A single service (out of +7K) is not
sending Critical notifications, it does send the recovery for that
service, but when it turns critical, the system shows as if a
notification is sent, but any record for that is not shown in the debug
log or the normal log, and not is the email arrives ("OK" emails show
and arrive)
Here is the Service and Host Object data:
Object 'tools.foo.com' of type 'Host':
% declared in '/var/lib/icinga2/api/zones/mtl/director/hosts.conf',
lines 14421:1-14421:32
* __name = "tools.foo.com"
* action_url = ""
* address = "10.0.11.186"
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/hosts.conf', lines 14425:5-14425:27
* address6 = ""
* check_command = "hostalive4"
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 2:5-2:31
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
4:5-4:32
* check_interval = 60
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 5:5-5:24
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
6:5-6:23
* check_period = "24x7"
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 4:5-4:25
* check_timeout = 15
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 7:5-7:23
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
8:5-8:23
* command_endpoint = ""
* display_name = "tools.foo.com"
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/hosts.conf', lines 14424:5-14424:39
* enable_active_checks = true
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 9:5-9:31
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
10:5-10:31
* enable_event_handler = true
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 11:5-11:31
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
12:5-12:31
* enable_flapping = false
* enable_notifications = true
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 8:5-8:31
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
9:5-9:31
* enable_passive_checks = false
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 10:5-10:33
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
11:5-11:33
* enable_perfdata = false
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 12:5-12:27
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
13:5-13:27
* event_command = ""
* flapping_threshold = 0
* flapping_threshold_high = 30
* flapping_threshold_low = 25
* groups = [ "foo_tools_IS" ]
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/hosts.conf', lines 14426:5-14426:36
* icon_image = ""
* icon_image_alt = ""
* max_check_attempts = 3
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 3:5-3:28
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
5:5-5:28
* name = "tools.foo.com"
* notes = ""
* notes_url = ""
* package = "_cluster"
* retry_interval = 30
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 6:5-6:23
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
7:5-7:24
* source_location
* first_column = 1
* first_line = 14421
* last_column = 32
* last_line = 14421
* path = "/var/lib/icinga2/api/zones/mtl/director/hosts.conf"
* templates = [ "tools.foo.com", "mtl-host", "generic-host" ]
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/hosts.conf', lines 14421:1-14421:32
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
1:0-1:23
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 1:0-1:27
* type = "Host"
* vars
* email_alert = true
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 14:5-14:27
* enable_sms = false
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 15:5-15:27
* notified_users = [ "taldr" ]
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/hosts.conf', lines 14427:5-14427:37
* user_groups = [ "foo_Tools_IS" ]
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/hosts.conf', lines 14428:5-14428:46
* zone = "mtl"
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
15:5-15:21
* volatile = false
% = modified in
'/var/lib/icinga2/api/zones/director-global/director/host_templates.conf',
lines 13:5-13:20
% = modified in
'/var/lib/icinga2/api/zones/mtl/director/host_templates.conf', lines
14:5-14:20
* zone = "mtl"
Object 'tools.foo.com!tools.foo.com_lunch_menu' of type 'Service':
% declared in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 37:1-37:46
* __name = "tools.foo.com!tools.foo.com_lunch_menu"
* action_url = ""
* check_command = "http"
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 68:5-68:26
* check_interval = 60
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 26:5-26:23
* check_period = ""
* check_timeout = 15
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 28:5-28:23
* command_endpoint = ""
* display_name = "tools.foo.com_lunch_menu"
* enable_active_checks = true
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 30:5-30:31
* enable_event_handler = false
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 32:5-32:32
* enable_flapping = false
* enable_notifications = true
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 29:5-29:31
* enable_passive_checks = false
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 31:5-31:33
* enable_perfdata = false
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 33:5-33:27
* event_command = ""
* flapping_threshold = 0
* flapping_threshold_high = 30
* flapping_threshold_low = 25
* groups = [ ]
* host_name = "tools.foo.com"
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 38:5-38:36
* icon_image = ""
* icon_image_alt = ""
* max_check_attempts = 3
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 25:5-25:28
* name = "tools.foo.com_lunch_menu"
* notes = ""
* notes_url = ""
* package = "director"
* retry_interval = 30
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 27:5-27:24
* source_location
* first_column = 1
* first_line = 37
* last_column = 46
* last_line = 37
* path =
"/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf"
* templates = [ "tools.foo.com_lunch_menu", "http-custom-check-tmpl",
"generic-tmpl" ]
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 37:1-37:46
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 65:1-65:41
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 24:1-24:31
* type = "Service"
* vars
* email_alert = true
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 41:5-41:27
* enable_sms = false
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 42:5-42:27
* http_address = "tools.foo.com"
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 43:5-43:44
* http_crit = 15
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 44:5-44:23
* http_onredirect = "follow"
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 45:5-45:35
* http_ssl = true
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 46:5-46:24
* http_string = "OK"
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 47:5-47:27
* http_uri = "/monitors/lunch_menu.php"
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 48:5-48:46
* http_vhost = "tools.foo.com"
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 49:5-49:42
* http_warn = 10
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/mtl/services.conf',
lines 50:5-50:23
* volatile = false
% = modified in
'/var/lib/icinga2/api/packages/director/l-ic-m-01-1523171022-1/zones.d/director-global/service_templates.conf',
lines 34:5-34:20
* zone = "mtl"
This is a sample of how the mail script trigger shows in the debug log:
[2018-04-09 13:55:40 +0300] notice/Process: Running command
'/usr/local/bin/gen-notification-new.py' 'mail' '--service' '-a'
'10.0.6.194' '-b' '' '-c' '' '-d' '2018-04-09 13:55:40' '-i' 'Windows
Host Memory use' '-l' 'MTL-SP13-APP02' '-m' '' '-o' 'No data was
received from host!
could not fetch information from server' '-s' 'WARNING' '-t' 'PROBLEM':
PID 19512
[2018-04-09 14:02:19 +0300] notice/Process: Running command
'/usr/local/bin/gen-notification-new.py' 'mail' '--service' '-a'
'10.0.6.194' '-b' '' '-c' '' '-d' '2018-04-09 14:02:19' '-i' 'Windows
Host Memory use' '-l' 'MTL-SP13-APP02' '-m' '' '-o' 'Memory usage:
total:49951.92 MB - used: 0.00 MB (0%) - free: 49951.92 MB (100%) ' '-s'
'OK' '-t' 'RECOVERY': PID 19559
and this is what the parameters the notification script is taking for a
service execution :
[root@r-ic-m-01 ~]# /usr/local/bin/gen-notification-new.py mail -h
usage: gen-notification-new.py mail [-h] [-a ADDRESS] [--host | --service]
[-b NOTIFICATION_AUTHOR]
[-c NOTIFICATION_COMMENT] [-d DATE]
[-l HOSTNAME] [-o OUTPUT] [-m EMAIL]
[-s STATE] [-t TYPE] [-i SERVICENAME]
optional arguments:
-h, --help show this help message and exit
-a ADDRESS, --address ADDRESS
host address
--host
--service
-b NOTIFICATION_AUTHOR, --notification_author NOTIFICATION_AUTHOR
notification author
-c NOTIFICATION_COMMENT, --notification_comment NOTIFICATION_COMMENT
notification comment
-d DATE, --date DATE notification date
-l HOSTNAME, --hostname HOSTNAME
host event occured on
-o OUTPUT, --output OUTPUT
host/service output
-m EMAIL, --email EMAIL
user email
-s STATE, --state STATE
host/service state
-t TYPE, --type TYPE notification type
-i SERVICENAME, --servicename SERVICENAME
servicename
Any thoughts as to why it is not sending the Critical notifications?
_______________________________________________
icinga-users mailing list
icinga-users@lists.icinga.org
https://lists.icinga.org/mailman/listinfo/icinga-users
