From: Stephan Müller Date: Tue, 6 Nov 2018 12:43:03 +0000 (+0100) Subject: mgr/dashboard: Add the Prometheus alerts X-Git-Tag: v14.1.0~218^2~3 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=8451e8c5959094f12b07fccfa57397fe7b7a21fe;p=ceph-ci.git mgr/dashboard: Add the Prometheus alerts The backend is now capable of receiving alert notifications from the Prometheus alertmanager and it can get all alerts with all kinds of parameters from the API of the same. In the frontend Prometheus alerts can be found in "Cluster > Alerts". Incoming notifications can be seen as usual in the notifications popover. To clarify: Prometheus alerts are received from the alertmanager API. Prometheus alert notification are send from the alertmanager to the backend receiver. An alert notification can have multiple alerts, but these alerts differ from the prometheus alerts. To clarify that, I've added some models and services. If one of the methods to get alerts contains changes the user will be notified. The documentation explains how to configure the alertmanager to use the dashboard receiver and how to connect the use of the alertmanager API. Further it explains where to find the alerts and what happens if they are configured and something is happening. Fixes: https://tracker.ceph.com/issues/36721 Signed-off-by: Stephan Müller --- diff --git a/doc/mgr/dashboard.rst b/doc/mgr/dashboard.rst index cd0b92b5ed2..15e793c2e21 100644 --- a/doc/mgr/dashboard.rst +++ b/doc/mgr/dashboard.rst @@ -392,6 +392,81 @@ To enable SSO:: $ ceph dashboard sso enable saml2 +Enabling Prometheus alerting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Using Prometheus for monitoring, you have to define `alerting rules +`_. +To manage them you need to use the `Alertmanager +`_. +If you are not using the Alertmanager yet, please `install it +`_ as it's mandatory in +order to receive and manage alerts from Prometheus. + +The Alertmanager capabilities can be consumed by the dashboard in three different +ways: + +#. Use the notification receiver of the dashboard. + +#. Use the Prometheus Alertmanager API. + +#. Use both sources simultaneously. + +All three methods are going to notify you about alerts. You won't be notified +twice if you use both sources. + +#. Use the notification receiver of the dashboard: + + This allows you to get notifications as `configured + `_ from the Alertmanager. + You will get notified inside the dashboard once a notification is send out, + but you are not able to manage alerts. + + Add the dashboard receiver and the new route to your Alertmanager configuration. + This should look like:: + + route: + receiver: 'ceph-dashboard' + ... + receivers: + - name: 'ceph-dashboard' + webhook_configs: + - url: '/api/prometheus_receiver' + + + Please make sure that the Alertmanager considers your SSL certificate in terms + of the dashboard as valid. For more information about the correct + configuration checkout the ` documentation + `_. + +#. Use the API of the Prometheus Alertmanager + + This allows you to manage alerts. You will see all alerts, the Alertmanager + currently knows of, in the alerts listing. It can be found in the *Cluster* + submenu as *Alerts*. The alerts can be sorted by name, job, severity, + state and start time. Unfortunately it's not possible to know when an alert + was sent out through a notification by the Alertmanager based on your + configuration, that's why the dashboard will notify the user on any visible + change to an alert and will notify the changed alert. + + Currently it's not yet possible to silence an alert and expire an silenced + alert, but this is work in progress and will be added in a future release. + + To use it, specify the host and port of the Alertmanager server:: + + $ ceph dashboard set-alertmanager-api-host # default: '' + + For example:: + + $ ceph dashboard set-alertmanager-api-host 'http://localhost:9093' + + +#. Use both methods + + The different behaviors of both methods are configured in a way that they + should not disturb each other through annoying duplicated notifications + popping up. + Accessing the dashboard ^^^^^^^^^^^^^^^^^^^^^^^ @@ -470,6 +545,7 @@ scopes are: management. - **log**: include all features related to Ceph logs management. - **grafana**: include all features related to Grafana proxy. +- **prometheus**: include all features related to Prometheus alert management. - **dashboard-settings**: allows to change dashboard settings. A *role* specifies a set of mappings between a *security scope* and a set of diff --git a/src/pybind/mgr/dashboard/controllers/prometheus.py b/src/pybind/mgr/dashboard/controllers/prometheus.py new file mode 100644 index 00000000000..d233361a5b1 --- /dev/null +++ b/src/pybind/mgr/dashboard/controllers/prometheus.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +from datetime import datetime +import json +import requests + +from . import Controller, ApiController, BaseController, RESTController, Endpoint +from ..security import Scope +from ..settings import Settings + + +@Controller('/api/prometheus_receiver', secure=False) +class PrometheusReceiver(BaseController): + # The receiver is needed in order to receive alert notifications (reports) + notifications = [] + + @Endpoint('POST', path='/') + def fetch_alert(self, **notification): + notification['notified'] = datetime.now().isoformat() + self.notifications.append(notification) + + +@ApiController('/prometheus', Scope.PROMETHEUS) +class Prometheus(RESTController): + def _get_api_url(self): + return Settings.ALERTMANAGER_API_HOST.rstrip('/') + '/api/v1' + + def _api_request(self, url_suffix, params=None): + url = self._get_api_url() + url_suffix + response = requests.request('GET', url, params=params) + payload = json.loads(response.content) + return payload['data'] if 'data' in payload else [] + + def list(self, **params): + return self._api_request('/alerts', params) + + @RESTController.Collection('POST') + def get_notifications_since(self, **last_notification): + notifications = PrometheusReceiver.notifications + if last_notification not in notifications: + return notifications[-1:] + index = notifications.index(last_notification) + return notifications[index + 1:] diff --git a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts index fe7ee88448f..fddc917b759 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts @@ -13,6 +13,7 @@ import { HostsComponent } from './ceph/cluster/hosts/hosts.component'; import { LogsComponent } from './ceph/cluster/logs/logs.component'; import { MonitorComponent } from './ceph/cluster/monitor/monitor.component'; import { OsdListComponent } from './ceph/cluster/osd/osd-list/osd-list.component'; +import { PrometheusListComponent } from './ceph/cluster/prometheus/prometheus-list/prometheus-list.component'; import { DashboardComponent } from './ceph/dashboard/dashboard/dashboard.component'; import { PerformanceCounterComponent } from './ceph/performance-counter/performance-counter/performance-counter.component'; import { PoolFormComponent } from './ceph/pool/pool-form/pool-form.component'; @@ -109,6 +110,12 @@ const routes: Routes = [ canActivate: [AuthGuardService], data: { breadcrumbs: 'Cluster/Logs' } }, + { + path: 'alerts', + component: PrometheusListComponent, + canActivate: [AuthGuardService], + data: { breadcrumbs: 'Cluster/Alerts' } + }, { path: 'perf_counters/:type/:id', component: PerformanceCounterComponent, diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts index 2adb5a015af..a8aeb7d855f 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts @@ -27,6 +27,7 @@ import { OsdPerformanceHistogramComponent } from './osd/osd-performance-histogra import { OsdRecvSpeedModalComponent } from './osd/osd-recv-speed-modal/osd-recv-speed-modal.component'; import { OsdReweightModalComponent } from './osd/osd-reweight-modal/osd-reweight-modal.component'; import { OsdScrubModalComponent } from './osd/osd-scrub-modal/osd-scrub-modal.component'; +import { PrometheusListComponent } from './prometheus/prometheus-list/prometheus-list.component'; @NgModule({ entryComponents: [ @@ -65,6 +66,7 @@ import { OsdScrubModalComponent } from './osd/osd-scrub-modal/osd-scrub-modal.co OsdReweightModalComponent, CrushmapComponent, LogsComponent, + PrometheusListComponent, OsdRecvSpeedModalComponent ] }) diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.html new file mode 100644 index 00000000000..50ac7bda711 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.html @@ -0,0 +1,27 @@ + + + + + + + + + + + Source + + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.scss new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.spec.ts new file mode 100644 index 00000000000..7901a05cb33 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.spec.ts @@ -0,0 +1,30 @@ +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { ToastModule } from 'ng2-toastr'; +import { TabsModule } from 'ngx-bootstrap/tabs'; + +import { configureTestBed, i18nProviders } from '../../../../../testing/unit-test-helper'; +import { SharedModule } from '../../../../shared/shared.module'; +import { PrometheusListComponent } from './prometheus-list.component'; + +describe('PrometheusListComponent', () => { + let component: PrometheusListComponent; + let fixture: ComponentFixture; + + configureTestBed({ + imports: [HttpClientTestingModule, TabsModule.forRoot(), ToastModule.forRoot(), SharedModule], + declarations: [PrometheusListComponent], + providers: [i18nProviders] + }); + + beforeEach(() => { + fixture = TestBed.createComponent(PrometheusListComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.ts new file mode 100644 index 00000000000..722046e001c --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.ts @@ -0,0 +1,70 @@ +import { Component, OnInit, TemplateRef, ViewChild } from '@angular/core'; +import { I18n } from '@ngx-translate/i18n-polyfill'; +import { CellTemplate } from '../../../../shared/enum/cell-template.enum'; +import { CdTableColumn } from '../../../../shared/models/cd-table-column'; +import { CdTableSelection } from '../../../../shared/models/cd-table-selection'; +import { CdDatePipe } from '../../../../shared/pipes/cd-date.pipe'; +import { PrometheusAlertService } from '../../../../shared/services/prometheus-alert.service'; + +@Component({ + selector: 'cd-prometheus-list', + templateUrl: './prometheus-list.component.html', + styleUrls: ['./prometheus-list.component.scss'] +}) +export class PrometheusListComponent implements OnInit { + @ViewChild('externalLinkTpl') + externalLinkTpl: TemplateRef; + columns: CdTableColumn[]; + selection = new CdTableSelection(); + customCss = { + 'label label-danger': 'active', + 'label label-warning': 'unprocessed', + 'label label-info': 'suppressed' + }; + + constructor( + // NotificationsComponent will refresh all alerts every 5s (No need to do it here as well) + public prometheusAlertService: PrometheusAlertService, + private i18n: I18n, + private cdDatePipe: CdDatePipe + ) {} + + ngOnInit() { + this.columns = [ + { + name: this.i18n('Name'), + prop: 'labels.alertname', + flexGrow: 2 + }, + { + name: this.i18n('Job'), + prop: 'labels.job', + flexGrow: 2 + }, + { + name: this.i18n('Severity'), + prop: 'labels.severity' + }, + { + name: this.i18n('State'), + prop: 'status.state', + cellTransformation: CellTemplate.classAdding + }, + { + name: this.i18n('Started'), + prop: 'startsAt', + pipe: this.cdDatePipe + }, + { + name: this.i18n('URL'), + prop: 'generatorURL', + sortable: false, + cellTemplate: this.externalLinkTpl + } + ]; + } + + updateSelection(selection: CdTableSelection) { + this.selection = selection; + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html index 7b5713515e7..7083d73690a 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html @@ -90,6 +90,12 @@ class="dropdown-item" routerLink="/logs">Logs +
  • + Alerts +
    • diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts index 87ce7e1c86e..4c304646a57 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts @@ -1,5 +1,6 @@ import { Component, OnInit } from '@angular/core'; +import { PrometheusService } from '../../../shared/api/prometheus.service'; import { Permissions } from '../../../shared/models/permissions'; import { AuthStorageService } from '../../../shared/services/auth-storage.service'; import { SummaryService } from '../../../shared/services/summary.service'; @@ -12,10 +13,13 @@ import { SummaryService } from '../../../shared/services/summary.service'; export class NavigationComponent implements OnInit { permissions: Permissions; summaryData: any; + isCollapsed = true; + prometheusConfigured = false; constructor( private authStorageService: AuthStorageService, + private prometheusService: PrometheusService, private summaryService: SummaryService ) { this.permissions = this.authStorageService.getPermissions(); @@ -28,6 +32,7 @@ export class NavigationComponent implements OnInit { } this.summaryData = data; }); + this.prometheusService.ifAlertmanagerConfigured(() => (this.prometheusConfigured = true)); } blockHealthColor() { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.spec.ts index 5b081d8d720..a11be2b8c68 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.spec.ts @@ -1,9 +1,14 @@ -import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { ComponentFixture, fakeAsync, TestBed, tick } from '@angular/core/testing'; import { ToastModule } from 'ng2-toastr'; import { PopoverModule } from 'ngx-bootstrap/popover'; import { configureTestBed, i18nProviders } from '../../../../testing/unit-test-helper'; +import { PrometheusService } from '../../../shared/api/prometheus.service'; +import { AuthStorageService } from '../../../shared/services/auth-storage.service'; +import { PrometheusAlertService } from '../../../shared/services/prometheus-alert.service'; +import { PrometheusNotificationService } from '../../../shared/services/prometheus-notification.service'; import { SharedModule } from '../../../shared/shared.module'; import { NotificationsComponent } from './notifications.component'; @@ -12,7 +17,12 @@ describe('NotificationsComponent', () => { let fixture: ComponentFixture; configureTestBed({ - imports: [PopoverModule.forRoot(), SharedModule, ToastModule.forRoot()], + imports: [ + HttpClientTestingModule, + PopoverModule.forRoot(), + SharedModule, + ToastModule.forRoot() + ], declarations: [NotificationsComponent], providers: i18nProviders }); @@ -20,10 +30,60 @@ describe('NotificationsComponent', () => { beforeEach(() => { fixture = TestBed.createComponent(NotificationsComponent); component = fixture.componentInstance; - fixture.detectChanges(); }); it('should create', () => { + fixture.detectChanges(); expect(component).toBeTruthy(); }); + + describe('prometheus alert handling', () => { + let prometheusAlertService: PrometheusAlertService; + let prometheusNotificationService: PrometheusNotificationService; + let prometheusAccessAllowed: boolean; + + const expectPrometheusServicesToBeCalledTimes = (n: number) => { + expect(prometheusNotificationService.refresh).toHaveBeenCalledTimes(n); + expect(prometheusAlertService.refresh).toHaveBeenCalledTimes(n); + }; + + beforeEach(() => { + prometheusAccessAllowed = true; + spyOn(TestBed.get(AuthStorageService), 'getPermissions').and.callFake(() => ({ + prometheus: { read: prometheusAccessAllowed } + })); + + spyOn(TestBed.get(PrometheusService), 'ifAlertmanagerConfigured').and.callFake((fn) => fn()); + + prometheusAlertService = TestBed.get(PrometheusAlertService); + spyOn(prometheusAlertService, 'refresh').and.stub(); + + prometheusNotificationService = TestBed.get(PrometheusNotificationService); + spyOn(prometheusNotificationService, 'refresh').and.stub(); + }); + + it('should not refresh prometheus services if not allowed', () => { + prometheusAccessAllowed = false; + fixture.detectChanges(); + + expectPrometheusServicesToBeCalledTimes(0); + }); + it('should first refresh prometheus notifications and alerts during init', () => { + fixture.detectChanges(); + + expect(prometheusAlertService.refresh).toHaveBeenCalledTimes(1); + expectPrometheusServicesToBeCalledTimes(1); + }); + + it('should refresh prometheus services every 5s', fakeAsync(() => { + fixture.detectChanges(); + + expectPrometheusServicesToBeCalledTimes(1); + tick(5000); + expectPrometheusServicesToBeCalledTimes(2); + tick(15000); + expectPrometheusServicesToBeCalledTimes(5); + component.ngOnDestroy(); + })); + }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.ts index d96b208de51..ad433c485a9 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.ts @@ -1,30 +1,58 @@ -import { Component, OnInit } from '@angular/core'; +import { Component, NgZone, OnDestroy, OnInit } from '@angular/core'; import * as _ from 'lodash'; import { NotificationType } from '../../../shared/enum/notification-type.enum'; import { CdNotification } from '../../../shared/models/cd-notification'; +import { AuthStorageService } from '../../../shared/services/auth-storage.service'; import { NotificationService } from '../../../shared/services/notification.service'; +import { PrometheusAlertService } from '../../../shared/services/prometheus-alert.service'; +import { PrometheusNotificationService } from '../../../shared/services/prometheus-notification.service'; @Component({ selector: 'cd-notifications', templateUrl: './notifications.component.html', styleUrls: ['./notifications.component.scss'] }) -export class NotificationsComponent implements OnInit { +export class NotificationsComponent implements OnInit, OnDestroy { notifications: CdNotification[]; - notificationType = NotificationType; + private interval: number; - constructor(private notificationService: NotificationService) { + constructor( + private notificationService: NotificationService, + private prometheusNotificationService: PrometheusNotificationService, + private authStorageService: AuthStorageService, + private prometheusAlertService: PrometheusAlertService, + private ngZone: NgZone + ) { this.notifications = []; } + ngOnDestroy() { + window.clearInterval(this.interval); + } + ngOnInit() { + if (this.authStorageService.getPermissions().prometheus.read) { + this.triggerPrometheusAlerts(); + this.ngZone.runOutsideAngular(() => { + this.interval = window.setInterval(() => { + this.ngZone.run(() => { + this.triggerPrometheusAlerts(); + }); + }, 5000); + }); + } this.notificationService.data$.subscribe((notifications: CdNotification[]) => { this.notifications = _.orderBy(notifications, ['timestamp'], ['desc']); }); } + private triggerPrometheusAlerts() { + this.prometheusAlertService.refresh(); + this.prometheusNotificationService.refresh(); + } + removeAll() { this.notificationService.removeAll(); } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.spec.ts new file mode 100644 index 00000000000..0a0c49fc3be --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.spec.ts @@ -0,0 +1,70 @@ +import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing'; +import { TestBed } from '@angular/core/testing'; + +import { configureTestBed } from '../../../testing/unit-test-helper'; +import { PrometheusService } from './prometheus.service'; +import { SettingsService } from './settings.service'; + +describe('PrometheusService', () => { + let service: PrometheusService; + let httpTesting: HttpTestingController; + + configureTestBed({ + providers: [PrometheusService, SettingsService], + imports: [HttpClientTestingModule] + }); + + beforeEach(() => { + service = TestBed.get(PrometheusService); + httpTesting = TestBed.get(HttpTestingController); + }); + + afterEach(() => { + httpTesting.verify(); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); + + it('should call list', () => { + service.list().subscribe(); + const req = httpTesting.expectOne('api/prometheus'); + expect(req.request.method).toBe('GET'); + }); + + it('should call getNotificationSince', () => { + service.getNotificationSince({}).subscribe(); + const req = httpTesting.expectOne('api/prometheus/get_notifications_since'); + expect(req.request.method).toBe('POST'); + }); + + describe('ifAlertmanagerConfigured', () => { + let x: any; + + const receiveConfig = (value) => { + const req = httpTesting.expectOne('api/settings/alertmanager-api-host'); + expect(req.request.method).toBe('GET'); + req.flush({ value }); + }; + + beforeEach(() => { + x = false; + TestBed.get(SettingsService)['settings'] = {}; + }); + + it('changes x in a valid case', () => { + service.ifAlertmanagerConfigured((v) => (x = v)); + expect(x).toBe(false); + const host = 'http://localhost:9093'; + receiveConfig(host); + expect(x).toBe(host); + }); + + it('does not change x in a invalid case', () => { + service.ifAlertmanagerConfigured((v) => (x = v)); + receiveConfig(''); + expect(x).toBe(false); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts new file mode 100644 index 00000000000..0fd288e9c01 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts @@ -0,0 +1,32 @@ +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; + +import { Observable } from 'rxjs'; + +import { PrometheusAlert, PrometheusNotification } from '../models/prometheus-alerts'; +import { ApiModule } from './api.module'; +import { SettingsService } from './settings.service'; + +@Injectable({ + providedIn: ApiModule +}) +export class PrometheusService { + private baseURL = 'api/prometheus'; + + constructor(private http: HttpClient, private settingsService: SettingsService) {} + + ifAlertmanagerConfigured(fn): void { + this.settingsService.ifSettingConfigured('api/settings/alertmanager-api-host', fn); + } + + list(params = {}): Observable { + return this.http.get(this.baseURL, { params }); + } + + getNotificationSince(notification): Observable { + return this.http.post( + `${this.baseURL}/get_notifications_since`, + notification + ); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permission.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permission.spec.ts new file mode 100644 index 00000000000..e2725bc856d --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permission.spec.ts @@ -0,0 +1,60 @@ +import { Permissions } from './permissions'; + +describe('cd-notification classes', () => { + it('should show empty permissions', () => { + expect(new Permissions({})).toEqual({ + cephfs: { create: false, delete: false, read: false, update: false }, + configOpt: { create: false, delete: false, read: false, update: false }, + grafana: { create: false, delete: false, read: false, update: false }, + hosts: { create: false, delete: false, read: false, update: false }, + iscsi: { create: false, delete: false, read: false, update: false }, + log: { create: false, delete: false, read: false, update: false }, + manager: { create: false, delete: false, read: false, update: false }, + monitor: { create: false, delete: false, read: false, update: false }, + osd: { create: false, delete: false, read: false, update: false }, + pool: { create: false, delete: false, read: false, update: false }, + prometheus: { create: false, delete: false, read: false, update: false }, + rbdImage: { create: false, delete: false, read: false, update: false }, + rbdMirroring: { create: false, delete: false, read: false, update: false }, + rgw: { create: false, delete: false, read: false, update: false }, + user: { create: false, delete: false, read: false, update: false } + }); + }); + + it('should show full permissions', () => { + const fullyGranted = { + cephfs: ['create', 'read', 'update', 'delete'], + 'config-opt': ['create', 'read', 'update', 'delete'], + grafana: ['create', 'read', 'update', 'delete'], + hosts: ['create', 'read', 'update', 'delete'], + iscsi: ['create', 'read', 'update', 'delete'], + log: ['create', 'read', 'update', 'delete'], + manager: ['create', 'read', 'update', 'delete'], + monitor: ['create', 'read', 'update', 'delete'], + osd: ['create', 'read', 'update', 'delete'], + pool: ['create', 'read', 'update', 'delete'], + prometheus: ['create', 'read', 'update', 'delete'], + 'rbd-image': ['create', 'read', 'update', 'delete'], + 'rbd-mirroring': ['create', 'read', 'update', 'delete'], + rgw: ['create', 'read', 'update', 'delete'], + user: ['create', 'read', 'update', 'delete'] + }; + expect(new Permissions(fullyGranted)).toEqual({ + cephfs: { create: true, delete: true, read: true, update: true }, + configOpt: { create: true, delete: true, read: true, update: true }, + grafana: { create: true, delete: true, read: true, update: true }, + hosts: { create: true, delete: true, read: true, update: true }, + iscsi: { create: true, delete: true, read: true, update: true }, + log: { create: true, delete: true, read: true, update: true }, + manager: { create: true, delete: true, read: true, update: true }, + monitor: { create: true, delete: true, read: true, update: true }, + osd: { create: true, delete: true, read: true, update: true }, + pool: { create: true, delete: true, read: true, update: true }, + prometheus: { create: true, delete: true, read: true, update: true }, + rbdImage: { create: true, delete: true, read: true, update: true }, + rbdMirroring: { create: true, delete: true, read: true, update: true }, + rgw: { create: true, delete: true, read: true, update: true }, + user: { create: true, delete: true, read: true, update: true } + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permissions.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permissions.ts index 0938e264b65..0d8d580266c 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permissions.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permissions.ts @@ -5,10 +5,9 @@ export class Permission { delete: boolean; constructor(serverPermission: Array = []) { - this.read = serverPermission.indexOf('read') !== -1; - this.create = serverPermission.indexOf('create') !== -1; - this.update = serverPermission.indexOf('update') !== -1; - this.delete = serverPermission.indexOf('delete') !== -1; + ['read', 'create', 'update', 'delete'].forEach( + (permission) => (this[permission] = serverPermission.includes(permission)) + ); } } @@ -27,6 +26,7 @@ export class Permissions { log: Permission; user: Permission; grafana: Permission; + prometheus: Permission; constructor(serverPermissions: any) { this.hosts = new Permission(serverPermissions['hosts']); @@ -43,5 +43,6 @@ export class Permissions { this.log = new Permission(serverPermissions['log']); this.user = new Permission(serverPermissions['user']); this.grafana = new Permission(serverPermissions['grafana']); + this.prometheus = new Permission(serverPermissions['prometheus']); } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/prometheus-alerts.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/prometheus-alerts.ts new file mode 100644 index 00000000000..204333110d5 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/prometheus-alerts.ts @@ -0,0 +1,52 @@ +class CommonAlert { + labels: { + alertname: string; + instance: string; + job: string; + severity: string; + }; + annotations: { + description: string; + summary: string; + }; + startsAt: string; + endsAt: string; + generatorURL: string; +} + +export class PrometheusAlert extends CommonAlert { + status: { + state: 'unprocessed' | 'active' | 'suppressed'; + silencedBy: null | string[]; + inhibitedBy: null | string[]; + }; + receivers: string[]; + fingerprint: string; +} + +export class PrometheusNotificationAlert extends CommonAlert { + status: 'firing' | 'resolved'; +} + +export class PrometheusNotification { + status: 'firing' | 'resolved'; + groupLabels: object; + commonAnnotations: object; + groupKey: string; + notified: string; + alerts: PrometheusNotificationAlert[]; + version: string; + receiver: string; + externalURL: string; + commonLabels: { + severity: string; + }; +} + +export class PrometheusCustomAlert { + status: 'resolved' | 'unprocessed' | 'active' | 'suppressed'; + name: string; + url: string; + summary: string; + fingerprint?: string | boolean; +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert-formatter.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert-formatter.spec.ts new file mode 100644 index 00000000000..a0a5b49403f --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert-formatter.spec.ts @@ -0,0 +1,98 @@ +import { TestBed } from '@angular/core/testing'; + +import { ToastModule } from 'ng2-toastr'; + +import { + configureTestBed, + i18nProviders, + PrometheusHelper +} from '../../../testing/unit-test-helper'; +import { NotificationType } from '../enum/notification-type.enum'; +import { CdNotificationConfig } from '../models/cd-notification'; +import { PrometheusCustomAlert } from '../models/prometheus-alerts'; +import { SharedModule } from '../shared.module'; +import { NotificationService } from './notification.service'; +import { PrometheusAlertFormatter } from './prometheus-alert-formatter'; + +describe('PrometheusAlertFormatter', () => { + let service: PrometheusAlertFormatter; + let notificationService: NotificationService; + let prometheus: PrometheusHelper; + + configureTestBed({ + imports: [ToastModule.forRoot(), SharedModule], + providers: [PrometheusAlertFormatter, i18nProviders] + }); + + beforeEach(() => { + prometheus = new PrometheusHelper(); + service = TestBed.get(PrometheusAlertFormatter); + notificationService = TestBed.get(NotificationService); + spyOn(notificationService, 'queueNotifications').and.stub(); + }); + + it('should create', () => { + expect(service).toBeTruthy(); + }); + + describe('sendNotifications', () => { + it('should not call queue notifications with no notification', () => { + service.sendNotifications([]); + expect(notificationService.queueNotifications).not.toHaveBeenCalled(); + }); + + it('should call queue notifications with notifications', () => { + const notifications = [new CdNotificationConfig(NotificationType.success, 'test')]; + service.sendNotifications(notifications); + expect(notificationService.queueNotifications).toHaveBeenCalledWith(notifications); + }); + }); + + describe('convertToCustomAlert', () => { + it('converts PrometheusAlert', () => { + expect(service.convertToCustomAlerts([prometheus.createAlert('Something')])).toEqual([ + { + status: 'active', + name: 'Something', + summary: 'Something is active', + url: 'http://Something', + fingerprint: 'Something' + } as PrometheusCustomAlert + ]); + }); + + it('converts PrometheusNotificationAlert', () => { + expect( + service.convertToCustomAlerts([prometheus.createNotificationAlert('Something')]) + ).toEqual([ + { + fingerprint: false, + status: 'active', + name: 'Something', + summary: 'Something is firing', + url: 'http://Something' + } as PrometheusCustomAlert + ]); + }); + }); + + it('converts custom alert into notification', () => { + const alert: PrometheusCustomAlert = { + status: 'active', + name: 'Some alert', + summary: 'Some alert is active', + url: 'http://some-alert', + fingerprint: '42' + }; + expect(service.convertAlertToNotification(alert)).toEqual( + new CdNotificationConfig( + NotificationType.error, + 'Some alert (active)', + 'Some alert is active ' + + '', + undefined, + 'Prometheus' + ) + ); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert-formatter.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert-formatter.ts new file mode 100644 index 00000000000..8fdc5ddb68f --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert-formatter.ts @@ -0,0 +1,76 @@ +import { Injectable } from '@angular/core'; + +import * as _ from 'lodash'; + +import { NotificationType } from '../enum/notification-type.enum'; +import { CdNotificationConfig } from '../models/cd-notification'; +import { + PrometheusAlert, + PrometheusCustomAlert, + PrometheusNotificationAlert +} from '../models/prometheus-alerts'; +import { NotificationService } from './notification.service'; +import { ServicesModule } from './services.module'; + +@Injectable({ + providedIn: ServicesModule +}) +export class PrometheusAlertFormatter { + constructor(private notificationService: NotificationService) {} + + sendNotifications(notifications: CdNotificationConfig[]) { + if (notifications.length > 0) { + this.notificationService.queueNotifications(notifications); + } + } + + convertToCustomAlerts( + alerts: (PrometheusNotificationAlert | PrometheusAlert)[] + ): PrometheusCustomAlert[] { + return _.uniqWith( + alerts.map((alert) => { + return { + status: _.isObject(alert.status) + ? (alert as PrometheusAlert).status.state + : this.getPrometheusNotificationStatus(alert as PrometheusNotificationAlert), + name: alert.labels.alertname, + url: alert.generatorURL, + summary: alert.annotations.summary, + fingerprint: _.isObject(alert.status) && (alert as PrometheusAlert).fingerprint + }; + }), + _.isEqual + ) as PrometheusCustomAlert[]; + } + + /* + * This is needed because NotificationAlerts don't use 'active' + */ + private getPrometheusNotificationStatus(alert: PrometheusNotificationAlert): string { + const state = alert.status; + return state === 'firing' ? 'active' : state; + } + + convertAlertToNotification(alert: PrometheusCustomAlert): CdNotificationConfig { + return new CdNotificationConfig( + this.formatType(alert.status), + `${alert.name} (${alert.status})`, + this.appendSourceLink(alert, alert.summary), + undefined, + 'Prometheus' + ); + } + + private formatType(status: string): NotificationType { + const types = { + error: ['firing', 'active'], + info: ['suppressed', 'unprocessed'], + success: ['resolved'] + }; + return NotificationType[_.findKey(types, (type) => type.includes(status))]; + } + + private appendSourceLink(alert: PrometheusCustomAlert, message: string): string { + return `${message} `; + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.spec.ts new file mode 100644 index 00000000000..36fe4c66fbd --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.spec.ts @@ -0,0 +1,154 @@ +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { TestBed } from '@angular/core/testing'; + +import { ToastModule } from 'ng2-toastr'; +import { of } from 'rxjs'; + +import { + configureTestBed, + i18nProviders, + PrometheusHelper +} from '../../../testing/unit-test-helper'; +import { PrometheusService } from '../api/prometheus.service'; +import { NotificationType } from '../enum/notification-type.enum'; +import { CdNotificationConfig } from '../models/cd-notification'; +import { PrometheusAlert } from '../models/prometheus-alerts'; +import { SharedModule } from '../shared.module'; +import { NotificationService } from './notification.service'; +import { PrometheusAlertFormatter } from './prometheus-alert-formatter'; +import { PrometheusAlertService } from './prometheus-alert.service'; + +describe('PrometheusAlertService', () => { + let service: PrometheusAlertService; + let notificationService: NotificationService; + let alerts: PrometheusAlert[]; + let prometheusService: PrometheusService; + let prometheus: PrometheusHelper; + + configureTestBed({ + imports: [ToastModule.forRoot(), SharedModule, HttpClientTestingModule], + providers: [PrometheusAlertService, PrometheusAlertFormatter, i18nProviders] + }); + + beforeEach(() => { + prometheus = new PrometheusHelper(); + }); + + it('should create', () => { + expect(TestBed.get(PrometheusAlertService)).toBeTruthy(); + }); + + it('tests error case ', () => { + const resp = { status: 500, error: {} }; + service = new PrometheusAlertService(null, { + ifAlertmanagerConfigured: (fn) => fn(), + list: () => ({ subscribe: (fn, err) => err(resp) }) + }); + + expect(service['connected']).toBe(true); + service.refresh(); + expect(service['connected']).toBe(false); + expect(resp['application']).toBe('Prometheus'); + expect(resp.error['detail']).toBe( + 'Please check if Prometheus Alertmanager is still running' + ); + }); + + describe('refresh', () => { + beforeEach(() => { + service = TestBed.get(PrometheusAlertService); + service['alerts'] = []; + service['canAlertsBeNotified'] = false; + + spyOn(window, 'setTimeout').and.callFake((fn: Function) => fn()); + + notificationService = TestBed.get(NotificationService); + spyOn(notificationService, 'queueNotifications').and.callThrough(); + spyOn(notificationService, 'show').and.stub(); + + prometheusService = TestBed.get(PrometheusService); + spyOn(prometheusService, 'ifAlertmanagerConfigured').and.callFake((fn) => fn()); + spyOn(prometheusService, 'list').and.callFake(() => of(alerts)); + + alerts = [prometheus.createAlert('alert0')]; + service.refresh(); + }); + + it('should not notify on first call', () => { + expect(notificationService.show).not.toHaveBeenCalled(); + }); + + it('should not notify with no change', () => { + service.refresh(); + expect(notificationService.show).not.toHaveBeenCalled(); + }); + + it('should notify on alert change', () => { + alerts = [prometheus.createAlert('alert0', 'suppressed')]; + service.refresh(); + expect(notificationService.queueNotifications).toHaveBeenCalledWith([ + new CdNotificationConfig( + NotificationType.info, + 'alert0 (suppressed)', + 'alert0 is suppressed ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ) + ]); + }); + + it('should notify on a new alert', () => { + alerts = [prometheus.createAlert('alert1'), prometheus.createAlert('alert0')]; + service.refresh(); + expect(notificationService.show).toHaveBeenCalledTimes(1); + expect(notificationService.show).toHaveBeenCalledWith( + new CdNotificationConfig( + NotificationType.error, + 'alert1 (active)', + 'alert1 is active ' + prometheus.createLink('http://alert1'), + undefined, + 'Prometheus' + ) + ); + }); + + it('should notify a resolved alert if it is not there anymore', () => { + alerts = []; + service.refresh(); + expect(notificationService.show).toHaveBeenCalledTimes(1); + expect(notificationService.show).toHaveBeenCalledWith( + new CdNotificationConfig( + NotificationType.success, + 'alert0 (resolved)', + 'alert0 is active ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ) + ); + }); + + it('should call multiple times for multiple changes', () => { + const alert1 = prometheus.createAlert('alert1'); + alerts.push(alert1); + service.refresh(); + alerts = [alert1, prometheus.createAlert('alert2')]; + service.refresh(); + expect(notificationService.queueNotifications).toHaveBeenCalledWith([ + new CdNotificationConfig( + NotificationType.error, + 'alert2 (active)', + 'alert2 is active ' + prometheus.createLink('http://alert2'), + undefined, + 'Prometheus' + ), + new CdNotificationConfig( + NotificationType.success, + 'alert0 (resolved)', + 'alert0 is active ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ) + ]); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.ts new file mode 100644 index 00000000000..9a6d26c88e2 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.ts @@ -0,0 +1,73 @@ +import { Injectable } from '@angular/core'; + +import * as _ from 'lodash'; + +import { PrometheusService } from '../api/prometheus.service'; +import { PrometheusAlert, PrometheusCustomAlert } from '../models/prometheus-alerts'; +import { PrometheusAlertFormatter } from './prometheus-alert-formatter'; +import { ServicesModule } from './services.module'; + +@Injectable({ + providedIn: ServicesModule +}) +export class PrometheusAlertService { + private canAlertsBeNotified = false; + private connected = true; + alerts: PrometheusAlert[] = []; + + constructor( + private alertFormatter: PrometheusAlertFormatter, + private prometheusService: PrometheusService + ) {} + + refresh() { + this.prometheusService.ifAlertmanagerConfigured((url) => { + if (this.connected) { + this.prometheusService.list().subscribe( + (alerts) => this.handleAlerts(alerts), + (resp) => { + const errorMsg = `Please check if Prometheus Alertmanager is still running`; + resp['application'] = 'Prometheus'; + if (resp.status === 500) { + this.connected = false; + resp.error.detail = errorMsg; + } + } + ); + } + }); + } + + private handleAlerts(alerts: PrometheusAlert[]) { + if (this.canAlertsBeNotified) { + this.notifyOnAlertChanges(alerts, this.alerts); + } + this.alerts = alerts; + this.canAlertsBeNotified = true; + } + + private notifyOnAlertChanges(alerts: PrometheusAlert[], oldAlerts: PrometheusAlert[]) { + const changedAlerts = this.getChangedAlerts( + this.alertFormatter.convertToCustomAlerts(alerts), + this.alertFormatter.convertToCustomAlerts(oldAlerts) + ); + const notifications = changedAlerts.map((alert) => + this.alertFormatter.convertAlertToNotification(alert) + ); + this.alertFormatter.sendNotifications(notifications); + } + + private getChangedAlerts(alerts: PrometheusCustomAlert[], oldAlerts: PrometheusCustomAlert[]) { + const updatedAndNew = _.differenceWith(alerts, oldAlerts, _.isEqual); + return updatedAndNew.concat(this.getVanishedAlerts(alerts, oldAlerts)); + } + + private getVanishedAlerts(alerts: PrometheusCustomAlert[], oldAlerts: PrometheusCustomAlert[]) { + return _.differenceWith(oldAlerts, alerts, (a, b) => a.fingerprint === b.fingerprint).map( + (alert) => { + alert.status = 'resolved'; + return alert; + } + ); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-notification.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-notification.service.spec.ts new file mode 100644 index 00000000000..d395e84064b --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-notification.service.spec.ts @@ -0,0 +1,197 @@ +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { TestBed } from '@angular/core/testing'; + +import { ToastModule } from 'ng2-toastr'; +import { of } from 'rxjs'; + +import { + configureTestBed, + i18nProviders, + PrometheusHelper +} from '../../../testing/unit-test-helper'; +import { PrometheusService } from '../api/prometheus.service'; +import { NotificationType } from '../enum/notification-type.enum'; +import { CdNotificationConfig } from '../models/cd-notification'; +import { PrometheusNotification } from '../models/prometheus-alerts'; +import { SharedModule } from '../shared.module'; +import { NotificationService } from './notification.service'; +import { PrometheusAlertFormatter } from './prometheus-alert-formatter'; +import { PrometheusNotificationService } from './prometheus-notification.service'; + +describe('PrometheusNotificationService', () => { + let service: PrometheusNotificationService; + let notificationService: NotificationService; + let notifications: PrometheusNotification[]; + let prometheusService: PrometheusService; + let prometheus: PrometheusHelper; + let shown: CdNotificationConfig[]; + + configureTestBed({ + imports: [ToastModule.forRoot(), SharedModule, HttpClientTestingModule], + providers: [PrometheusNotificationService, PrometheusAlertFormatter, i18nProviders] + }); + + beforeEach(() => { + prometheus = new PrometheusHelper(); + + service = TestBed.get(PrometheusNotificationService); + service['notifications'] = []; + + notificationService = TestBed.get(NotificationService); + spyOn(notificationService, 'queueNotifications').and.callThrough(); + shown = []; + spyOn(notificationService, 'show').and.callFake((n) => shown.push(n)); + + spyOn(window, 'setTimeout').and.callFake((fn: Function) => fn()); + + prometheusService = TestBed.get(PrometheusService); + spyOn(prometheusService, 'getNotificationSince').and.callFake(() => of(notifications)); + + notifications = [prometheus.createNotification()]; + }); + + it('should create', () => { + expect(service).toBeTruthy(); + }); + + describe('getLastNotification', () => { + it('returns an empty object on the first call', () => { + service.refresh(); + expect(prometheusService.getNotificationSince).toHaveBeenCalledWith({}); + expect(service['notifications'].length).toBe(1); + }); + + it('returns last notification on any other call', () => { + service.refresh(); + notifications = [prometheus.createNotification(1, 'resolved')]; + service.refresh(); + expect(prometheusService.getNotificationSince).toHaveBeenCalledWith( + service['notifications'][0] + ); + expect(service['notifications'].length).toBe(2); + + notifications = [prometheus.createNotification(2)]; + service.refresh(); + notifications = [prometheus.createNotification(3, 'resolved')]; + service.refresh(); + expect(prometheusService.getNotificationSince).toHaveBeenCalledWith( + service['notifications'][2] + ); + expect(service['notifications'].length).toBe(4); + }); + }); + + it('notifies not on the first call', () => { + service.refresh(); + expect(notificationService.show).not.toHaveBeenCalled(); + }); + + describe('looks of fired notifications', () => { + beforeEach(() => { + service.refresh(); + service.refresh(); + shown = []; + }); + + it('notifies on the second call', () => { + expect(notificationService.show).toHaveBeenCalledTimes(1); + }); + + it('notify looks on single notification with single alert like', () => { + expect(notificationService.queueNotifications).toHaveBeenCalledWith([ + new CdNotificationConfig( + NotificationType.error, + 'alert0 (active)', + 'alert0 is firing ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ) + ]); + }); + + it('raises multiple pop overs for a single notification with multiple alerts', () => { + notifications[0].alerts.push(prometheus.createNotificationAlert('alert1', 'resolved')); + service.refresh(); + expect(shown).toEqual([ + new CdNotificationConfig( + NotificationType.error, + 'alert0 (active)', + 'alert0 is firing ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ), + new CdNotificationConfig( + NotificationType.success, + 'alert1 (resolved)', + 'alert1 is resolved ' + prometheus.createLink('http://alert1'), + undefined, + 'Prometheus' + ) + ]); + }); + + it('should raise multiple notifications if they do not look like each other', () => { + notifications[0].alerts.push(prometheus.createNotificationAlert('alert1')); + notifications.push(prometheus.createNotification()); + notifications[1].alerts.push(prometheus.createNotificationAlert('alert2')); + service.refresh(); + expect(shown).toEqual([ + new CdNotificationConfig( + NotificationType.error, + 'alert0 (active)', + 'alert0 is firing ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ), + new CdNotificationConfig( + NotificationType.error, + 'alert1 (active)', + 'alert1 is firing ' + prometheus.createLink('http://alert1'), + undefined, + 'Prometheus' + ), + new CdNotificationConfig( + NotificationType.error, + 'alert2 (active)', + 'alert2 is firing ' + prometheus.createLink('http://alert2'), + undefined, + 'Prometheus' + ) + ]); + }); + + it('only shows toasties if it got new data', () => { + expect(notificationService.show).toHaveBeenCalledTimes(1); + notifications = []; + service.refresh(); + service.refresh(); + expect(notificationService.show).toHaveBeenCalledTimes(1); + notifications = [prometheus.createNotification()]; + service.refresh(); + expect(notificationService.show).toHaveBeenCalledTimes(2); + service.refresh(); + expect(notificationService.show).toHaveBeenCalledTimes(3); + }); + + it('filters out duplicated and non user visible changes in notifications', () => { + // Return 2 notifications with 3 duplicated alerts and 1 non visible changed alert + const secondAlert = prometheus.createNotificationAlert('alert0'); + secondAlert.endsAt = new Date().toString(); // Should be ignored as it's not visible + notifications[0].alerts.push(secondAlert); + notifications.push(prometheus.createNotification()); + notifications[1].alerts.push(prometheus.createNotificationAlert('alert0')); + notifications[1].notified = 'by somebody else'; + service.refresh(); + + expect(shown).toEqual([ + new CdNotificationConfig( + NotificationType.error, + 'alert0 (active)', + 'alert0 is firing ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ) + ]); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-notification.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-notification.service.ts new file mode 100644 index 00000000000..b931c26e09b --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-notification.service.ts @@ -0,0 +1,52 @@ +import { Injectable } from '@angular/core'; + +import * as _ from 'lodash'; + +import { PrometheusService } from '../api/prometheus.service'; +import { CdNotificationConfig } from '../models/cd-notification'; +import { PrometheusNotification } from '../models/prometheus-alerts'; +import { PrometheusAlertFormatter } from './prometheus-alert-formatter'; +import { ServicesModule } from './services.module'; + +@Injectable({ + providedIn: ServicesModule +}) +export class PrometheusNotificationService { + private notifications: PrometheusNotification[]; + + constructor( + private alertFormatter: PrometheusAlertFormatter, + private prometheusService: PrometheusService + ) { + this.notifications = []; + } + + refresh() { + const last = this.getLastNotification(); + this.prometheusService + .getNotificationSince(last) + .subscribe((notifications) => this.handleNotifications(notifications)); + } + + private getLastNotification() { + return _.last(this.notifications) || {}; + } + + private handleNotifications(notifications: PrometheusNotification[]) { + if (notifications.length === 0) { + return; + } + if (this.notifications.length > 0) { + this.alertFormatter.sendNotifications( + _.flatten(notifications.map((notification) => this.formatNotification(notification))) + ); + } + this.notifications = this.notifications.concat(notifications); + } + + private formatNotification(notification: PrometheusNotification): CdNotificationConfig[] { + return this.alertFormatter + .convertToCustomAlerts(notification.alerts) + .map((alert) => this.alertFormatter.convertAlertToNotification(alert)); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/testing/unit-test-helper.ts b/src/pybind/mgr/dashboard/frontend/src/testing/unit-test-helper.ts index b294aab4e65..e4dcd176deb 100644 --- a/src/pybind/mgr/dashboard/frontend/src/testing/unit-test-helper.ts +++ b/src/pybind/mgr/dashboard/frontend/src/testing/unit-test-helper.ts @@ -9,6 +9,11 @@ import * as _ from 'lodash'; import { TableActionsComponent } from '../app/shared/datatable/table-actions/table-actions.component'; import { CdFormGroup } from '../app/shared/forms/cd-form-group'; import { Permission } from '../app/shared/models/permissions'; +import { + PrometheusAlert, + PrometheusNotification, + PrometheusNotificationAlert +} from '../app/shared/models/prometheus-alerts'; import { _DEV_ } from '../unit-test-configuration'; export function configureTestBed(configuration, useOldMethod?) { @@ -182,6 +187,48 @@ export class FormHelper { } } +export class PrometheusHelper { + createAlert(name, state = 'active', timeMultiplier = 1) { + return { + fingerprint: name, + status: { state }, + labels: { + alertname: name + }, + annotations: { + summary: `${name} is ${state}` + }, + generatorURL: `http://${name}`, + startsAt: new Date(new Date('2022-02-22').getTime() * timeMultiplier).toString() + } as PrometheusAlert; + } + + createNotificationAlert(name, status = 'firing') { + return { + status: status, + labels: { + alertname: name + }, + annotations: { + summary: `${name} is ${status}` + }, + generatorURL: `http://${name}` + } as PrometheusNotificationAlert; + } + + createNotification(alertNumber = 1, status = 'firing') { + const alerts = []; + for (let i = 0; i < alertNumber; i++) { + alerts.push(this.createNotificationAlert('alert' + i, status)); + } + return { alerts, status } as PrometheusNotification; + } + + createLink(url) { + return ``; + } +} + const XLIFF = ` diff --git a/src/pybind/mgr/dashboard/security.py b/src/pybind/mgr/dashboard/security.py index 64209e7f445..501c07f1789 100644 --- a/src/pybind/mgr/dashboard/security.py +++ b/src/pybind/mgr/dashboard/security.py @@ -23,6 +23,7 @@ class Scope(object): MANAGER = "manager" LOG = "log" GRAFANA = "grafana" + PROMETHEUS = "prometheus" USER = "user" DASHBOARD_SETTINGS = "dashboard-settings" diff --git a/src/pybind/mgr/dashboard/settings.py b/src/pybind/mgr/dashboard/settings.py index 73c490297e7..0bae36104af 100644 --- a/src/pybind/mgr/dashboard/settings.py +++ b/src/pybind/mgr/dashboard/settings.py @@ -43,6 +43,10 @@ class Options(object): # Orchestrator settings ORCHESTRATOR_BACKEND = ('', str) + # Prometheus settings + PROMETHEUS_API_HOST = ('', str) # Not in use ATM + ALERTMANAGER_API_HOST = ('', str) + @staticmethod def has_default_value(name): return getattr(Settings, name, None) is None or \ diff --git a/src/pybind/mgr/dashboard/tests/test_prometheus.py b/src/pybind/mgr/dashboard/tests/test_prometheus.py new file mode 100644 index 00000000000..5961187c8ce --- /dev/null +++ b/src/pybind/mgr/dashboard/tests/test_prometheus.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +from .. import mgr +from ..controllers import BaseController, Controller +from ..controllers.prometheus import Prometheus, PrometheusReceiver + +from .helper import ControllerTestCase + + +@Controller('alertmanager/mocked/api/v1/alerts', secure=False) +class AlertManagerMockInstance(BaseController): + def __call__(self, path, **params): + return 'Some Api {}'.format(path) + + +class PrometheusControllerTest(ControllerTestCase): + @classmethod + def setup_server(cls): + settings = { + 'ALERTMANAGER_API_HOST': 'http://localhost:{}/alertmanager/mocked/'.format(54583) + } + mgr.get_module_option.side_effect = settings.get + Prometheus._cp_config['tools.authenticate.on'] = False # pylint: disable=protected-access + cls.setup_controllers([AlertManagerMockInstance, Prometheus, PrometheusReceiver]) + + def test_list(self): + self._get('/api/prometheus') + self.assertStatus(200) + + def test_post_on_receiver(self): + PrometheusReceiver.notifications = [] + self._post('/api/prometheus_receiver', {'name': 'foo'}) + self.assertEqual(len(PrometheusReceiver.notifications), 1) + notification = PrometheusReceiver.notifications[0] + self.assertEqual(notification['name'], 'foo') + self.assertTrue(len(notification['notified']) > 20) + + def test_get_last_notification_with_empty_notifications(self): + PrometheusReceiver.notifications = [] + self._post('/api/prometheus_receiver', {'name': 'foo'}) + self._post('/api/prometheus_receiver', {'name': 'bar'}) + self._post('/api/prometheus/get_notifications_since', {}) + self.assertStatus(200) + last = PrometheusReceiver.notifications[1] + self.assertEqual(self.jsonBody(), [last]) + + def test_get_no_notification_since_with_last_notification(self): + PrometheusReceiver.notifications = [] + self._post('/api/prometheus_receiver', {'name': 'foo'}) + notification = PrometheusReceiver.notifications[0] + self._post('/api/prometheus/get_notifications_since', notification) + self.assertBody('[]') + + def test_get_empty_list_with_no_notifications(self): + PrometheusReceiver.notifications = [] + self._post('/api/prometheus/get_notifications_since', {}) + self.assertEqual(self.jsonBody(), []) + + def test_get_notifications_since_last_notification(self): + PrometheusReceiver.notifications = [] + self._post('/api/prometheus_receiver', {'name': 'foobar'}) + next_to_last = PrometheusReceiver.notifications[0] + self._post('/api/prometheus_receiver', {'name': 'foo'}) + self._post('/api/prometheus_receiver', {'name': 'bar'}) + self._post('/api/prometheus/get_notifications_since', next_to_last) + foreLast = PrometheusReceiver.notifications[1] + last = PrometheusReceiver.notifications[2] + self.assertEqual(self.jsonBody(), [foreLast, last])