ServiceNow Prometheus and Grafana Integration Guide

(function process(request, response) {
    try {
        var payload = JSON.parse(request.body.data);
        var alerts = payload.alerts || [];
        
        for (var i = 0; i < alerts.length; i++) {
            var alert = alerts[i];
            if (alert.status === 'firing') {
                var inc = new GlideRecord('incident');
                inc.initialize();
                inc.short_description = alert.labels.alertname + ' - ' + alert.labels.instance;
                inc.description = alert.annotations.description || alert.annotations.summary;
                inc.urgency = alert.labels.severity === 'critical' ? '1' : '3';
                inc.impact = '2';
                inc.caller_id = gs.getUserID();
                inc.assignment_group = 'Network';
                inc.work_notes = 'Created from Prometheus alert: ' + alert.generatorURL;
                var incidentSysId = inc.insert();
                gs.log('Created incident ' + inc.number + ' from Prometheus alert: ' + alert.labels.alertname);
            }
        }
        
        response.setStatus(200);
        response.setBody({status: 'success', processed: alerts.length});
    } catch (e) {
        gs.error('Prometheus webhook processing error: ' + e.message);
        response.setStatus(400);
        response.setBody({error: e.message});
    }
})(request, response);

global:
  smtp_smarthost: 'localhost:587'

route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 1h
  receiver: 'servicenow-webhook'

receivers:
- name: 'servicenow-webhook'
  webhook_configs:
  - url: 'https://your-instance.service-now.com/api/now/prometheus_webhook/alert'
    http_config:
      basic_auth:
        username: 'integration_user'
        password: 'your_password'
    send_resolved: true

(function process(request, response) {
    try {
        var payload = JSON.parse(request.body.data);
        
        if (payload.state === 'alerting') {
            // Check for existing incident to prevent duplicates
            var existingInc = new GlideRecord('incident');
            existingInc.addQuery('work_notes', 'CONTAINS', 'Grafana Alert ID: ' + payload.ruleId);
            existingInc.query();
            
            if (!existingInc.hasNext()) {
                var inc = new GlideRecord('incident');
                inc.initialize();
                inc.short_description = payload.ruleName + ' - ' + payload.title;
                inc.description = payload.message;
                inc.urgency = payload.state === 'alerting' ? '2' : '3';
                inc.impact = '2';
                inc.caller_id = gs.getUserID();
                inc.assignment_group = 'Network';
                inc.work_notes = 'Created from Grafana alert. Dashboard: ' + payload.ruleName + ', Alert ID: ' + payload.ruleId + ', URL: ' + payload.ruleUrl;
                inc.insert();
                gs.log('Created incident from Grafana alert: ' + payload.ruleName);
            }
        }
        
        response.setStatus(200);
        response.setBody({status: 'processed'});
    } catch (e) {
        gs.error('Grafana webhook processing error: ' + e.message);
        response.setStatus(400);
        response.setBody({error: e.message});
    }
})(request, response);

{
  "name": "ServiceNow Integration",
  "type": "webhook",
  "settings": {
    "url": "https://your-instance.service-now.com/api/now/prometheus_webhook/grafana",
    "username": "integration_user",
    "password": "your_password",
    "httpMethod": "POST",
    "maxAlerts": 10
  }
}

(function process(request, response) {
    try {
        var payload = JSON.parse(request.body.data);
        var alerts = payload.alerts || [];
        
        for (var i = 0; i < alerts.length; i++) {
            var alert = alerts[i];
            if (alert.status === 'resolved') {
                var inc = new GlideRecord('incident');
                inc.addQuery('work_notes', 'CONTAINS', alert.labels.alertname);
                inc.addQuery('state', '!=', '6'); // Not resolved
                inc.addQuery('state', '!=', '7'); // Not closed
                inc.query();
                
                while (inc.next()) {
                    inc.state = '6'; // Resolved
                    inc.resolution_code = 'Solved Remotely (Hardware)';
                    inc.resolution_notes = 'Alert resolved automatically by monitoring system at ' + alert.endsAt;
                    inc.update();
                    gs.log('Auto-resolved incident ' + inc.number + ' - Alert cleared: ' + alert.labels.alertname);
                }
            }
        }
        
        response.setStatus(200);
        response.setBody({status: 'resolved alerts processed'});
    } catch (e) {
        response.setStatus(400);
        response.setBody({error: e.message});
    }
})(request, response);

var restMessage = new sn_ws.RESTMessageV2('PrometheusAPI', 'POST');
restMessage.setStringParameterNoEscape('alertname', current.work_notes.toString().match(/alertname=([^\s,]+)/)[1]);
restMessage.setRequestBody(JSON.stringify({
    'matchers': [{
        'name': 'alertname',
        'value': alertname,
        'isRegex': false
    }],
    'startsAt': new GlideDateTime().getDisplayValue(),
    'endsAt': new GlideDateTime().addSeconds(3600).getDisplayValue(),
    'createdBy': 'ServiceNow',
    'comment': 'Alert silenced - ServiceNow incident ' + current.number + ' assigned to ' + current.assigned_to.getDisplayValue()
}));

var response = restMessage.execute();
var responseBody = response.getBody();
var httpStatus = response.getStatusCode();

if (httpStatus == 200) {
    gs.log('Alert silenced in Prometheus for incident: ' + current.number);
} else {
    gs.error('Failed to silence alert in Prometheus: ' + responseBody);
}

// Test script to validate webhook endpoints
var testPayload = {
    'alerts': [{
        'status': 'firing',
        'labels': {
            'alertname': 'TestAlert',
            'instance': 'test-server',
            'severity': 'warning'
        },
        'annotations': {
            'description': 'This is a test alert for integration validation',
            'summary': 'Test Alert Summary'
        },
        'generatorURL': 'http://prometheus:9090/graph'
    }]
};

var request = new sn_ws.RESTMessageV2();
request.setEndpoint('https://your-instance.service-now.com/api/now/prometheus_webhook/alert');
request.setHttpMethod('POST');
request.setBasicAuth('integration_user', 'your_password');
request.setRequestHeader('Content-Type', 'application/json');
request.setRequestBody(JSON.stringify(testPayload));

var response = request.execute();
gs.log('Test webhook response: ' + response.getStatusCode() + ' - ' + response.getBody());