将Alertmanager创建的Silence通知到Slack

最近我有机会使用Prometheus的Alertmanager，所以我制作了一个小工具。记下来备忘。

想做什么呢？

通过在 Alertmanager 中注册 Silence，可以控制生成警报时的通知与否。Silence 具有 Active、Pending 和 Expired 三种状态，在指定时间之后会自动转换状态。然而，不会自动确认状态转换，并且已经过数小时的 Expired Silence 会被垃圾回收而无法追踪其历史记录。这样就无法实时跟踪操作，并且稍显不便。因此，我希望创建一个工具，可以将 Silence 的状态变化通知到 Slack。

执行

内容很简洁，只是通过Alertmanager API定期获取Silence，并通知到Slack，提示与上次获取的差异。

只需要指定API的获取主机、端口等即可。

./notifier \ 
 --alertmanager.host=localhost \
 --alertmanager.port=9093 \
 --slack.username=Bot \
 --slack.channel=general \
 --slack.token={your webhook token} \
 --interval=10s \
 --timerange=5m

• 一定間隔でAPIからSilenceを取得

• 前回取得との差分を抽出

slackに出力

这种感觉是在重复这个过程。
在API中，alertmanager已经提供了一个可以很好地获取静默信息的库，
而且已经存在一个可以将消息发送到slack webhook的库，就是”go-slack-webhook”。
所以，基本上没有什么需要手工制作的，感觉很好。

package main

import (
    "fmt"
    "os"
    "time"
    "net/url"
    "gopkg.in/alecthomas/kingpin.v2"
    "github.com/prometheus/alertmanager/api/v2/client/silence"
    "github.com/prometheus/alertmanager/api/v2/models"
    "github.com/prometheus/alertmanager/cli"
    "github.com/ashwanthkumar/slack-go-webhook"
)

func main() {
    os.Exit(run())
}

type IdAndState struct {
    id string
    state string
}

func run() int {
    var (
        host         = kingpin.Flag("alertmanager.host", "Alertmanager host.").Default("localhost").String()
        port         = kingpin.Flag("alertmanager.port", "Alertmanager port.").Default("9093").String()
        username     = kingpin.Flag("slack.username", "username of slack bot.").Default("Bot").String()
        channel      = kingpin.Flag("slack.channel", "post channel.").Default("general").String()
        token        = kingpin.Flag("slack.token", "slack api token.").Default("xxx").String()
        interval     = kingpin.Flag("interval", "api polling interval.").Default("5s").Duration()
        timerange    = kingpin.Flag("timerange", "api polling time range.").Default("5m").Duration()
    )
    kingpin.CommandLine.GetFlag("help").Short('h')
    kingpin.Parse()

    u, _ := url.Parse("http://" + *host + ":" + *port)
    silenceParams := silence.NewGetSilencesParams()
    amclient := cli.NewAlertmanagerClient(u)

    prev := make([]IdAndState, 0)
    getOk, err := amclient.Silence.GetSilences(silenceParams)
    if err != nil {
        fmt.Println(err)
        return 1
    }
    for _, silence := range getOk.Payload {
        if time.Time(*silence.EndsAt).After(time.Now().UTC().Add(- *timerange)) {
            prev = append(prev, IdAndState{*silence.ID, *silence.Status.State})
        }
    }

    for {
        tmp := make([]IdAndState, 0)
        getOk, err = amclient.Silence.GetSilences(silenceParams)
        if err != nil {
            fmt.Println(err)
        } else {
            for _, silence := range getOk.Payload {
                if time.Time(*silence.EndsAt).After(time.Now().UTC().Add(-5 * time.Minute)) {
                    if CompareSilences(prev, *silence.ID, *silence.Status.State) {
                        PostSlack(*silence,*username,*channel,*token, *host, *port)
                    }
                    tmp = append(tmp, IdAndState{*silence.ID, *silence.Status.State})
                }
            }
            prev = tmp
        }
        time.Sleep(*interval)
    }
    return 0
}

func CompareSilences(list []IdAndState, id string, state string) bool {
    for _, v := range list {
        if v.id == id && v.state == state {
            return false
        }
    }
    return true
}

func PostSlack(s models.GettableSilence, username string, channel string, token string, host string, port string) error {
    titleStr := *s.Status.State + " : " + *s.ID
    valueStr := "Starts at: " + s.Silence.StartsAt.String() + "\n" +
        "Ends at     : " + s.Silence.EndsAt.String() + "\n" +
        "Updated at  : " + s.UpdatedAt.String() + "\n" +
        "Created by  : " + *s.Silence.CreatedBy + "\n" +
        "Comment     : " + *s.Silence.Comment + "\n" +
        "Matchers:\n"
    for _, matcher := range s.Silence.Matchers {
        var operator string
        if *matcher.IsRegex {
            operator = "~="
        } else {
            operator = "="
        }
        valueStr += *matcher.Name + operator + *matcher.Value + "\n"
    }
    silenceUrl := "http://" + host + ":" + port + "/#/silences/" + *s.ID
    attachment := slack.Attachment{}
    attachment.AddField(slack.Field{ Title: titleStr, Value: valueStr })
    attachment.AddAction(slack.Action { Type: "button", Text: "View", Url: silenceUrl })
    var color string
    var msg string
    if *s.Status.State == "active" {
        color = "good"
        msg = "New Silence!"
    } else if *s.Status.State == "pending" {
        color = "warning"
        msg = "New Silence!"
    } else {
        color = "danger"
        msg = "Expired Silence!"
    }
    attachment.Color = &color
    payload := slack.Payload {
        Username: username,
        Channel: channel,
        Text: msg,
        Attachments: []slack.Attachment{attachment},
    }
    err := slack.Send("https://hooks.slack.com/services/" + token, "", payload)
    if err != nil {
        return err[0]
    }
    return nil
}