从Golang程序中调用Prometheus API，以提取由导出器收集的指标信息
作为从Prometheus服务器提取数据的一种方式，Prometheus提供了HTTP API。
我发现有很多关于如何定制和创建Exporter的文章，但是关于如何从Prometheus的HTTP API获取数据的信息并不常见，所以我想记录下来以备忘。
尝试使用curl调用Prometheus的HTTP API
$ curl -G "http://<prometheus ホスト:ポート>/api/v1/targets" | jq .
{
"status": "success",
"data": {
"activeTargets": [
{
"discoveredLabels": {
"__address__": "192.168.212.101:8001",
"__metrics_path__": "/metrics",
"__scheme__": "http",
"job": "kong"
},
"labels": {
"instance": "192.168.212.101:8001",
"job": "kong"
},
"scrapeUrl": "http://192.168.212.101:8001/metrics",
"lastError": "",
"lastScrape": "2019-10-29T11:07:40.26459926Z",
"health": "up"
},
{
"discoveredLabels": {
"__address__": "localhost:9090",
"__metrics_path__": "/metrics",
"__scheme__": "http",
"job": "prometheus"
},
"labels": {
"instance": "localhost:9090",
"job": "prometheus"
},
"scrapeUrl": "http://localhost:9090/metrics",
"lastError": "",
"lastScrape": "2019-10-29T11:07:33.962713961Z",
"health": "up"
}
],
"droppedTargets": []
}
}
job名としてkongと設定したtargetのメタデータを取得する例です。(kong api gatewayのメトリクスリストが取れているのがわかります。)
$ curl -G 'http://<prometheus ホスト:ポート>/api/v1/targets/metadata' --data-urlencode 'match_target={job="kong"}' | jq .
{
"status": "success",
"data": [
{
"target": {
"instance": "192.168.212.101:8001",
"job": "kong"
},
"metric": "kong_nginx_http_current_connections",
"type": "gauge",
"help": "Number of HTTP connections",
"unit": ""
},
{
"target": {
"instance": "192.168.212.101:8001",
"job": "kong"
},
"metric": "kong_nginx_metric_errors_total",
"type": "counter",
"help": "Number of nginx-lua-prometheus errors",
"unit": ""
},
{
"target": {
"instance": "192.168.212.101:8001",
"job": "kong"
},
"metric": "kong_bandwidth",
"type": "counter",
"help": "Total bandwidth in bytes consumed per service in Kong",
"unit": ""
},
{
"target": {
"instance": "192.168.212.101:8001",
"job": "kong"
},
"metric": "kong_datastore_reachable",
"type": "gauge",
"help": "Datastore reachable from Kong, 0 is unreachable",
"unit": ""
},
{
"target": {
"instance": "192.168.212.101:8001",
"job": "kong"
},
"metric": "kong_http_status",
"type": "counter",
"help": "HTTP status codes per service in Kong",
"unit": ""
},
{
"target": {
"instance": "192.168.212.101:8001",
"job": "kong"
},
"metric": "kong_latency",
"type": "histogram",
"help": "Latency added by Kong, total request time and upstream latency for each service in Kong",
"unit": ""
}
]
}
kong_nginx_http_current_connectionsメトリクスの値に対して、Tue 29 Oct 2019 08:36:42 PM JSTからTue 29 Oct 2019 08:46:42 PM JSTの間の10分間のデータを1分間隔で集計した時系列データを取得する例。
$ curl -G http://<prometheus ホスト:ポート>/api/v1/query_range --data-urlencode 'query=kong_nginx_http_current_connections' -d 'start=1572349002' -d 'end=1572349602'
-d 'step=60s' | jq .
{
"status": "success",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"__name__": "kong_nginx_http_current_connections",
"instance": "192.168.212.101:8001",
"job": "kong",
"state": "accepted"
},
"values": [
[
1572349002,
"6"
],
[
1572349062,
"6"
],
[
1572349122,
"6"
],
[
1572349182,
"6"
],
[
1572349242,
"6"
],
[
1572349302,
"6"
],
[
1572349362,
"6"
],
[
1572349422,
"6"
],
[
1572349482,
"6"
],
[
1572349542,
"6"
],
[
1572349602,
"6"
]
]
},
{
"metric": {
"__name__": "kong_nginx_http_current_connections",
"instance": "192.168.212.101:8001",
"job": "kong",
"state": "active"
},
"values": [
[
1572349002,
"3"
],
[
1572349062,
"3"
],
[
1572349122,
"3"
],
[
1572349182,
"3"
],
[
1572349242,
"3"
],
[
1572349302,
"3"
],
[
1572349362,
"3"
],
[
1572349422,
"3"
],
[
1572349482,
"3"
],
[
1572349542,
"3"
],
[
1572349602,
"3"
]
]
},
{
"metric": {
"__name__": "kong_nginx_http_current_connections",
"instance": "192.168.212.101:8001",
"job": "kong",
"state": "handled"
},
"values": [
[
1572349002,
"6"
],
[
1572349062,
"6"
],
[
1572349122,
"6"
],
[
1572349182,
"6"
],
[
1572349242,
"6"
],
[
1572349302,
"6"
],
[
1572349362,
"6"
],
[
1572349422,
"6"
],
[
1572349482,
"6"
],
[
1572349542,
"6"
],
[
1572349602,
"6"
]
]
},
・・・略
}
]
}
}
尝试使用Prometheus的golang客户端库进行获取
要使用库，首先需要通过go get命令获取如下所示的库。
$ go get github.com/prometheus/client_golang/api
$ go get github.com/prometheus/client_golang/api/prometheus/v1
让我们在golang中尝试获取与上述curl处理相同的处理。
package main
import (
"github.com/prometheus/client_golang/api"
v1 "github.com/prometheus/client_golang/api/prometheus/v1"
"fmt"
"os"
"time"
"context"
)
func main() {
client, err := api.NewClient(api.Config{
Address: "http://192.168.212.101:9099",
})
if err != nil {
fmt.Printf("Error creating client: %v\n", err)
os.Exit(1)
}
api := v1.NewAPI(client)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
// /api/v1/targets
result, err := api.Targets(ctx)
if err != nil {
fmt.Printf("Error get Targets: %v\n", err)
os.Exit(1)
}
fmt.Printf("Result:\n%v\n", result)
// /api/v1/target/metadata
result_metric_metadata, err := api.TargetsMetadata(ctx, "{job=\"kong\"}", "", "")
if err != nil {
fmt.Printf("Error get target metadata: %v\n", err)
os.Exit(1)
}
fmt.Printf("Result TargetsMetadata: \n%v\n", result_metric_metadata)
// /api/v1/query_range
query := "kong_nginx_http_current_connections"
now := time.Now()
range_param := v1.Range{Start: now.Add(-time.Hour), End: now, Step: 60 * time.Second}
result_query_range, warning, err := api.QueryRange(ctx, query, range_param)
if err != nil {
fmt.Printf("Error get query range: %v\n", err)
os.Exit(1)
}
if len(warning) > 0 {
fmt.Printf("Warning QueryRange: %v\n", warning)
}
fmt.Printf("Result QueryRange: \n%v\n", result_query_range)
}
当你尝试执行时，会得到以下的结果。
$ go run prometheus_api_sample.go
Result:
{[{map[__address__:192.168.212.101:8001 __metrics_path__:/metrics __scheme__:http job:kong] {instance="192.168.212.101:8001", job="kong"} http://192.168.212.101:8001/metrics 2019-10-29
15:06:10.892269867 +0000 UTC up} {map[__address__:localhost:9090 __metrics_path__:/metrics __scheme__:http job:prometheus] {instance="localhost:9090", job="prometheus"} http://localhost:9090/metrics 2019-10-29 15:06:19.589531881 +0000 UTC up}] []}
Result TargetsMetadata:
[{map[instance:192.168.212.101:8001 job:kong] kong_nginx_http_current_connections gauge Number of HTTP connections } {map[instance:192.168.212.101:8001 job:kong] kong_nginx_metric_errors_total counter Number of nginx-lua-prometheus errors } {map[instance:192.168.212.101:8001 job:kong] kong_bandwidth counter Total bandwidth in bytes consumed per service in Kong } {map[instance:192.168.212.101:8001 job:kong] kong_datastore_reachable gauge Datastore reachable from Kong, 0 is unreachable } {map[instance:192.168.212.101:8001 job:kong] kong_http_status counter HTTP status codes per service in Kong } {map[instance:192.168.212.101:8001 job:kong] kong_latency histogram Latency added by Kong, total request time and upstream latency for each service in Kong }]
Result QueryRange:
kong_nginx_http_current_connections{instance="192.168.212.101:8001", job="kong", state="accepted"} =>
14 @[1572358764.66]
14 @[1572358824.66]
14 @[1572358884.66]
・・・略
17 @[1572361464.66]
17 @[1572361524.66]
17 @[1572361584.66]
kong_nginx_http_current_connections{instance="192.168.212.101:8001", job="kong", state="active"} =>
3 @[1572358764.66]
3 @[1572358824.66]
・・・略
3 @[1572359724.66]
3 @[1572359784.66]
・・・略
