Add dashboard for monitoring conntrack race failures. (#6329)

Connection tracking is an important optimization of the linux kernel to reduce the overhead of applying network filter rules to all network packets. Unfortunately, the connection tracking code in the linux kernel has some race conditions, which can lead to connection attempts loosing their connection tracking entry. This is especially possible when network address translation is used. The result is a failed connection attempt. This is why this change introduces two dashboards for the corresponding metric.
gardener · Jul 14, 2022 · d31e21c · d31e21c
1 parent e4d3e9f
commit d31e21c
Show file tree

Hide file tree

Showing 2 changed files with 215 additions and 1 deletion.
diff --git a/charts/seed-monitoring/charts/grafana/dashboards/owners/node-details-dashboard.json b/charts/seed-monitoring/charts/grafana/dashboards/owners/node-details-dashboard.json
@@ -1119,7 1119,7 @@
  "h": 9,
  "w": 24,
  "x": 0,
- "y": 25
  "y": 47
  },
  "hiddenSeries": false,
  "id": 77,
@@ -1185,6 1185,107 @@
  "show": true
  },
  {
  "$$hashKey": "object:137",
  "format": "short",
  "label": null,
  "logBase": 1,
  "max": null,
  "min": null,
  "show": true
  }
  ],
  "yaxis": {
  "align": false,
  "alignLevel": null
  }
  },
  {
  "aliasColors": {},
  "bars": false,
  "dashLength": 10,
  "dashes": false,
  "datasource": null,
  "description": "Shows the amount of failed inserts into the connection tracking table (conntrack). Should be close to zero.",
  "fieldConfig": {
  "defaults": {},
  "overrides": []
  },
  "fill": 0,
  "fillGradient": 0,
  "gridPos": {
  "h": 9,
  "w": 24,
  "x": 0,
  "y": 56
  },
  "hiddenSeries": false,
  "id": 82,
  "legend": {
  "alignAsTable": false,
  "avg": false,
  "current": false,
  "max": false,
  "min": false,
  "rightSide": false,
  "show": true,
  "total": false,
  "values": false
  },
  "lines": true,
  "linewidth": 1,
  "nullPointMode": "null",
  "options": {
  "alertThreshold": true
  },
  "percentage": false,
  "pluginVersion": "7.5.16",
  "pointradius": 2,
  "points": false,
  "renderer": "flot",
  "seriesOverrides": [],
  "spaceLength": 10,
  "stack": false,
  "steppedLine": false,
  "targets": [
  {
  "exemplar": true,
  "expr": "sum(node_nf_conntrack_stat_insert_failed{node=~\"$Node\"})",
  "interval": "",
  "legendFormat": "failed inserts",
  "refId": "A"
  }
  ],
  "thresholds": [],
  "timeFrom": null,
  "timeRegions": [],
  "timeShift": null,
  "title": "Failed Connection Tracking Inserts",
  "tooltip": {
  "shared": false,
  "sort": 0,
  "value_type": "individual"
  },
  "type": "graph",
  "xaxis": {
  "buckets": null,
  "mode": "time",
  "name": null,
  "show": true,
  "values": []
  },
  "yaxes": [
  {
  "$$hashKey": "object:134",
  "decimals": 0,
  "format": "short",
  "label": null,
  "logBase": 1,
  "max": null,
  "min": "0",
  "show": true
  },
  {
  "$$hashKey": "object:135",
  "format": "short",
  "label": null,
  "logBase": 1,

diff --git a/charts/seed-monitoring/charts/grafana/dashboards/owners/node-pool-dashboard.json b/charts/seed-monitoring/charts/grafana/dashboards/owners/node-pool-dashboard.json
@@ -609,6 609,119 @@
  "title": "Overview of all Nodes",
  "transform": "table",
  "type": "table"
  },
  {
  "collapsed": true,
  "datasource": null,
  "gridPos": {
  "h": 1,
  "w": 24,
  "x": 0,
  "y": 24
  },
  "id": 16,
  "panels": [
  {
  "aliasColors": {},
  "bars": false,
  "dashLength": 10,
  "dashes": false,
  "datasource": null,
  "description": "Shows the amount of failed inserts into the connection tracking table (conntrack). Should be close to zero.",
  "fieldConfig": {
  "defaults": {},
  "overrides": []
  },
  "fill": 0,
  "fillGradient": 0,
  "gridPos": {
  "h": 8,
  "w": 24,
  "x": 0,
  "y": 25
  },
  "hiddenSeries": false,
  "id": 14,
  "legend": {
  "avg": false,
  "current": false,
  "max": false,
  "min": false,
  "show": true,
  "total": false,
  "values": false
  },
  "lines": true,
  "linewidth": 1,
  "nullPointMode": "null",
  "options": {
  "alertThreshold": true
  },
  "percentage": false,
  "pluginVersion": "7.5.16",
  "pointradius": 2,
  "points": false,
  "renderer": "flot",
  "seriesOverrides": [],
  "spaceLength": 10,
  "stack": false,
  "steppedLine": false,
  "targets": [
  {
  "exemplar": true,
  "expr": "sum(node_nf_conntrack_stat_insert_failed) by (node)",
  "interval": "",
  "legendFormat": "{{node}}",
  "refId": "A"
  }
  ],
  "thresholds": [],
  "timeFrom": null,
  "timeRegions": [],
  "timeShift": null,
  "title": "Failed Connection Tracking Inserts",
  "tooltip": {
  "shared": true,
  "sort": 0,
  "value_type": "individual"
  },
  "type": "graph",
  "xaxis": {
  "buckets": null,
  "mode": "time",
  "name": null,
  "show": true,
  "values": []
  },
  "yaxes": [
  {
  "$$hashKey": "object:115",
  "decimals": 0,
  "format": "short",
  "label": null,
  "logBase": 1,
  "max": null,
  "min": "0",
  "show": true
  },
  {
  "$$hashKey": "object:116",
  "format": "short",
  "label": null,
  "logBase": 1,
  "max": null,
  "min": null,
  "show": true
  }
  ],
  "yaxis": {
  "align": false,
  "alignLevel": null
  }
  }
  ],
  "title": "Node Exporter",
  "type": "row"
  }
  ],
  "refresh": "1m",