1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
# Copyright 2018 BMW Car IT GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import math
import psutil
from zuul.executor.sensors import SensorInterface
from zuul.lib.config import get_default
CGROUP_STATS_FILE = '/sys/fs/cgroup/memory/memory.stat'
def get_avail_mem_pct():
avail_mem_pct = 100.0 - psutil.virtual_memory().percent
return avail_mem_pct
class RAMSensor(SensorInterface):
log = logging.getLogger("zuul.executor.sensor.ram")
def __init__(self, config=None):
self.min_avail_mem = float(get_default(config, 'executor',
'min_avail_mem', '5.0'))
self.cgroup_stats_file = CGROUP_STATS_FILE
def _read_cgroup_stat(self):
stat = {}
try:
with open(self.cgroup_stats_file) as f:
for line in f.readlines():
key, value = line.split(' ')
stat[key] = int(value.strip())
except Exception:
pass
return stat
def _get_cgroup_limit(self):
stat = self._read_cgroup_stat()
limit = stat.get('hierarchical_memory_limit', math.inf)
mem_total = psutil.virtual_memory().total
if limit < mem_total:
return limit
else:
return math.inf
def _get_avail_mem_pct_cgroup(self):
stat = self._read_cgroup_stat()
limit = stat.get('hierarchical_memory_limit', math.inf)
usage = stat.get('total_rss', math.inf)
if math.isinf(limit) or math.isinf(usage):
# pretend we have all memory available if we got infs
return 100
return 100.0 - usage / limit * 100
def isOk(self):
avail_mem_pct = get_avail_mem_pct()
if avail_mem_pct < self.min_avail_mem:
return False, "low memory {:3.1f}% < {}".format(
avail_mem_pct, self.min_avail_mem)
if math.isinf(self._get_cgroup_limit()):
# we have no cgroup defined limit so we're done now
return True, "{:3.1f}% <= {}".format(
avail_mem_pct, self.min_avail_mem)
avail_mem_pct_cgroup = self._get_avail_mem_pct_cgroup()
if avail_mem_pct_cgroup < self.min_avail_mem:
return False, "low memory cgroup {:3.1f}% < {}".format(
avail_mem_pct_cgroup, self.min_avail_mem)
return True, "{:3.1f}% <= {}, {:3.1f}% <= {}".format(
avail_mem_pct, self.min_avail_mem,
avail_mem_pct_cgroup, self.min_avail_mem)
def reportStats(self, statsd, base_key):
avail_mem_pct = get_avail_mem_pct()
statsd.gauge(base_key + '.pct_used_ram',
int((100.0 - avail_mem_pct) * 100))
if math.isfinite(self._get_cgroup_limit()):
avail_mem_pct_cgroup = self._get_avail_mem_pct_cgroup()
statsd.gauge(base_key + '.pct_used_ram_cgroup',
int((100.0 - avail_mem_pct_cgroup) * 100))
|
