Skip to content

Commit 4dbba42

Browse files
MustafaSaberMustafaSaber
committed
Ignore services with 1 replica if needed
Signed-off-by: Mostafa Saber <mostafa.saber3789@gmail.com>
1 parent 57909ce commit 4dbba42

File tree

3 files changed

+78
-26
lines changed

3 files changed

+78
-26
lines changed

‎chaoskube/chaoskube.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ type Chaoskube struct {
7474
Notifier notifier.Notifier
7575
// namespace scope for the Kubernetes client
7676
ClientNamespaceScope string
77+
78+
// ignoreSingleReplicas will make chaoskube ignore services with one replica when needed.
79+
IgnoreSingleReplicas bool
7780
}
7881

7982
var (
@@ -97,7 +100,7 @@ var (
97100
// * a logger implementing logrus.FieldLogger to send log output to
98101
// * what specific terminator to use to imbue chaos on victim pods
99102
// * whether to enable/disable dry-run mode
100-
func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier, clientNamespaceScope string) *Chaoskube {
103+
func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier, clientNamespaceScope string, ignoreSingleReplicas bool) *Chaoskube {
101104
broadcaster := record.NewBroadcaster()
102105
broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(clientNamespaceScope)})
103106
recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"})
@@ -124,6 +127,7 @@ func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, na
124127
MaxKill: maxKill,
125128
Notifier: notifier,
126129
ClientNamespaceScope: clientNamespaceScope,
130+
IgnoreSingleReplicas: ignoreSingleReplicas,
127131
}
128132
}
129133

@@ -239,7 +243,7 @@ func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) {
239243
pods = filterTerminatingPods(pods)
240244
pods = filterByMinimumAge(pods, c.MinimumAge, c.Now())
241245
pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames)
242-
pods = filterByOwnerReference(pods)
246+
pods = filterByOwnerReference(pods, c.IgnoreSingleReplicas)
243247

244248
return pods, nil
245249
}
@@ -513,7 +517,7 @@ func filterByPodName(pods []v1.Pod, includedPodNames, excludedPodNames *regexp.R
513517
return filteredList
514518
}
515519

516-
func filterByOwnerReference(pods []v1.Pod) []v1.Pod {
520+
func filterByOwnerReference(pods []v1.Pod, ignoreSingleReplicas bool) []v1.Pod {
517521
owners := make(map[types.UID][]v1.Pod)
518522
filteredList := []v1.Pod{}
519523
for _, pod := range pods {
@@ -531,6 +535,9 @@ func filterByOwnerReference(pods []v1.Pod) []v1.Pod {
531535

532536
// For each owner reference select a random pod from its group
533537
for _, pods := range owners {
538+
if ignoreSingleReplicas && len(pods) <= 1 {
539+
continue
540+
}
534541
filteredList = append(filteredList, util.RandomPodSubSlice(pods, 1)...)
535542
}
536543

‎chaoskube/chaoskube_test.go‎

Lines changed: 64 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -48,22 +48,23 @@ func (suite *Suite) SetupTest() {
4848
// TestNew tests that arguments are passed to the new instance correctly
4949
func (suite *Suite) TestNew() {
5050
var (
51-
client = fake.NewSimpleClientset()
52-
labelSelector, _ = labels.Parse("foo=bar")
53-
annotations, _ = labels.Parse("baz=waldo")
54-
kinds, _ = labels.Parse("job")
55-
namespaces, _ = labels.Parse("qux")
56-
namespaceLabels, _ = labels.Parse("taz=wubble")
57-
includedPodNames = regexp.MustCompile("foo")
58-
excludedPodNames = regexp.MustCompile("bar")
59-
excludedWeekdays = []time.Weekday{time.Friday}
60-
excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}}
61-
excludedDaysOfYear = []time.Time{time.Now()}
62-
minimumAge = time.Duration(42)
63-
dryRun = true
64-
terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second)
65-
maxKill = 1
66-
notifier = testNotifier
51+
client = fake.NewSimpleClientset()
52+
labelSelector, _ = labels.Parse("foo=bar")
53+
annotations, _ = labels.Parse("baz=waldo")
54+
kinds, _ = labels.Parse("job")
55+
namespaces, _ = labels.Parse("qux")
56+
namespaceLabels, _ = labels.Parse("taz=wubble")
57+
includedPodNames = regexp.MustCompile("foo")
58+
excludedPodNames = regexp.MustCompile("bar")
59+
excludedWeekdays = []time.Weekday{time.Friday}
60+
excludedTimesOfDay = []util.TimePeriod{{}}
61+
excludedDaysOfYear = []time.Time{time.Now()}
62+
minimumAge = time.Duration(42)
63+
dryRun = true
64+
terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second)
65+
maxKill = 1
66+
notifier = testNotifier
67+
ignoreSingleReplicas = false
6768
)
6869

6970
chaoskube := New(
@@ -86,6 +87,7 @@ func (suite *Suite) TestNew() {
8687
maxKill,
8788
notifier,
8889
v1.NamespaceAll,
90+
ignoreSingleReplicas,
8991
)
9092
suite.Require().NotNil(chaoskube)
9193

@@ -105,6 +107,7 @@ func (suite *Suite) TestNew() {
105107
suite.Equal(logger, chaoskube.Logger)
106108
suite.Equal(dryRun, chaoskube.DryRun)
107109
suite.Equal(terminator, chaoskube.Terminator)
110+
suite.Equal(ignoreSingleReplicas, chaoskube.IgnoreSingleReplicas)
108111
}
109112

110113
// TestRunContextCanceled tests that a canceled context will exit the Run function.
@@ -126,6 +129,7 @@ func (suite *Suite) TestRunContextCanceled() {
126129
10,
127130
1,
128131
v1.NamespaceAll,
132+
false,
129133
)
130134

131135
ctx, cancel := context.WithCancel(context.Background())
@@ -182,6 +186,7 @@ func (suite *Suite) TestCandidates() {
182186
false,
183187
10,
184188
v1.NamespaceAll,
189+
false,
185190
)
186191

187192
suite.assertCandidates(chaoskube, tt.pods)
@@ -228,6 +233,7 @@ func (suite *Suite) TestCandidatesNamespaceLabels() {
228233
false,
229234
10,
230235
v1.NamespaceAll,
236+
false,
231237
)
232238

233239
suite.assertCandidates(chaoskube, tt.pods)
@@ -262,6 +268,7 @@ func (suite *Suite) TestCandidatesClientNamespaceScope() {
262268
false,
263269
10,
264270
tt.clientNamespaceScope,
271+
false,
265272
)
266273

267274
suite.assertCandidates(chaoskube, tt.pods)
@@ -306,6 +313,7 @@ func (suite *Suite) TestCandidatesPodNameRegexp() {
306313
false,
307314
10,
308315
v1.NamespaceAll,
316+
false,
309317
)
310318

311319
suite.assertCandidates(chaoskube, tt.pods)
@@ -347,6 +355,7 @@ func (suite *Suite) TestVictim() {
347355
false,
348356
10,
349357
v1.NamespaceAll,
358+
false,
350359
)
351360

352361
suite.assertVictim(chaoskube, tt.victim)
@@ -402,6 +411,7 @@ func (suite *Suite) TestVictims() {
402411
10,
403412
tt.maxKill,
404413
v1.NamespaceAll,
414+
false,
405415
)
406416
suite.createPods(chaoskube.Client, podsInfo)
407417

@@ -428,6 +438,7 @@ func (suite *Suite) TestNoVictimReturnsError() {
428438
10,
429439
1,
430440
v1.NamespaceAll,
441+
false,
431442
)
432443

433444
_, err := chaoskube.Victims(context.Background())
@@ -463,6 +474,7 @@ func (suite *Suite) TestDeletePod() {
463474
tt.dryRun,
464475
10,
465476
v1.NamespaceAll,
477+
false,
466478
)
467479

468480
victim := util.NewPod("default", "foo", v1.PodRunning)
@@ -494,6 +506,7 @@ func (suite *Suite) TestDeletePodNotFound() {
494506
10,
495507
1,
496508
v1.NamespaceAll,
509+
false,
497510
)
498511

499512
victim := util.NewPod("default", "foo", v1.PodRunning)
@@ -726,6 +739,7 @@ func (suite *Suite) TestTerminateVictim() {
726739
false,
727740
10,
728741
v1.NamespaceAll,
742+
false,
729743
)
730744
chaoskube.Now = tt.now
731745

@@ -758,6 +772,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() {
758772
10,
759773
1,
760774
v1.NamespaceAll,
775+
false,
761776
)
762777

763778
err := chaoskube.TerminateVictims(context.Background())
@@ -792,7 +807,7 @@ func (suite *Suite) assertNotified(notifier *notifier.Noop) {
792807
suite.Assert().Greater(notifier.Calls, 0)
793808
}
794809

795-
func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, clientNamespaceScope string) *Chaoskube {
810+
func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, clientNamespaceScope string, ignoreSingleReplicas bool) *Chaoskube {
796811
chaoskube := suite.setup(
797812
labelSelector,
798813
annotations,
@@ -810,6 +825,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab
810825
gracePeriod,
811826
1,
812827
clientNamespaceScope,
828+
ignoreSingleReplicas,
813829
)
814830

815831
for _, namespace := range []v1.Namespace{
@@ -845,7 +861,7 @@ func (suite *Suite) createPods(client kubernetes.Interface, podsInfo []podInfo)
845861
}
846862
}
847863

848-
func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, maxKill int, clientNamespaceScope string) *Chaoskube {
864+
func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, maxKill int, clientNamespaceScope string, ignoreSingleReplicas bool) *Chaoskube {
849865
logOutput.Reset()
850866

851867
client := fake.NewSimpleClientset()
@@ -871,6 +887,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele
871887
maxKill,
872888
testNotifier,
873889
clientNamespaceScope,
890+
ignoreSingleReplicas,
874891
)
875892
}
876893

@@ -977,6 +994,7 @@ func (suite *Suite) TestMinimumAge() {
977994
10,
978995
1,
979996
v1.NamespaceAll,
997+
false,
980998
)
981999
chaoskube.Now = tt.now
9821000

@@ -1091,10 +1109,11 @@ func (suite *Suite) TestFilterByOwnerReference() {
10911109
baz1 := util.NewPod("default", "baz-1", v1.PodRunning)
10921110

10931111
for _, tt := range []struct {
1094-
seed int64
1095-
name string
1096-
pods []v1.Pod
1097-
expected []v1.Pod
1112+
seed int64
1113+
name string
1114+
pods []v1.Pod
1115+
expected []v1.Pod
1116+
ignoreSingleReplicas bool
10981117
}{
10991118
{
11001119
seed: 1000,
@@ -1126,10 +1145,31 @@ func (suite *Suite) TestFilterByOwnerReference() {
11261145
pods: []v1.Pod{baz, baz1},
11271146
expected: []v1.Pod{baz, baz1},
11281147
},
1148+
{
1149+
seed: 1000,
1150+
name: "1 pod, with parent, don't pick",
1151+
pods: []v1.Pod{foo},
1152+
expected: []v1.Pod{},
1153+
ignoreSingleReplicas: true,
1154+
},
1155+
{
1156+
seed: 1000,
1157+
name: "3 pods, 2 with same parent one without, pick first from first parent and ignore second",
1158+
pods: []v1.Pod{foo, foo1, bar},
1159+
expected: []v1.Pod{foo},
1160+
ignoreSingleReplicas: true,
1161+
},
1162+
{
1163+
seed: 1000,
1164+
name: "2 pods, different parents, don't pick",
1165+
pods: []v1.Pod{foo, bar},
1166+
expected: []v1.Pod{},
1167+
ignoreSingleReplicas: true,
1168+
},
11291169
} {
11301170
rand.Seed(tt.seed)
11311171

1132-
results := filterByOwnerReference(tt.pods)
1172+
results := filterByOwnerReference(tt.pods, tt.ignoreSingleReplicas)
11331173
suite.Require().Len(results, len(tt.expected))
11341174

11351175
// ensure returned pods are ordered by name
@@ -1160,6 +1200,7 @@ func (suite *Suite) TestNotifierCall() {
11601200
false,
11611201
10,
11621202
v1.NamespaceAll,
1203+
false,
11631204
)
11641205

11651206
victim := util.NewPod("default", "foo", v1.PodRunning)

‎main.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ var (
6262
logCaller bool
6363
slackWebhook string
6464
clientNamespaceScope string
65+
ignoreSingleReplicas bool
6566
)
6667

6768
func cliEnvVar(name string) string {
@@ -97,6 +98,7 @@ func init() {
9798
kingpin.Flag("log-caller", "Include the calling function name and location in the log messages.").Envar(cliEnvVar("LOG_CALLER")).BoolVar(&logCaller)
9899
kingpin.Flag("slack-webhook", "The address of the slack webhook for notifications").Envar(cliEnvVar("SLACK_WEBHOOK")).StringVar(&slackWebhook)
99100
kingpin.Flag("client-namespace-scope", "Scope Kubernetes API calls to the given namespace. Defaults to v1.NamespaceAll which requires global read permission.").Envar(cliEnvVar("CLIENT_NAMESPACE_SCOPE")).Default(v1.NamespaceAll).StringVar(&clientNamespaceScope)
101+
kingpin.Flag("ignore-single-replicas", "Ignore services with one replica when needed.").Envar(cliEnvVar("IGNORE_SINGLE_REPLICAS")).BoolVar(&ignoreSingleReplicas)
100102
}
101103

102104
func main() {
@@ -141,6 +143,7 @@ func main() {
141143
"logFormat": logFormat,
142144
"slackWebhook": slackWebhook,
143145
"clientNamespaceScope": clientNamespaceScope,
146+
"ignoreSingleReplicas": ignoreSingleReplicas,
144147
}).Debug("reading config")
145148

146149
log.WithFields(log.Fields{
@@ -234,6 +237,7 @@ func main() {
234237
maxKill,
235238
notifiers,
236239
clientNamespaceScope,
240+
ignoreSingleReplicas,
237241
)
238242

239243
if metricsAddress != "" {

0 commit comments

Comments
 (0)