testing/kubelet: Failed to run on ppc64le with missing deferreturn error
kubelet v1.18.6 fails to start. The error output is:
$ kubelet
I0716 14:29:02.074584 4612 server.go:417] Version: v1.18.6
I0716 14:29:02.074811 4612 plugins.go:100] No cloud provider specified.
W0716 14:29:02.074832 4612 server.go:560] standalone mode, no API client
W0716 14:29:02.085164 4612 info.go:51] Couldn't collect info from any of the files in "/etc/machine-id,/var/lib/dbus/machine-id"
W0716 14:29:02.085492 4612 server.go:474] No api server defined - no events will be sent to API server.
I0716 14:29:02.085510 4612 server.go:647] --cgroups-per-qos enabled, but --cgroup-root was not specified. defaulting to /
I0716 14:29:02.085856 4612 container_manager_linux.go:266] container manager verified user specified cgroup-root exists: []
fatal error: missing deferreturn
runtime stack:
runtime.throw(0x139415b9, 0x13)
/usr/lib/go/src/runtime/panic.go:1116 +0x5c
runtime.addOneOpenDeferFrame.func1.1(0x7fff8e03f7e8, 0x0, 0x164145c0)
/usr/lib/go/src/runtime/panic.go:753 +0x258
runtime.gentraceback(0x1004fc44, 0xc000fbd310, 0x0, 0xc000000180, 0x0, 0x0, 0x7fffffff, 0x7fff8e03f8b8, 0x0, 0x0, ...)
/usr/lib/go/src/runtime/traceback.go:334 +0xea0
runtime.addOneOpenDeferFrame.func1()
/usr/lib/go/src/runtime/panic.go:721 +0x8c
runtime.systemstack(0x0)
/usr/lib/go/src/runtime/asm_ppc64x.s:269 +0x94
runtime.mstart()
/usr/lib/go/src/runtime/proc.go:1041
goroutine 1 [running]:
runtime.systemstack_switch()
/usr/lib/go/src/runtime/asm_ppc64x.s:216 +0x10 fp=0xc000fbd1f0 sp=0xc000fbd1d0 pc=0x10068f20
runtime.addOneOpenDeferFrame(0xc000000180, 0x1004fc44, 0xc000fbd310)
/usr/lib/go/src/runtime/panic.go:720 +0x7c fp=0xc000fbd240 sp=0xc000fbd1f0 pc=0x1003649c
panic(0x130d2380, 0x163c1f20)
/usr/lib/go/src/runtime/panic.go:929 +0xdc fp=0xc000fbd310 sp=0xc000fbd240 pc=0x10036adc
runtime.panicmem(...)
/usr/lib/go/src/runtime/panic.go:212
runtime.sigpanic()
/usr/lib/go/src/runtime/signal_unix.go:695 +0x3f4 fp=0xc000fbd350 sp=0xc000fbd310 pc=0x1004fc44
k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/api/resource.(*Quantity).String(0x0, 0x138b6480, 0x0)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/api/resource/quantity.go:601 +0x28 fp=0xc000fbd410 sp=0xc000fbd370 pc=0x104cf2b8
fmt.(*pp).handleMethods(0xc00019c5b0, 0xc000000076, 0x101)
/usr/lib/go/src/fmt/print.go:630 +0x28c fp=0xc000fbd6b0 sp=0xc000fbd410 pc=0x100e8bfc
fmt.(*pp).printValue(0xc00019c5b0, 0x138b6480, 0xc000050c60, 0x196, 0x76, 0x5)
/usr/lib/go/src/fmt/print.go:727 +0x211c fp=0xc000fbd8a0 sp=0xc000fbd6b0 pc=0x100eba7c
fmt.(*pp).printValue(0xc00019c5b0, 0x132a7660, 0xc000050c60, 0x199, 0x76, 0x4)
/usr/lib/go/src/fmt/print.go:810 +0x170c fp=0xc000fbda90 sp=0xc000fbd8a0 pc=0x100eb06c
fmt.(*pp).printValue(0xc00019c5b0, 0x13574320, 0xc000050c40, 0x199, 0x76, 0x3)
/usr/lib/go/src/fmt/print.go:810 +0x170c fp=0xc000fbdc80 sp=0xc000fbda90 pc=0x100eb06c
fmt.(*pp).printValue(0xc00019c5b0, 0x12d41d40, 0xc000036f60, 0x97, 0x76, 0x2)
/usr/lib/go/src/fmt/print.go:869 +0x3dc fp=0xc000fbde70 sp=0xc000fbdc80 pc=0x100e9d3c
fmt.(*pp).printValue(0xc00019c5b0, 0x136734a0, 0xc000036f20, 0x99, 0x76, 0x1)
/usr/lib/go/src/fmt/print.go:810 +0x170c fp=0xc000fbe060 sp=0xc000fbde70 pc=0x100eb06c
fmt.(*pp).printValue(0xc00019c5b0, 0x13861e20, 0xc000036ea0, 0x99, 0x76, 0x0)
/usr/lib/go/src/fmt/print.go:810 +0x170c fp=0xc000fbe250 sp=0xc000fbe060 pc=0x100eb06c
fmt.(*pp).printArg(0xc00019c5b0, 0x13861e20, 0xc000036ea0, 0x76)
/usr/lib/go/src/fmt/print.go:716 +0x2a8 fp=0xc000fbe2f8 sp=0xc000fbe250 pc=0x100e91f8
fmt.(*pp).doPrintf(0xc00019c5b0, 0x13a0db2b, 0x3b, 0xc000fbe6b8, 0x1, 0x1)
/usr/lib/go/src/fmt/print.go:1030 +0x140 fp=0xc000fbe408 sp=0xc000fbe2f8 pc=0x100ec5a0
fmt.Fprintf(0x14107920, 0xc000264460, 0x13a0db2b, 0x3b, 0xc000fbe6b8, 0x1, 0x1, 0x1, 0x1391d423, 0x13861e20)
/usr/lib/go/src/fmt/print.go:204 +0x58 fp=0xc000fbe480 sp=0xc000fbe408 pc=0x100e5cc8
k8s.io/kubernetes/vendor/k8s.io/klog.(*loggingT).printf(0x164cd900, 0xc000000000, 0x13a0db2b, 0x3b, 0xc000fbe6b8, 0x1, 0x1)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/klog/klog.go:703 +0x98 fp=0xc000fbe510 sp=0xc000fbe480 pc=0x1030eab8
k8s.io/kubernetes/vendor/k8s.io/klog.Infof(...)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/klog/klog.go:1201
k8s.io/kubernetes/pkg/kubelet/cm.NewContainerManager(0x141afc00, 0xc0004abbc0, 0x141d84c0, 0xc0008be9f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/pkg/kubelet/cm/container_manager_linux.go:271 +0x560 fp=0xc000fbea20 sp=0xc000fbe510 pc=0x1151fdf0
k8s.io/kubernetes/cmd/kubelet/app.run(0xc000892600, 0xc000781080, 0x7fff8e74a2c0, 0xc000474940, 0xc000364780, 0x1, 0x1)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/cmd/kubelet/app/server.go:715 +0xd60 fp=0xc000fbf9d8 sp=0xc000fbea20 pc=0x12a63880
k8s.io/kubernetes/cmd/kubelet/app.Run(0xc000892600, 0xc000781080, 0x7fff8e74a2c0, 0xc000474940, 0xc000364780, 0x0, 0x1016d30c)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/cmd/kubelet/app/server.go:421 +0xfc fp=0xc000fbfb38 sp=0xc000fbf9d8 pc=0x12a6238c
k8s.io/kubernetes/cmd/kubelet/app.NewKubeletCommand.func1(0xc0005b4000, 0xc0001921a0, 0x0, 0x0)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/cmd/kubelet/app/server.go:273 +0x51c fp=0xc000fbfd10 sp=0xc000fbfb38 pc=0x12a6837c
k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).execute(0xc0005b4000, 0xc0001921a0, 0x0, 0x0, 0xc0005b4000, 0xc0001921a0)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:830 +0x208 fp=0xc000fbfdf8 sp=0xc000fbfd10 pc=0x128b82a8
k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc0005b4000, 0x162241f5d11d704e, 0x164cd2c0, 0x1003989c)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:914 +0x258 fp=0xc000fbfee0 sp=0xc000fbfdf8 pc=0x128b8c98
k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).Execute(...)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:864
main.main()
_output/local/go/src/k8s.io/kubernetes/cmd/kubelet/kubelet.go:41 +0xdc fp=0xc000fbff50 sp=0xc000fbfee0 pc=0x12a693cc
runtime.main()
/usr/lib/go/src/runtime/proc.go:203 +0x214 fp=0xc000fbffc0 sp=0xc000fbff50 pc=0x10039914
runtime.goexit()
/usr/lib/go/src/runtime/asm_ppc64x.s:884 +0x4 fp=0xc000fbffc0 sp=0xc000fbffc0 pc=0x1006b644
goroutine 19 [chan receive]:
k8s.io/kubernetes/vendor/k8s.io/klog.(*loggingT).flushDaemon(0x164cd900)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/klog/klog.go:1010 +0x78
created by k8s.io/kubernetes/vendor/k8s.io/klog.init.0
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/klog/klog.go:411 +0xe0
goroutine 118 [sleep]:
time.Sleep(0x3b9aca00)
/usr/lib/go/src/runtime/time.go:188 +0xc4
k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1.(*metricsRecorder).run(0xc0001991a0)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1/metrics_recorder.go:87 +0x40
created by k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1.newMetricsRecorder
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1/metrics_recorder.go:59 +0xf4
goroutine 194 [select]:
k8s.io/kubernetes/vendor/go.opencensus.io/stats/view.(*worker).start(0xc000180190)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/go.opencensus.io/stats/view/worker.go:154 +0xd8
created by k8s.io/kubernetes/vendor/go.opencensus.io/stats/view.init.0
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/go.opencensus.io/stats/view/worker.go:32 +0x64
goroutine 223 [select]:
k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x13ab6570, 0x14106f60, 0xc000299b90, 0x1, 0xc0001820c0)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:167 +0x120
k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x13ab6570, 0x12a05f200, 0x0, 0xc000982a01, 0xc0001820c0)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x90
k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait.Until(...)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90
k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait.Forever(0x13ab6570, 0x12a05f200)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:81 +0x50
created by k8s.io/kubernetes/vendor/k8s.io/component-base/logs.InitLogs
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/component-base/logs/logs.go:58 +0x88
goroutine 224 [syscall]:
os/signal.signal_recv(0x0)
/usr/lib/go/src/runtime/sigqueue.go:147 +0xf8
os/signal.loop()
/usr/lib/go/src/os/signal/signal_unix.go:23 +0x24
created by os/signal.Notify.func1
/usr/lib/go/src/os/signal/signal.go:127 +0x4c
goroutine 226 [chan receive]:
k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.SetupSignalHandler.func1(0xc000364780)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/signal.go:38 +0x38
created by k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server.SetupSignalHandler
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/signal.go:37 +0xd8
goroutine 206 [chan receive]:
k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/watch.(*Broadcaster).loop(0xc0000def40)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/watch/mux.go:207 +0x58
created by k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/watch.NewBroadcaster
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/watch/mux.go:75 +0xc4
goroutine 207 [chan receive]:
k8s.io/kubernetes/vendor/k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x14116e80, 0xc0008bea50, 0xc0004f2650)
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/client-go/tools/record/event.go:288 +0x98
created by k8s.io/kubernetes/vendor/k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher
/home/buildozer/aports/testing/kubernetes/src/kubernetes-1.18.6/_output/local/go/src/k8s.io/kubernetes/vendor/k8s.io/client-go/tools/record/event.go:286 +0x68
I can confirm the app runs fine if compiled with go v1.13.x. This issue only impact ppc64le machine.
From the onset I thought this issue might be this reported upstream one https://github.com/golang/go/issues/39049, however I still run into the same issue with kubelet v1.18.6 (which I believe it is compiled with go 1.14.5).