diff --git a/tools/flashy/checks_and_remediations/yosemite4/00_kill_mfg_tool_processes.go b/tools/flashy/checks_and_remediations/yosemite4/00_kill_mfg_tool_processes.go new file mode 100644 index 000000000000..514958b82693 --- /dev/null +++ b/tools/flashy/checks_and_remediations/yosemite4/00_kill_mfg_tool_processes.go @@ -0,0 +1,62 @@ +/** + * Copyright 2020-present Facebook. All Rights Reserved. + * + * This program file is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program in a file named COPYING; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301 USA + */ + +package remediations_yosemite4 + +import ( + "github.com/facebook/openbmc/tools/flashy/lib/step" + "github.com/facebook/openbmc/tools/flashy/lib/utils" + "log" + "os" + "regexp" + "syscall" +) + +func init() { + step.RegisterStep(killMfgToolProcesses) +} + +func killMfgToolProcesses(stepParams step.StepParams) step.StepExitError { + // Find all running mfg-tool processes + mfgToolRegex := regexp.MustCompile(`(^|/)mfg-tool(\x00|$)`) + mfgToolProcs, err := utils.ListProcessesMatchingRegex(mfgToolRegex) + if err != nil { + return step.ExitSafeToReboot{Err: err} + } + log.Printf("Found %d mfg-tool running processes", len(mfgToolProcs)) + + // S458061: If there's more than one running mfg-tool process, kill them all + // to prevent stuck mfg-tool instances causing OOM problems during flashing + if len(mfgToolProcs) > 1 { + for _, proc := range mfgToolProcs { + log.Printf("Killing mfg-tool process with pid=%d", proc.Pid) + err := kill(proc) + if err != nil { + log.Printf("Failed to kill mfg-tool process with pid=%d: %v", proc.Pid, err) + } + } + } + + return nil +} + +var kill = func(proc *os.Process) error { + // Just a simple wrapper so it's easier to mock in tests + return proc.Signal(syscall.SIGKILL) +} diff --git a/tools/flashy/checks_and_remediations/yosemite4/00_kill_mfg_tool_processes_test.go b/tools/flashy/checks_and_remediations/yosemite4/00_kill_mfg_tool_processes_test.go new file mode 100644 index 000000000000..140353e2961d --- /dev/null +++ b/tools/flashy/checks_and_remediations/yosemite4/00_kill_mfg_tool_processes_test.go @@ -0,0 +1,114 @@ +/** + * Copyright 2020-present Facebook. All Rights Reserved. + * + * This program file is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program in a file named COPYING; if not, write to the + * Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301 USA + */ + + package remediations_yosemite4 + + import ( + "github.com/facebook/openbmc/tools/flashy/lib/step" + "github.com/facebook/openbmc/tools/flashy/lib/utils" + "log" + "testing" + "bytes" + "os" + "regexp" + "strings" + ) + + func TestKillMfgToolProcesses(t *testing.T) { + // save log output into buf for testing + var logBuffer bytes.Buffer + log.SetOutput(&logBuffer) + + // mock cleanup + realListProcessesMatchingRegex := utils.ListProcessesMatchingRegex + realKill := kill + + defer func() { + log.SetOutput(os.Stderr) + kill = realKill + utils.ListProcessesMatchingRegex = realListProcessesMatchingRegex + }() + + t.Run("Must not kill mfg-tool if there's a single process", func(t *testing.T) { + logBuffer.Reset() + + // Simulate a single mfg-tool process + utils.ListProcessesMatchingRegex = func(regex *regexp.Regexp) ([]*os.Process, error) { + return []*os.Process{&os.Process{Pid: 1234}}, nil + } + + // Mock kill + killCalled := false + kill = func (proc *os.Process) error { + killCalled = true + return nil; + } + + res := killMfgToolProcesses(step.StepParams{}) + + if res != nil { + t.Errorf("Expected killMfgToolProcesses() to return nil, got %v", res) + } + + if !strings.Contains(logBuffer.String(), "Found 1 mfg-tool running processes") { + t.Errorf("Expected 'Found 1 mfg-tool running processes' log message, got %v", logBuffer.String()) + } + + if killCalled { + t.Errorf("Expected killMfgToolProcesses() to not call kill() if there's a single mfg-tool process") + } + }) + + t.Run("Must kill all mfg-tool if there are multiple mfg-tool processes", func(t *testing.T) { + logBuffer.Reset() + + // Simulate multiple mfg-tool processes + utils.ListProcessesMatchingRegex = func(regex *regexp.Regexp) ([]*os.Process, error) { + return []*os.Process{&os.Process{Pid: 1234}, &os.Process{Pid: 4567}}, nil + } + + // Mock kill + killCalled := false + kill = func (proc *os.Process) error { + killCalled = true + return nil; + } + + res := killMfgToolProcesses(step.StepParams{}) + + if res != nil { + t.Errorf("Expected killMfgToolProcesses() to return nil, got %v", res) + } + + if !strings.Contains(logBuffer.String(), "Found 2 mfg-tool running processes") { + t.Errorf("Expected 'Found 2 mfg-tool running processes' log message, got %v", logBuffer.String()) + } + + // Ensure all mfg-tool processes were killed + if !strings.Contains(logBuffer.String(), "Killing mfg-tool process with pid=1234") || !strings.Contains(logBuffer.String(), "Killing mfg-tool process with pid=4567") { + t.Errorf("Expected logs showing all mfg-tool processes being killed got %v", logBuffer.String()) + } + + if !killCalled { + t.Errorf("Expected killMfgToolProcesses() to call kill() on multiple mfg-tool processes") + } + }) + + +} diff --git a/tools/flashy/install/install.go b/tools/flashy/install/install.go index fa9a2eff0cea..6767f635ddda 100644 --- a/tools/flashy/install/install.go +++ b/tools/flashy/install/install.go @@ -35,6 +35,7 @@ import ( _ "github.com/facebook/openbmc/tools/flashy/checks_and_remediations/wedge100" _ "github.com/facebook/openbmc/tools/flashy/checks_and_remediations/yamp" _ "github.com/facebook/openbmc/tools/flashy/checks_and_remediations/grandteton" + _ "github.com/facebook/openbmc/tools/flashy/checks_and_remediations/yosemite4" _ "github.com/facebook/openbmc/tools/flashy/flash_procedure" _ "github.com/facebook/openbmc/tools/flashy/utilities" ) diff --git a/tools/flashy/lib/utils/system.go b/tools/flashy/lib/utils/system.go index 3bae1f06e27a..67d22cf3f49f 100644 --- a/tools/flashy/lib/utils/system.go +++ b/tools/flashy/lib/utils/system.go @@ -407,6 +407,31 @@ var getOtherProcCmdlinePaths = func() []string { return otherCmdlines } +var ListProcessesMatchingRegex = func(regex *regexp.Regexp) ([]*os.Process, error) { + // Get all pids by listing /proc/[0-9]*/ + allProcCmdlines, _ := fileutils.Glob("/proc/[0-9]*/cmdline") + + procs := []*os.Process{} + pidRegex := regexp.MustCompile(`/proc/([0-9]+)/`) + + for _, cmdlinePath := range allProcCmdlines { + buf, err := fileutils.ReadFile(cmdlinePath) + + // Check if regex matches /proc/pid/cmdline and add to procs list + if err == nil && regex.Match(buf) { + pidRegexMatch := pidRegex.FindStringSubmatch(cmdlinePath) + + pid, _ := strconv.Atoi(pidRegexMatch[1]) + if proc, err := os.FindProcess(pid); err == nil { + procs = append(procs, proc) + } + + } + } + + return procs, nil +} + // Examine arguments for well known base names and refine the match. var refineBaseNameMatch = func(baseName string, params []string) bool { if baseName == "fw-util" { diff --git a/tools/flashy/lib/utils/system_test.go b/tools/flashy/lib/utils/system_test.go index 14b853ef24e2..9c8375bea362 100644 --- a/tools/flashy/lib/utils/system_test.go +++ b/tools/flashy/lib/utils/system_test.go @@ -27,6 +27,7 @@ import ( "reflect" "testing" "time" + "regexp" "github.com/facebook/openbmc/tools/flashy/lib/fileutils" "github.com/facebook/openbmc/tools/flashy/tests" @@ -1069,3 +1070,26 @@ func TestIsDataPartitionMounted(t *testing.T) { }) } } + +func TestListProcessesMatchingRegex(t *testing.T) { + // Tests using real /proc/ just to get some basic test coverage + t.Run("Process list matching .* must return all processes", func(t *testing.T) { + procs, err := ListProcessesMatchingRegex(regexp.MustCompile(`.*`)) + if err != nil { + t.Errorf("Error listing processes: %v", err) + } + if len(procs) < 1 { + t.Errorf("Expected at least one process matching `.*`, got %d", len(procs)) + } + }) + + t.Run("Process list matching ^\b$ must return no processes", func(t *testing.T) { + procs, err := ListProcessesMatchingRegex(regexp.MustCompile(`^\b$`)) + if err != nil { + t.Errorf("Error listing processes: %v", err) + } + if len(procs) != 0 { + t.Errorf("Expected no process matching `^\b$`, got %d", len(procs)) + } + }) +}