fix --wait's failure to work on coredns pods (#19748)

* fix --wait's failure to work on coredns pods

* fix ha test failure

pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go

@@ -76,7 +76,8 @@ func WaitForSystemPods(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg con
 }
 
 // ExpectAppsRunning returns whether or not all expected k8s-apps are running. (without waiting for them)
-func ExpectAppsRunning(cs *kubernetes.Clientset, expected []string) error {
+func ExpectAppsRunning(cfg *config.ClusterConfig, cs *kubernetes.Clientset, expected []string) error {
+
 	found := map[string]bool{}
 
 	pods, err := cs.CoreV1().Pods("kube-system").List(context.Background(), meta.ListOptions{})
@@ -85,12 +86,32 @@ func ExpectAppsRunning(cs *kubernetes.Clientset, expected []string) error {
 	}
 	klog.Infof("%d kube-system pods found", len(pods.Items))
 
+	for !config.IsHA(*cfg) && !cfg.DisableOptimizations {
+		// when --disable-optimization is not specified
+		// for non-HA cluster
+		// core dns deployment has been scaled to 1 pods, wait until there is only 1 pod
+		corednsPods, err := cs.CoreV1().Pods("kube-system").List(context.Background(), meta.ListOptions{
+			LabelSelector: "k8s-app=kube-dns",
+		})
+		if err != nil {
+			return err
+		}
+		if len(corednsPods.Items) == 1 {
+			break
+		}
+	}
+
 	for _, pod := range pods.Items {
 		klog.Info(podStatusMsg(pod))
 
 		if pod.Status.Phase != core.PodRunning {
 			continue
 		}
+		for _, cs := range pod.Status.ContainerStatuses {
+			if !cs.Ready {
+				continue
+			}
+		}
 
 		for k, v := range pod.ObjectMeta.Labels {
 			if k == "component" || k == "k8s-app" {
@@ -112,12 +133,12 @@ func ExpectAppsRunning(cs *kubernetes.Clientset, expected []string) error {
 }
 
 // WaitForAppsRunning waits for expected Apps To be running
-func WaitForAppsRunning(cs *kubernetes.Clientset, expected []string, timeout time.Duration) error {
+func WaitForAppsRunning(cfg *config.ClusterConfig, cs *kubernetes.Clientset, expected []string, timeout time.Duration) error {
 	klog.Info("waiting for k8s-apps to be running ...")
 	start := time.Now()
 
 	checkRunning := func() error {
-		return ExpectAppsRunning(cs, expected)
+		return ExpectAppsRunning(cfg, cs, expected)
 	}
 
 	if err := retry.Local(checkRunning, timeout); err != nil {

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -567,7 +567,7 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time
 		}
 
 		if cfg.VerifyComponents[kverify.AppsRunningKey] {
-			if err := kverify.WaitForAppsRunning(client, kverify.AppsRunningList, timeout); err != nil {
+			if err := kverify.WaitForAppsRunning(&cfg, client, kverify.AppsRunningList, timeout); err != nil {
 				return errors.Wrap(err, "waiting for apps_running")
 			}
 		}