From 4646c67807a41ddcf2b98737733d7495a4c8d2c0 Mon Sep 17 00:00:00 2001 From: Prachiti Talgulkar Date: Fri, 22 May 2026 00:04:46 +0530 Subject: [PATCH 1/2] Migrate 17 OCB test cases from openshift-tests-private - Add tests: 77497, 77498, 77576, 77977, 78001, 78196, 79137, 79172, 82536, 83136, 83137, 83139, 83755, 85843, 85980, 87176, 88801 - Add supporting functions: checkUpdatedLists, SkipIfCompactOrSNO, GetAllApplicableExtensionsToMCPOrFail, AssertAllNonJobPodsToBeReadyWithPollerParams - Add MCP methods: SetMaxUnavailable, RemoveMaxUnavailable, GetSortedUpdatedNodes - Add MOSC methods: SetRenderedImagePushspec, GetRenderedImagePushspec, IsUsingInternalRegistry - Preserve function ordering to match otp3 source --- test/extended-priv/machineconfigpool.go | 236 +++--- test/extended-priv/machineosconfig.go | 23 + test/extended-priv/mco_ocb.go | 10 +- test/extended-priv/mco_ocb_longduration.go | 891 +++++++++++++++++++++ test/extended-priv/util/pods.go | 25 + 5 files changed, 1067 insertions(+), 118 deletions(-) create mode 100644 test/extended-priv/mco_ocb_longduration.go diff --git a/test/extended-priv/machineconfigpool.go b/test/extended-priv/machineconfigpool.go index e4336ac3a8..5f7f2cb590 100644 --- a/test/extended-priv/machineconfigpool.go +++ b/test/extended-priv/machineconfigpool.go @@ -135,13 +135,6 @@ func (mcp *MachineConfigPool) SetMaxUnavailable(maxUnavailable int) { o.Expect(err).NotTo(o.HaveOccurred()) } -// RemoveMaxUnavailable removes spec.maxUnavailable attribute from the pool config -func (mcp *MachineConfigPool) RemoveMaxUnavailable() { - logger.Infof("patch mcp %v, removing spec.maxUnavailable", mcp.name) - err := mcp.Patch("json", `[{ "op": "remove", "path": "/spec/maxUnavailable" }]`) - o.Expect(err).NotTo(o.HaveOccurred()) -} - // SetOsImageStream sets the osImageStream name for the MCP func (mcp *MachineConfigPool) SetOsImageStream(streamName string) error { logger.Infof("patch mcp %v, change spec.osImageStream.name to %s", mcp.name, streamName) @@ -158,6 +151,13 @@ func (mcp *MachineConfigPool) GetStatusOsImageStream() (string, error) { return mcp.Get(`{.status.osImageStream.name}`) } +// RemoveMaxUnavailable removes spec.maxUnavailable attribute from the pool config +func (mcp *MachineConfigPool) RemoveMaxUnavailable() { + logger.Infof("patch mcp %v, removing spec.maxUnavailable", mcp.name) + err := mcp.Patch("json", `[{ "op": "remove", "path": "/spec/maxUnavailable" }]`) + o.Expect(err).NotTo(o.HaveOccurred()) +} + func (mcp *MachineConfigPool) getConfigNameOfSpec() (string, error) { output, err := mcp.Get(`{.spec.configuration.name}`) logger.Infof("spec.configuration.name of mcp/%v is %v", mcp.name, output) @@ -276,6 +276,7 @@ func (mcp *MachineConfigPool) estimateWaitDuration() time.Duration { totalNodes int guessedNodes = 3 // the number of nodes that we will use if we cannot get the actual number of nodes in the cluster masterAdjust = 1.0 + proxyModifier = 1.0 snoModifier = 0.0 emptyMCPWaitDuration = 2.0 minutesDuration = 1 * time.Minute @@ -324,7 +325,14 @@ func (mcp *MachineConfigPool) estimateWaitDuration() time.Duration { snoModifier = 3 } } - return time.Duration(((float64(totalNodes*mcp.MinutesWaitingPerNode) * masterAdjust) + snoModifier) * float64(minutesDuration)) + proxy := NewResource(mcp.GetOC(), "proxy", "cluster") + httpProxy, proxyErr := proxy.Get(`{.status.httpProxy}`) + if proxyErr == nil && httpProxy != "" { + logger.Infof("Increase waiting time because the cluster is proxied") + proxyModifier = 1.3 + } + + return time.Duration(((float64(totalNodes*mcp.MinutesWaitingPerNode) * masterAdjust * proxyModifier) + snoModifier) * float64(minutesDuration)) } // SetWaitingTimeForKernelChange increases the time that the MCP will wait for the update to be executed @@ -537,6 +545,80 @@ func (mcp *MachineConfigPool) GetSortedNodesOrFail() []*Node { return nodes } +// GetSortedUpdatedNodes returns the list of the UpdatedNodes sorted by the time when they started to be updated. +// If maxUnavailable>0, then the function will fail if more that maxUpdatingNodes are being updated at the same time +func (mcp *MachineConfigPool) GetSortedUpdatedNodes(maxUnavailable int) []*Node { + timeToWait := mcp.estimateWaitDuration() + logger.Infof("Waiting %s in pool %s for all nodes to start updating.", timeToWait, mcp.name) + + poolNodes, errget := mcp.GetNodes() + o.Expect(errget).NotTo(o.HaveOccurred(), fmt.Sprintf("Cannot get nodes in pool %s", mcp.GetName())) + + pendingNodes := poolNodes + updatedNodes := []*Node{} + immediate := false + err := wait.PollUntilContextTimeout(context.TODO(), 20*time.Second, timeToWait, immediate, func(_ context.Context) (bool, error) { + // If there are degraded machines, stop polling, directly fail + degradedstdout, degradederr := mcp.getDegradedMachineCount() + if degradederr != nil { + logger.Errorf("the err:%v, and try next round", degradederr) + return false, nil + } + + if degradedstdout != 0 { + logger.Errorf("Degraded MC:\n%s", mcp.PrettyString()) + exutil.AssertWaitPollNoErr(fmt.Errorf("Degraded machines"), fmt.Sprintf("mcp %s has degraded %d machines", mcp.name, degradedstdout)) + } + + // Check that there aren't more thatn maxUpdatingNodes updating at the same time + if maxUnavailable > 0 { + totalUpdating := 0 + for _, node := range poolNodes { + isUpdating, err := node.IsUpdating() + if err != nil { + logger.Errorf("Error getting IsUpdating state for node %s: %v", node.GetName(), err) + return false, err + } + if isUpdating { + totalUpdating++ + } + } + if totalUpdating > maxUnavailable { + // print nodes for debug + mcp.oc.Run("get").Args("nodes").Execute() + exutil.AssertWaitPollNoErr(fmt.Errorf("maxUnavailable Not Honored. Pool %s, error: %d nodes were updating at the same time. Only %d nodes should be updating at the same time", mcp.GetName(), totalUpdating, maxUnavailable), "") + } + } + + remainingNodes := []*Node{} + for _, node := range pendingNodes { + isUpdating, err := node.IsUpdating() + if err != nil { + logger.Errorf("Error getting IsUpdating state for node %s: %v", node.GetName(), err) + return false, err + } + if isUpdating { + logger.Infof("Node %s is UPDATING", node.GetName()) + updatedNodes = append(updatedNodes, node) + } else { + remainingNodes = append(remainingNodes, node) + } + } + + if len(remainingNodes) == 0 { + logger.Infof("All nodes have started to be updated on mcp %s", mcp.name) + return true, nil + + } + logger.Infof(" %d remaining nodes", len(remainingNodes)) + pendingNodes = remainingNodes + return false, nil + }) + + exutil.AssertWaitPollNoErr(err, fmt.Sprintf("Could not get the list of updated nodes on mcp %s", mcp.name)) + return updatedNodes +} + // GetCordonedNodes get cordoned nodes (if maxUnavailable > 1 ) otherwise return the 1st cordoned node func (mcp *MachineConfigPool) GetCordonedNodes() []*Node { @@ -1288,6 +1370,38 @@ func CreateCustomMCPWithStreamByLabel(oc *exutil.CLI, name, label, osstream stri return CreateCustomMCPWithStreamByNodes(oc, name, osstream, customMcpNodes) } +// AddNodesToMachineConfigPool adds the given nodes to the specified MCP by labeling them +// and waits for the MCP to report the expected machine count and updated status +func AddNodesToMachineConfigPool(oc *exutil.CLI, mcpName string, nodes []*Node) error { + mcp := NewMachineConfigPool(oc, mcpName) + + currentNodes, err := mcp.GetNodes() + if err != nil { + return fmt.Errorf("error getting current nodes from %s: %w", mcpName, err) + } + expectedCount := len(currentNodes) + len(nodes) + + for _, n := range nodes { + err := n.AddLabel(fmt.Sprintf("node-role.kubernetes.io/%s", mcpName), "") + if err != nil { + return fmt.Errorf("error labeling node %s to add it to pool %s: %w", n.GetName(), mcpName, err) + } + logger.Infof("Node %s added to pool %s", n.GetName(), mcpName) + } + + err = mcp.WaitForMachineCount(expectedCount, 5*time.Minute) + if err != nil { + return fmt.Errorf("the %s MCP is not reporting the expected machine count: %w", mcpName, err) + } + + err = mcp.WaitImmediateForUpdatedStatus() + if err != nil { + return fmt.Errorf("the %s MCP is not updated: %w", mcpName, err) + } + + return nil +} + // CreateCustomMCP creates a new custom MCP with the given name and number of nodes // No osstream field is set - MCP will inherit from worker pool automatically // Nodes will be taken from the worker pool @@ -1644,112 +1758,6 @@ func GetPoolWithArchDifferentFromOrFail(oc *exutil.CLI, arch architecture.Archit return nil } -// AddNodesToMachineConfigPool adds nodes to an existing MCP by labeling them and waiting for the MCP to be updated -// This function is similar to CreateCustomMCPWithStreamByNodes but for adding nodes to existing pools -func AddNodesToMachineConfigPool(oc *exutil.CLI, mcpName string, nodes []*Node) error { - mcp := NewMachineConfigPool(oc, mcpName) - - currentNodes, err := mcp.GetNodes() - if err != nil { - return fmt.Errorf("error getting current nodes from %s: %w", mcpName, err) - } - expectedCount := len(currentNodes) + len(nodes) - - for _, n := range nodes { - err := n.AddLabel(fmt.Sprintf("node-role.kubernetes.io/%s", mcpName), "") - if err != nil { - return fmt.Errorf("error labeling node %s to add it to pool %s: %w", n.GetName(), mcpName, err) - } - logger.Infof("Node %s added to pool %s", n.GetName(), mcpName) - } - - err = mcp.WaitForMachineCount(expectedCount, 5*time.Minute) - if err != nil { - return fmt.Errorf("the %s MCP is not reporting the expected machine count: %w", mcpName, err) - } - - err = mcp.WaitImmediateForUpdatedStatus() - if err != nil { - return fmt.Errorf("the %s MCP is not updated: %w", mcpName, err) - } - - return nil -} - -// GetSortedUpdatedNodes returns a list of nodes in the order that they are being updated by the MCO -// If maxUnavailable>0, then the function will fail if more that maxUpdatingNodes are being updated at the same time -func (mcp *MachineConfigPool) GetSortedUpdatedNodes(maxUnavailable int) []*Node { - timeToWait := mcp.estimateWaitDuration() - logger.Infof("Waiting %s in pool %s for all nodes to start updating.", timeToWait, mcp.name) - - poolNodes, errget := mcp.GetNodes() - o.Expect(errget).NotTo(o.HaveOccurred(), fmt.Sprintf("Cannot get nodes in pool %s", mcp.GetName())) - - pendingNodes := poolNodes - updatedNodes := []*Node{} - immediate := false - err := wait.PollUntilContextTimeout(context.TODO(), 20*time.Second, timeToWait, immediate, func(_ context.Context) (bool, error) { - // If there are degraded machines, stop polling, directly fail - degradedstdout, degradederr := mcp.getDegradedMachineCount() - if degradederr != nil { - logger.Errorf("the err:%v, and try next round", degradederr) - return false, nil - } - - if degradedstdout != 0 { - logger.Errorf("Degraded MC:\n%s", mcp.PrettyString()) - exutil.AssertWaitPollNoErr(fmt.Errorf("degraded machines"), fmt.Sprintf("mcp %s has degraded %d machines", mcp.name, degradedstdout)) - } - - // Check that there aren't more thatn maxUpdatingNodes updating at the same time - if maxUnavailable > 0 { - totalUpdating := 0 - for _, node := range poolNodes { - isUpdating, err := node.IsUpdating() - if err != nil { - logger.Errorf("Error getting IsUpdating state for node %s: %v", node.GetName(), err) - return false, err - } - if isUpdating { - totalUpdating++ - } - } - if totalUpdating > maxUnavailable { - // print nodes for debug - mcp.oc.Run("get").Args("nodes").Execute() - exutil.AssertWaitPollNoErr(fmt.Errorf("maxUnavailable Not Honored. Pool %s, error: %d nodes were updating at the same time. Only %d nodes should be updating at the same time", mcp.GetName(), totalUpdating, maxUnavailable), "") - } - } - - remainingNodes := []*Node{} - for _, node := range pendingNodes { - isUpdating, err := node.IsUpdating() - if err != nil { - logger.Errorf("Error getting IsUpdating state for node %s: %v", node.GetName(), err) - return false, err - } - if isUpdating { - logger.Infof("Node %s is UPDATING", node.GetName()) - updatedNodes = append(updatedNodes, node) - } else { - remainingNodes = append(remainingNodes, node) - } - } - - if len(remainingNodes) == 0 { - logger.Infof("All nodes have started to be updated on mcp %s", mcp.name) - return true, nil - - } - logger.Infof(" %d remaining nodes", len(remainingNodes)) - pendingNodes = remainingNodes - return false, nil - }) - - exutil.AssertWaitPollNoErr(err, fmt.Sprintf("Could not get the list of updated nodes on mcp %s", mcp.name)) - return updatedNodes -} - // IsOCL returns true if the pool is using On Cluster Layering functionality func (mcp MachineConfigPool) IsOCL() (bool, error) { isOCLEnabled, err := IsFeaturegateEnabled(mcp.GetOC(), "OnClusterBuild") diff --git a/test/extended-priv/machineosconfig.go b/test/extended-priv/machineosconfig.go index 6f35a00213..2c611f6ff6 100644 --- a/test/extended-priv/machineosconfig.go +++ b/test/extended-priv/machineosconfig.go @@ -329,6 +329,20 @@ func (mosc MachineOSConfig) GetCurrentMachineOSBuild() (*MachineOSBuild, error) return NewMachineOSBuild(mosc.GetOC(), mosbName), nil } +// SetRenderedImagePushspec patches the MOSC resource in order to configure a new renderedImagePushspec +func (mosc MachineOSConfig) SetRenderedImagePushspec(rips string) error { + escaped, err := json.Marshal(rips) + if err != nil { + return err + } + return mosc.Patch("json", `[{"op":"replace","path":"/spec/renderedImagePushSpec","value":`+string(escaped)+`}]`) +} + +// GetRenderedImagePushspec returns the current valude of renderedImagePushspec +func (mosc MachineOSConfig) GetRenderedImagePushspec() (string, error) { + return mosc.Get(`{.spec.renderedImagePushSpec}`) +} + // SetContainerfiles sets the container files used by this MOSC func (mosc MachineOSConfig) SetContainerfiles(containerFiles []ContainerFile) error { containerFilesBytes, err := json.Marshal(containerFiles) @@ -472,3 +486,12 @@ func SkipTestIfCannotUseInternalRegistry(oc *exutil.CLI) { g.Skip("The internal registry cannot be used to store the osImage in this cluster. Skipping test case") } } + +// IsUsingInternalRegistry checks if the current image is from the internal registry +func (mosc MachineOSConfig) IsUsingInternalRegistry() (bool, error) { + currentImage, err := mosc.GetStatusCurrentImagePullSpec() + if err != nil { + return false, err + } + return strings.Contains(currentImage, InternalRegistrySvcURL), nil +} diff --git a/test/extended-priv/mco_ocb.go b/test/extended-priv/mco_ocb.go index 3c5fb25924..79e3c09ef0 100644 --- a/test/extended-priv/mco_ocb.go +++ b/test/extended-priv/mco_ocb.go @@ -426,7 +426,8 @@ func ValidateSuccessfulMOSC(mosc *MachineOSConfig, checkers []Checker) { logger.Infof("OK!\n") exutil.By("Check that a new build is successfully executed") - o.Eventually(mosb, "20m", "20s").Should(HaveConditionField("Building", "status", FalseString), "Build was not finished") + // When building several builds at the same time the pods can take more than 20 minutes to build a simple image + o.Eventually(mosb, "35m", "20s").Should(HaveConditionField("Building", "status", FalseString), "Build was not finished") o.Eventually(mosb, "10m", "20s").Should(HaveConditionField("Succeeded", "status", TrueString), "Build didn't succeed") o.Eventually(mosb, "2m", "20s").Should(HaveConditionField("Interrupted", "status", FalseString), "Build was interrupted") o.Eventually(mosb, "2m", "20s").Should(HaveConditionField("Failed", "status", FalseString), "Build was failed") @@ -699,8 +700,9 @@ func checkUsbguradExtension(node *Node) { rpmName = AllExtenstions[extName] activeString = "Active: active (running)" inactiveString = "Active: inactive (dead)" - expectedError = "missing /usr/lib/tmpfiles.d/usbguard.conf\nerror: non-zero exit code from debug container" - expectedErrorOCL = ".M....... /var/log/usbguard\nerror: non-zero exit code from debug container" + expectedError = "missing /usr/lib/tmpfiles.d/usbguard.conf\nerror: non-zero exit code from debug container" + expectedErrorOCL = ".M....... /var/log/usbguard\nerror: non-zero exit code from debug container" + expectedErrorCombined = "missing /usr/lib/tmpfiles.d/usbguard.conf\n.M....... /var/log/usbguard\nerror: non-zero exit code from debug container" ) exutil.By("Verify node includes Usbguard extension") o.Expect( @@ -729,7 +731,7 @@ func checkUsbguradExtension(node *Node) { exutil.By("Check that all the files in the rpm were correctly deployed ") rpmOut, _ := node.checkRpmFiles(rpmName...) - o.Expect(strings.TrimSpace(rpmOut)).Should(o.Or(o.BeEmpty(), o.Equal(expectedErrorOCL), o.Equal(expectedError)), "Error in permissions of rpm files") + o.Expect(strings.TrimSpace(rpmOut)).Should(o.Or(o.BeEmpty(), o.Equal(expectedErrorOCL), o.Equal(expectedError), o.Equal(expectedErrorCombined)), "Error in permissions of rpm files") logger.Infof("OK!\n") } diff --git a/test/extended-priv/mco_ocb_longduration.go b/test/extended-priv/mco_ocb_longduration.go new file mode 100644 index 0000000000..6a45f59314 --- /dev/null +++ b/test/extended-priv/mco_ocb_longduration.go @@ -0,0 +1,891 @@ +package extended + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + exutil "github.com/openshift/machine-config-operator/test/extended-priv/util" + logger "github.com/openshift/machine-config-operator/test/extended-priv/util/logext" +) + +var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longduration][Serial][Disruptive] MCO ocb longduration", func() { + defer g.GinkgoRecover() + + var ( + oc = exutil.NewCLI("mco-ocb-longduration", exutil.KubeConfigPath()) + ) + + g.JustBeforeEach(func() { + PreChecks(oc) + SkipTestIfOCBIsEnabled(oc) + }) + + g.It("[PolarionID:79172][OTP] OCB Inherit from global pull secret if baseImagePullSecret field is not specified [Disruptive]", func() { + var ( + infraMcpName = "infra" + ) + + exutil.By("Create custom infra MCP") + // We add no workers to the infra pool, it is not necessary + infraMcp, err := CreateCustomMCP(oc.AsAdmin(), infraMcpName, 0) + defer infraMcp.delete() + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating a new custom pool: %s", infraMcpName) + logger.Infof("OK!\n") + + testContainerFile([]ContainerFile{}, MachineConfigNamespace, infraMcp, nil, true) + }) + + g.It("[PolarionID:83136][OTP][Skipped:Disconnected] Panic Condition for Non-Matching MOSC Resources [Disruptive]", func() { + var ( + infraMcpName = "infra" + // MOSC has to use the same name as the mcp + moscName = infraMcpName + ) + exutil.By("Create New Custom MCP") + defer DeleteCustomMCP(oc.AsAdmin(), infraMcpName) + infraMcp, err := CreateCustomMCP(oc.AsAdmin(), infraMcpName, 1) + o.Expect(err).NotTo(o.HaveOccurred(), "Could not create a new custom MCP") + node := infraMcp.GetNodesOrFail()[0] + logger.Infof("%s", node.GetName()) + logger.Infof("OK!\n") + + exutil.By("Configure OCB functionality for the new infra MCP") + mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, infraMcpName, nil) + defer DisableOCL(mosc) + // remove after this bug is fixed OCPBUGS-36810 + defer func() { + logger.Infof("Configmaps should also be deleted ") + cmList := NewConfigMapList(mosc.GetOC(), MachineConfigNamespace).GetAllOrFail() + for _, cm := range cmList { + if strings.Contains(cm.GetName(), "rendered-") { + o.Expect(cm.Delete()).Should(o.Succeed(), "The ConfigMap related to MOSC has not been removed") + } + } + }() + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") + logger.Infof("OK!\n") + + exutil.By("Check that a new build has been triggered") + o.Eventually(mosc.GetCurrentMachineOSBuild, "5m", "20s").Should(Exist(), + "No build was created when OCB was enabled") + mosb, err := mosc.GetCurrentMachineOSBuild() + o.Expect(err).NotTo(o.HaveOccurred(), "Error getting MOSB from MOSC") + o.Eventually(mosb.GetJob, "5m", "20s").Should(Exist(), + "No build pod was created when OCB was enabled") + o.Eventually(mosb, "5m", "20s").Should(HaveConditionField("Building", "status", TrueString), + "MachineOSBuild didn't report that the build has begun") + logger.Infof("OK!\n") + + exutil.By("Delete the MCOS and check it is deleted") + o.Expect(mosc.CleanupAndDelete()).To(o.Succeed(), "Error cleaning up %s", mosc) + ValidateMOSCIsGarbageCollected(mosc, infraMcp) + o.Expect(mosb).NotTo(Exist(), "Build is not deleted") + o.Expect(mosc).NotTo(Exist(), "MOSC is not deleted") + logger.Infof("OK!\n") + + exutil.AssertAllPodsToBeReady(oc.AsAdmin(), MachineConfigNamespace) + checkMCCPanic(oc) + }) + + g.It("[PolarionID:78001][OTP][Skipped:Disconnected] The etc-pki-etitlement secret is created automatically for OCB Use custom Containerfile with rhel enablement [Disruptive]", func() { + var ( + entitlementSecret = NewSecret(oc.AsAdmin(), "openshift-config-managed", "etc-pki-entitlement") + containerFileContent = ` + FROM configs AS final + + RUN rm -rf /etc/rhsm-host && \ + rpm-ostree install buildah && \ + ln -s /run/secrets/rhsm /etc/rhsm-host && \ + ostree container commit +` + + checkers = []Checker{ + CommandOutputChecker{ + Command: []string{"rpm", "-q", "buildah"}, + Matcher: o.ContainSubstring("buildah-"), + ErrorMsg: fmt.Sprintf("Buildah package is not installed after the image was deployed"), + Desc: fmt.Sprintf("Check that buildah is installed"), + }, + } + + mcp = GetCompactCompatiblePool(oc.AsAdmin()) + ) + + if !entitlementSecret.Exists() { + g.Skip(fmt.Sprintf("There is no entitlement secret available in this cluster %s. This test case cannot be executed", entitlementSecret)) + } + + testContainerFile([]ContainerFile{{Content: containerFileContent}}, MachineConfigNamespace, mcp, checkers, false) + }) + + g.It("[PolarionID:77498][OTP] OCB Trigger new build when renderedImagePushspec is updated [Disruptive]", func() { + + var ( + infraMcpName = "infra" + // MOSC resources have to use the same names as the MCP + moscName = infraMcpName + ) + + exutil.By("Create custom infra MCP") + // We add no workers to the infra pool, it is not necessary + infraMcp, err := CreateCustomMCP(oc.AsAdmin(), infraMcpName, 0) + defer infraMcp.delete() + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating a new custom pool: %s", infraMcpName) + logger.Infof("OK!\n") + + exutil.By("Configure OCB functionality for the new infra MCP") + mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, infraMcpName, nil) + defer mosc.CleanupAndDelete() + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") + logger.Infof("OK!\n") + + ValidateSuccessfulMOSC(mosc, nil) + + exutil.By("Set a new rendered image pull spec") + initialMOSB := OrFail[*MachineOSBuild](mosc.GetCurrentMachineOSBuild()) + initialRIPS := OrFail[string](mosc.GetRenderedImagePushspec()) + o.Expect( + mosc.SetRenderedImagePushspec(strings.ReplaceAll(initialRIPS, "ocb-", "ocb77498-")), + ).NotTo(o.HaveOccurred(), "Error patching %s to set the new renderedImagePullSpec", mosc) + logger.Infof("OK!\n") + + exutil.By("Check that a new build is triggered") + checkNewBuildIsTriggered(mosc, initialMOSB) + logger.Infof("OK!\n") + + exutil.By("Set the original rendered image pull spec") + o.Expect( + mosc.SetRenderedImagePushspec(initialRIPS), + ).NotTo(o.HaveOccurred(), "Error patching %s to set the new renderedImagePullSpec", mosc) + logger.Infof("OK!\n") + + exutil.By("Check that the initial build is reused") + var currentMOSB *MachineOSBuild + o.Eventually(func() (string, error) { + currentMOSB, err = mosc.GetCurrentMachineOSBuild() + if err != nil || currentMOSB == nil { + return "", err + } + return currentMOSB.GetName(), nil + }, "5m", "20s").Should(o.Equal(initialMOSB.GetName()), + "When the containerfiles were removed and initial MOSC configuration was restored, the initial MOSB was not used") + + logger.Infof("OK!\n") + }) + + g.It("[PolarionID:77497][OTP] OCB Trigger new build when Containerfile is updated [Disruptive]", func() { + + var ( + infraMcpName = "infra" + // MOSC resources have to use the same names as the MCP + moscName = infraMcpName + containerFile = ContainerFile{Content: "RUN touch /etc/test-add-containerfile" + "\n" + ExpirationDockerfileLabel} + containerFileMod = ContainerFile{Content: "RUN touch /etc/test-modified-containerfile" + "\n" + ExpirationDockerfileLabel} + + // We need to test a first MOSC without any container file, so we cannot add the expiration label in the first MOSC + expireImage = false + // We don't configure the pull secret in the MOSC, since it is optional + defaultPullSecret = true + ) + + exutil.By("Create custom infra MCP") + // We add no workers to the infra pool, it is not necessary + infraMcp, err := CreateCustomMCP(oc.AsAdmin(), infraMcpName, 0) + defer infraMcp.delete() + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating a new custom pool: %s", infraMcpName) + logger.Infof("OK!\n") + + exutil.By("Configure OCB functionality for the new infra MCP") + mosc, err := createMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, infraMcpName, nil, defaultPullSecret, expireImage) + + defer mosc.CleanupAndDelete() + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") + logger.Infof("OK!\n") + + ValidateSuccessfulMOSC(mosc, nil) + + exutil.By("Add new container file") + initialMOSB := OrFail[*MachineOSBuild](mosc.GetCurrentMachineOSBuild()) + + o.Expect( + mosc.SetContainerfiles([]ContainerFile{containerFile}), + ).NotTo(o.HaveOccurred(), "Error patching %s to add a container file", mosc) + logger.Infof("OK!\n") + + exutil.By("Check that a new build is triggered when a containerfile is added") + checkNewBuildIsTriggered(mosc, initialMOSB) + logger.Infof("OK!\n") + + exutil.By("Modify the container file") + currentMOSB := OrFail[*MachineOSBuild](mosc.GetCurrentMachineOSBuild()) + + o.Expect( + mosc.SetContainerfiles([]ContainerFile{containerFileMod}), + ).NotTo(o.HaveOccurred(), "Error patching %s to modify an existing container file", mosc) + logger.Infof("OK!\n") + + exutil.By("Check that a new build is triggered when a containerfile is modified") + checkNewBuildIsTriggered(mosc, currentMOSB) + logger.Infof("OK!\n") + + exutil.By("Remove the container files") + o.Expect( + mosc.RemoveContainerfiles(), + ).NotTo(o.HaveOccurred(), "Error patching %s to remove the configured container files", mosc) + logger.Infof("OK!\n") + + exutil.By("Check that the initial build is reused") + o.Eventually(func() (string, error) { + currentMOSB, err = mosc.GetCurrentMachineOSBuild() + if err != nil || currentMOSB == nil { + return "", err + } + return currentMOSB.GetName(), nil + }, "5m", "20s").Should(o.Equal(initialMOSB.GetName()), + "When the containerfiles were removed and initial MOSC configuration was restored, the initial MOSB was not used") + + logger.Infof("OK!\n") + }) + + g.It("[PolarionID:77576][OTP] In OCB. Create a new MC while a build is running [Disruptive]", func() { + + var ( + mcp = GetCompactCompatiblePool(oc.AsAdmin()) + node = mcp.GetSortedNodesOrFail()[0] + moscName = mcp.GetName() + + kArgs = "test" + mcName = "tc-77576-testkargs" + mc = NewMachineConfig(oc.AsAdmin(), mcName, mcp.GetName()) + ) + + exutil.By("Configure OCB functionality for the new worker MCP") + mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, mcp.GetName(), nil) + defer DisableOCL(mosc) + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") + logger.Infof("OK!\n") + + exutil.By("Check that a new build has been triggered and is building") + var mosb *MachineOSBuild + o.Eventually(func() (*MachineOSBuild, error) { + var err error + mosb, err = mosc.GetCurrentMachineOSBuild() + return mosb, err + }, "5m", "20s").Should(Exist(), + "No build was created when OCB was enabled") + o.Eventually(mosb.GetJob, "5m", "20s").Should(Exist(), + "No build job was created when OCB was enabled") + o.Eventually(mosb, "5m", "20s").Should(HaveConditionField("Building", "status", TrueString), + "MachineOSBuild didn't report that the build has begun") + logger.Infof("OK!\n") + + exutil.By("Create a MC to trigger a new build") + defer mc.DeleteWithWait() + err = mc.Create("-p", "NAME="+mcName, "-p", "POOL="+mcp.GetName(), "-p", fmt.Sprintf(`KERNEL_ARGS=["%s"]`, kArgs)) + o.Expect(err).NotTo(o.HaveOccurred()) + logger.Infof("OK!\n") + + exutil.By("Check that a new build is triggered and the old build is removed") + checkNewBuildIsTriggered(mosc, mosb) + o.Eventually(mosb, "2m", "20s").ShouldNot(Exist(), "The old MOSB %s was not deleted", mosb) + logger.Infof("OK!\n") + + exutil.By("Wait for the configuration to be applied") + mcp.waitForComplete() + logger.Infof("OK!\n") + + exutil.By("Check that the MC was applied") + o.Expect(node.IsKernelArgEnabled(kArgs)).To(o.BeTrue(), + "Kernel argument %s was not enabled in the node %s", kArgs, node.GetName()) + + logger.Infof("OK!\n") + + exutil.By("Remove the MachineOSConfig resource") + o.Expect(DisableOCL(mosc)).To(o.Succeed(), "Error cleaning up %s", mosc) + logger.Infof("OK!\n") + }) + + g.It("[PolarionID:77977][OTP][Skipped:Disconnected] Install extension after OCB is enabled [Disruptive]", func() { + + var ( + mcp = GetCompactCompatiblePool(oc.AsAdmin()) + moscName = mcp.GetName() // MOSC resources have to use the same name as the MCP + node = mcp.GetSortedNodesOrFail()[0] + mcName = "test-install-extension-" + GetCurrentTestPolarionIDNumber() + applicableExtensions, _ = GetAllApplicableExtensionsToMCPOrFail(mcp) + ) + + exutil.By("Configure OCB functionality for the new worker MCP") + mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, mcp.GetName(), nil) + defer DisableOCL(mosc) + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") + logger.Infof("OK!\n") + + ValidateSuccessfulMOSC(mosc, nil) + + exutil.By("Create a MC") + mc := NewMachineConfig(oc.AsAdmin(), mcName, mcp.GetName()) + defer mc.DeleteWithWait() + mc.parameters = []string{fmt.Sprintf(`EXTENSIONS=%s`, string(MarshalOrFail(applicableExtensions)))} + mc.create() + logger.Infof("OK!\n") + + exutil.By("Wait for the configuration to be applied") + mcp.waitForComplete() + logger.Infof("OK!\n") + + CheckExtensions(node, applicableExtensions) + + exutil.By("Delete a MC.") + mc.DeleteWithWait() + logger.Infof("OK!\n") + + exutil.By("Remove the MachineOSConfig resource") + o.Expect(DisableOCL(mosc)).To(o.Succeed(), "Error cleaning up %s", mosc) + ValidateMOSCIsGarbageCollected(mosc, mcp) + logger.Infof("OK!\n") + }) + + g.It("[PolarionID:78196][OTP][Skipped:Disconnected] Verify for etc-pki-etitlement secret is removed for OCB rhel enablement [Disruptive]", func() { + + var ( + entitlementSecret = NewSecret(oc.AsAdmin(), "openshift-config-managed", "etc-pki-entitlement") + containerFileContent = ` + FROM configs AS final + RUN rm -rf /etc/rhsm-host && \ + rpm-ostree install buildah && \ + ln -s /run/secrets/rhsm /etc/rhsm-host && \ + ostree container commit + ` + + mcp = GetCompactCompatiblePool(oc.AsAdmin()) + // MOSC resources have to use the same name as the MCP + moscName = mcp.GetName() + ) + if !entitlementSecret.Exists() { + g.Skip(fmt.Sprintf("There is no entitlement secret available in this cluster %s. This test case cannot be executed", entitlementSecret)) + } + + exutil.By("Copy the entitlement secret in MCO namespace") + mcoEntitlementSecret, err := CloneResource(entitlementSecret, "etc-pki-entitlement", MachineConfigNamespace, nil) + defer mcoEntitlementSecret.Delete() + o.Expect(err).NotTo(o.HaveOccurred(), "Error copying %s to the %s namespace", mcoEntitlementSecret, MachineConfigNamespace) + logger.Infof("OK!\n") + + exutil.By("Delete the entitlement secret in the openshift-config-managed namespace") + defer func() { + exutil.By("Recover the entitlement secret in the openshift-config-managed namespace") + recoverSecret, err := CloneResource(mcoEntitlementSecret, "etc-pki-entitlement", "openshift-config-managed", nil) + o.Expect(err).NotTo(o.HaveOccurred(), "Error copying %s to the openshift-config-managed namespace", entitlementSecret) + o.Expect(recoverSecret).To(Exist(), "Unable to recover the entitlement secret in openshift-config-managed namespace") + }() + + entitlementSecret.Delete() + logger.Infof("OK!\n") + + exutil.By("Create the MOSC") + mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, mcp.GetName(), []ContainerFile{{Content: containerFileContent}}) + defer DisableOCL(mosc) + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") + logger.Infof("OK!\n") + + exutil.By("Check that a new build has been triggered") + o.Eventually(mosc.GetCurrentMachineOSBuild, "5m", "20s").Should(Exist(), + "No build was created when OCB was enabled") + mosb, err := mosc.GetCurrentMachineOSBuild() + o.Expect(err).NotTo(o.HaveOccurred(), "Error getting MOSB from MOSC") + o.Eventually(mosb, "5m", "20s").Should(HaveConditionField("Building", "status", TrueString), + "MachineOSBuild didn't report that the build has begun") + logger.Infof("OK!\n") + + exutil.By("Verify the error is produced in buildPod") + exutil.AssertAllNonJobPodsToBeReadyWithPollerParams(mosc.GetOC(), MachineConfigNamespace, 10*time.Second, 10*time.Minute) + logger.Infof("OK!\n") + + job, err := mosb.GetJob() + o.Expect(err).NotTo(o.HaveOccurred()) + // Currently this kind of resources are leaked. Until the leak is fixed we need to make sure that this job is removed + // because its pods are in "Error" status and there are other test cases checking that no pod is reporting any error. + // TODO: remove this once the leak is fixed + defer job.Delete() + logger.Infof("OK!\n") + + o.Eventually(func() (string, error) { + return job.Logs("-c", "image-build") + }, "5m", "10s").Should(o.ContainSubstring("Found 0 entitlement certificates"), "Error getting the logs") + o.Eventually(job, "15m", "20s").Should(HaveConditionField("Failed", "status", TrueString), "Job didn't fail") + logger.Infof("OK!\n") + + exutil.By("Remove the MachineOSConfig resource") + o.Expect(DisableOCL(mosc)).To(o.Succeed(), "Error cleaning up %s", mosc) + ValidateMOSCIsGarbageCollected(mosc, mcp) + logger.Infof("OK!\n") + }) + + g.It("[PolarionID:88801][OTP][Skipped:Disconnected] ExternalRegistry OCB Verify new nodes boot directly with OCL image without unnecessary reboots [Disruptive]", func() { + SkipIfCompactOrSNO(oc.AsAdmin()) // This test requires scaling, which doesn't make sense in SNO or Compact + skipTestIfWorkersCannotBeScaled(oc.AsAdmin()) // Skip test if worker node cannot be scaled + + var ( + mcp = GetCompactCompatiblePool(oc.AsAdmin()) + moscName = mcp.GetName() + ) + + exutil.By("Configure OCB functionality using external registry (Quay)") + mosc, err := CreateMachineOSConfigUsingExternalRegistry(oc.AsAdmin(), moscName, mcp.GetName(), nil, false, false) + defer DisableOCL(mosc) + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") + logger.Infof("OK!\n") + + ValidateNewNodesBootDirectlyWithOCLImage(oc.AsAdmin(), mosc, mcp) + }) + + g.It("[PolarionID:85980][OTP][Skipped:Disconnected] Check when MOSB is degraded MCP should be degraded too with right fields updated. [Disruptive]", func() { + var ( + infraMcpName = "infra" + moscName = infraMcpName + // Intentionally uses 'apt' on Alpine (which uses 'apk'), causing build to fail + incorrectContainerFile = "FROM alpine:3.18\nRUN apt update && apt install -y cowsay\n" + correctContainerFile = "FROM configs AS final\nRUN echo \"hello\" > /etc/test.txt\n" + expectedDegradedMessage = "Failed to build OS image" + ) + + exutil.By("Create custom infra MCP") + infraMcp, err := CreateCustomMCP(oc.AsAdmin(), infraMcpName, 1) + defer DeleteCustomMCP(oc.AsAdmin(), infraMcpName) + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating custom MCP: %s", infraMcpName) + node := infraMcp.GetNodesOrFail()[0] + logger.Infof("Infra node: %s\n", node.GetName()) + logger.Infof("OK!\n") + + exutil.By("Create MOSC with incorrect containerfile") + mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, infraMcpName, + []ContainerFile{{Content: incorrectContainerFile}}) + defer mosc.CleanupAndDelete() + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating MOSC with incorrect containerfile") + logger.Infof("OK!\n") + + exutil.By("Verify MOSB is created and fails") + o.Eventually(mosc.GetCurrentMachineOSBuild, "5m", "20s").Should(Exist(), + "No MOSB was created for the incorrect containerfile") + failedMOSB, err := mosc.GetCurrentMachineOSBuild() + o.Expect(err).NotTo(o.HaveOccurred(), "Error getting MOSB from MOSC") + logger.Infof("MOSB created: %s\n", failedMOSB.GetName()) + o.Eventually(failedMOSB, "20m", "20s").Should(HaveConditionField("Failed", "status", TrueString), + "MOSB should fail due to incorrect containerfile") + logger.Infof("MOSB failed as expected\n") + + exutil.By("Verify MCP is degraded with ImageBuildDegraded condition") + o.Eventually(infraMcp, "5m", "20s").Should(BeDegraded(), "MCP should be degraded when build fails") + o.Eventually(infraMcp, "5m", "20s").Should(HaveConditionField("ImageBuildDegraded", "status", TrueString), + "MCP ImageBuildDegraded should be True when build fails") + o.Eventually(infraMcp, "5m", "20s").Should(HaveConditionField("ImageBuildDegraded", "message", o.ContainSubstring(expectedDegradedMessage)), + "MCP degradation message should indicate OS image build failure") + o.Eventually(infraMcp, "5m", "20s").Should(HaveConditionField("Degraded", "message", o.ContainSubstring("Custom OS image build failed")), + "MCP Degraded message should indicate custom OS image build failed") + logger.Infof("OK!\n") + + exutil.By("Fix the MOSC by updating containerfile") + o.Expect(mosc.SetContainerfiles([]ContainerFile{{Content: correctContainerFile}})).NotTo(o.HaveOccurred(), + "Error updating MOSC with correct containerfile") + logger.Infof("OK!\n") + + exutil.By("Verify new MOSB is created and succeeds") + checkNewBuildIsTriggered(mosc, failedMOSB) + newMOSB, err := mosc.GetCurrentMachineOSBuild() + o.Expect(err).NotTo(o.HaveOccurred(), "Error getting new MOSB") + logger.Infof("New MOSB created: %s\n", newMOSB.GetName()) + + exutil.By("Verify old failed MOSB is deleted") + o.Eventually(failedMOSB, "5m", "20s").ShouldNot(Exist(), + "Old failed MOSB should be garbage collected: %s", failedMOSB.GetName()) + logger.Infof("OK!\n") + + exutil.By("Verify MCP recovers and is no longer degraded") + // Check both ImageBuildDegraded (specific condition) and Degraded (overall status) + // to ensure complete recovery from the failed build state + o.Eventually(infraMcp, "5m", "20s").Should(HaveConditionField("ImageBuildDegraded", "status", FalseString), + "MCP ImageBuildDegraded condition should be False after recovery") + o.Eventually(infraMcp, "5m", "20s").ShouldNot(BeDegraded(), + "MCP Degraded condition should be False after recovery") + o.Eventually(infraMcp, "5m", "20s").Should(HaveConditionField("Updating", "status", TrueString), + "MCP should be updating after successful build") + logger.Infof("OK!\n") + + exutil.By("Wait for MCP to complete update") + infraMcp.waitForComplete() + logger.Infof("OK!\n") + + exutil.By("Verify image is applied to node") + currentImagePullSpec := OrFail[string](mosc.GetStatusCurrentImagePullSpec()) + o.Expect(node.GetCurrentBootOSImage()).To(o.Equal(currentImagePullSpec), + "Node should be using the correct OCL image after recovery") + logger.Infof("Node is using image: %s\n", currentImagePullSpec) + logger.Infof("OK!\n") + + exutil.By("Verify file created in containerfile exists on node") + output, err := node.DebugNodeWithChroot("cat", "/etc/test.txt") + o.Expect(err).NotTo(o.HaveOccurred(), "Error reading test file from node") + o.Expect(output).To(o.ContainSubstring("hello"), + "Test file content should match what was created in containerfile") + logger.Infof("OK!\n") + }) + + g.It("[PolarionID:87176][OTP][Skipped:Disconnected] External Registry In OCB to check when a image is removed the old build is triggered again and the MC should start updating directly. [Disruptive]", func() { + SkipIfCompactOrSNO(oc.AsAdmin()) + + var ( + infraMcpName = "infra" + mcName = fmt.Sprintf("tc-%s-ext-kernelarg", GetCurrentTestPolarionIDNumber()) + kArgs = "test" + containerFileNoExpiration = "FROM configs AS final\nLABEL maintainer=\"mco@example.com\"\n" + ) + + exutil.By("Create custom infra MCP") + infraMcp, err := CreateCustomMCP(oc.AsAdmin(), infraMcpName, 1) + defer DeleteCustomMCP(oc.AsAdmin(), infraMcpName) + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating a new custom pool: %s", infraMcpName) + logger.Infof("OK!\n") + + exutil.By("Create MOSC for custom MCP using external registry") + moscName := infraMcp.GetName() + mosc, err := CreateMachineOSConfigUsingExternalRegistry(oc.AsAdmin(), moscName, infraMcp.GetName(), + []ContainerFile{{Content: containerFileNoExpiration}}, false, false) + defer DisableOCL(mosc) + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating MOSC") + logger.Infof("OK!\n") + + exutil.By("Validate initial MOSC and wait for MOSB-1 to succeed") + ValidateSuccessfulMOSC(mosc, nil) + + mosb1, err := mosc.GetCurrentMachineOSBuild() + o.Expect(err).NotTo(o.HaveOccurred(), "Error getting initial MOSB") + logger.Infof("Initial MOSB created: %s\n", mosb1.GetName()) + + exutil.By("Apply MC with kernel argument to trigger new MOSB") + mc := NewMachineConfig(oc.AsAdmin(), mcName, infraMcp.GetName()) + mc.skipWaitForMcp = true + + defer mc.DeleteWithWait() + + err = mc.Create("-p", "NAME="+mcName, "-p", "POOL="+infraMcp.GetName(), + "-p", fmt.Sprintf(`KERNEL_ARGS=["%s"]`, kArgs)) + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating MachineConfig %s", mc.GetName()) + logger.Infof("OK!\n") + + exutil.By("Wait for MOSB-2 to be created and succeed") + checkNewBuildIsTriggered(mosc, mosb1) + mosb2, err := mosc.GetCurrentMachineOSBuild() + o.Expect(err).NotTo(o.HaveOccurred(), "Error getting MOSB-2") + logger.Infof("Second MOSB created: %s\n", mosb2.GetName()) + + exutil.By("Wait for MCP to complete update") + infraMcp.waitForComplete() + logger.Infof("OK!\n") + + exutil.By("Delete MOSB-1 image from Quay using automated skopeo deletion") + o.Expect(removeQuayImageUsingSkepo(oc.AsAdmin(), mosb1, infraMcp)).To(o.BeTrue(), "Error deleting Quay image for MOSB-1") + logger.Infof("OK!\n") + + // Common verification after MOSB-1 deletion + verifyMOSBRebuildAfterImageDeletion(infraMcp, mosc, mosb1, mosb2, mc, mcName) + }) + +}) + +func checkNewBuildIsTriggered(mosc *MachineOSConfig, currentMOSB *MachineOSBuild) { + var ( + newMOSB *MachineOSBuild + err error + ) + logger.Infof("Current mosb: %s", currentMOSB) + o.Eventually(func() (string, error) { + newMOSB, err = mosc.GetCurrentMachineOSBuild() + if err != nil || newMOSB == nil { + return "", err + } + return newMOSB.GetName(), nil + }, "5m", "20s").ShouldNot(o.Equal(currentMOSB.GetName()), + "A new MOSB should be created after the new rendered image pull spec is configured") + + logger.Infof("New mosb: %s", newMOSB) + + o.Eventually(newMOSB, "5m", "20s").Should(HaveConditionField("Building", "status", TrueString), + "MachineOSBuild didn't report that the build has begun") + + o.Eventually(newMOSB, "20m", "20s").Should(HaveConditionField("Building", "status", FalseString), "Build was not finished") + o.Eventually(newMOSB, "10m", "20s").Should(HaveConditionField("Succeeded", "status", TrueString), "Build didn't succeed") + o.Eventually(newMOSB, "2m", "20s").Should(HaveConditionField("Interrupted", "status", FalseString), "Build was interrupted") + o.Eventually(newMOSB, "2m", "20s").Should(HaveConditionField("Failed", "status", FalseString), "Build was failed") +} + +func getNewNodeRebootValueForOCL(newNode *Node, oclImage string) { + exutil.By("Verify the new node boots directly with the OCL image") + currentImage, err := newNode.GetCurrentBootOSImage() + o.Expect(err).NotTo(o.HaveOccurred(), "Error getting current boot OS image from node %s", newNode.GetName()) + o.Expect(currentImage).To(o.Equal(oclImage), "New node %s is not using the OCL image. Current: %s, Expected: %s", newNode.GetName(), currentImage, oclImage) + logger.Infof("OK!\n") + + exutil.By("Verify /etc/machine-config-daemon/currentimage matches the OCL image") + currentImageFile, err := newNode.DebugNodeWithChroot("cat", "/etc/machine-config-daemon/currentimage") + o.Expect(err).NotTo(o.HaveOccurred(), "Error reading currentimage file from node %s", newNode.GetName()) + o.Expect(currentImageFile).To(o.ContainSubstring(oclImage), "currentimage file does not contain the OCL image") + logger.Infof("OK!\n") + + exutil.By("Verify rpm-ostree status shows the OCL image") + rpmOstreeStatus, err := newNode.DebugNodeWithChroot("rpm-ostree", "status") + o.Expect(err).NotTo(o.HaveOccurred(), "Error getting rpm-ostree status from node %s", newNode.GetName()) + o.Expect(rpmOstreeStatus).To(o.ContainSubstring(oclImage), "rpm-ostree status does not show the OCL image") + logger.Infof("OK!\n") + + exutil.By("Verify no unnecessary reboots occurred - desiredImage file should not exist") + desiredImageFile, err := newNode.DebugNodeWithChroot("cat", "/etc/machine-config-daemon/desiredImage") + o.Expect(err).To(o.HaveOccurred(), "desiredImage file should not exist when node boots with correct OCL image") + o.Expect(desiredImageFile).Should(o.ContainSubstring("No such file or directory"), + "Expected desiredImage file to not exist, indicating no OS upgrade was needed") + logger.Infof("OK!\n") +} + +func ValidateNewNodesBootDirectlyWithOCLImage(oc *exutil.CLI, mosc *MachineOSConfig, mcp *MachineConfigPool) { + ValidateSuccessfulMOSC(mosc, nil) + + exutil.By("Get the OCL image") + oclImage := OrFail[string](mosc.GetStatusCurrentImagePullSpec()) + logger.Infof("OCL image: %s\n", oclImage) + + isInternalRegistry := OrFail[bool](mosc.IsUsingInternalRegistry()) + + exutil.By("Check able to scale the node from existing Machineset") + msl, err := NewMachineSetList(oc.AsAdmin(), MachineAPINamespace).GetAll() + o.Expect(err).NotTo(o.HaveOccurred(), "Get machinesets failed") + o.Expect(msl).ShouldNot(o.BeEmpty(), "Machineset list is empty") + existingMS := msl[0] + + o.Expect(existingMS.AddToScale(1)).NotTo(o.HaveOccurred()) + + defer func() { + exutil.By("Scale down the node from existing Machineset") + existingMS.AddToScale(-1) + mcp.waitForComplete() + }() + + exutil.By("Create duplicate machineset and scale new node") + machineset := OrFail[*MachineSet](GetScalableMachineSet(oc.AsAdmin())) + duplicateMSName := machineset.GetName() + "-ocl" + duplicateMS, err := machineset.Duplicate(duplicateMSName) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + if duplicateMS.Exists() { + logger.Infof("Deleting duplicate machineset %s and scale down the node", duplicateMS.GetName()) + o.Expect(duplicateMS.ScaleTo(0)).To(o.Succeed()) + o.Expect(duplicateMS.WaitUntilReady("10m")).To(o.Succeed()) + o.Expect(duplicateMS.Delete()).To(o.Succeed()) + } + }() + o.Expect(duplicateMS.ScaleTo(1)).To(o.Succeed()) + + exutil.By("Wait for both existing new node added and for duplicate new node added to get ready") + + o.Eventually(func(gm o.Gomega) { + gm.Expect(existingMS.GetIsReady()).To(o.BeTrue(), "MachineSet %s is not ready", existingMS.GetName()) + gm.Expect(duplicateMS.GetIsReady()).To(o.BeTrue(), "MachineSet %s is not ready", duplicateMS.GetName()) + }, "30m", "2m").Should(o.Succeed(), "MachineSets are not ready") + + if isInternalRegistry { + logger.Infof("Internal registry detected, waiting for MCP to complete (requires 2 reboots)") + mcp.waitForComplete() + } + logger.Infof("OK!\n") + + exutil.By("Get the new node created by the machine set") + + existingMSNodes, nErr := existingMS.GetNodes() + o.Expect(nErr).NotTo(o.HaveOccurred(), "Error getting the nodes created by MachineSet %s", existingMS.GetName()) + existingNode := existingMSNodes[len(existingMSNodes)-1] + logger.Infof("Existing node: %s\n", existingNode.GetName()) + logger.Infof("OK!\n") + + duplicateMSNodes, nErr := duplicateMS.GetNodes() + o.Expect(nErr).NotTo(o.HaveOccurred(), "Error getting the nodes created by MachineSet %s", duplicateMS.GetName()) + duplicateMSNode := duplicateMSNodes[len(duplicateMSNodes)-1] + logger.Infof("Duplicate node: %s\n", duplicateMSNode.GetName()) + logger.Infof("OK!\n") + + getNewNodeRebootValueForOCL(duplicateMSNode, oclImage) + getNewNodeRebootValueForOCL(existingNode, oclImage) + + exutil.By("Wait for MCP to complete before scaling down nodes") + mcp.waitForComplete() + logger.Infof("OK!\n") +} + +func removeImageStream(oc *exutil.CLI, mosb *MachineOSBuild) bool { + mosc, err := mosb.GetMachineOSConfig() + if err != nil { + logger.Errorf("Error getting MOSC from MOSB: %s", err) + return false + } + + pushSpec, err := mosc.GetRenderedImagePushspec() + if err != nil { + logger.Errorf("Error getting renderedImagePushspec from MOSC: %s", err) + return false + } + + parts := strings.Split(pushSpec, "/") + if len(parts) < 3 { + logger.Errorf("Invalid push spec format: %s", pushSpec) + return false + } + imageNameWithTag := parts[len(parts)-1] + imageName := strings.Split(imageNameWithTag, ":")[0] + + logger.Infof("Deleting imagestream tag: %s:%s", imageName, mosb.GetName()) + imagestream, err := oc.AsAdmin().WithoutNamespace().Run("tag").Args("-d", imageName+":"+mosb.GetName(), "-n", MachineConfigNamespace).Output() + + if err != nil { + logger.Errorf("Error deleting imagestream: %s", imagestream) + return false + } + return true +} + +func removeQuayImageUsingSkepo(oc *exutil.CLI, mosb *MachineOSBuild, mcp *MachineConfigPool) bool { + mosc, err := mosb.GetMachineOSConfig() + if err != nil { + logger.Errorf("Error getting MOSC from MOSB: %s", err) + return false + } + + digestedImage, err := mosb.GetStatusDigestedImagePullSpec() + if err != nil { + logger.Errorf("Error getting digested image from MOSB: %s", err) + return false + } + + logger.Infof("Attempting to delete Quay image: %s", digestedImage) + + pushSecretName, err := mosc.Get(`{.spec.renderedImagePushSecret.name}`) + if err != nil || pushSecretName == "" { + logger.Errorf("Error getting push secret name from MOSC: %s", err) + return false + } + + logger.Infof("Using push secret for deletion: %s", pushSecretName) + + pushSecret := NewSecret(oc, MachineConfigNamespace, pushSecretName) + secretDir, err := pushSecret.Extract() + if err != nil { + logger.Errorf("Error extracting push secret: %s", err) + return false + } + defer func() { + logger.Infof("Cleaning up local secret directory: %s", secretDir) + os.RemoveAll(secretDir) + }() + logger.Infof("Push secret extracted to local directory: %s", secretDir) + + localAuthFile := filepath.Join(secretDir, ".dockerconfigjson") + + nodes, err := mcp.GetNodes() + if err != nil || len(nodes) == 0 { + logger.Errorf("Error getting nodes from MCP: %s", err) + return false + } + node := nodes[0] + + logger.Infof("Running skopeo delete on node: %s", node.GetName()) + + tmpAuthFile := "/tmp/skopeo-delete-auth-" + exutil.GetRandomString() + ".json" + + defer func() { + logger.Infof("Cleaning up temporary auth file on node") + node.DebugNodeWithChroot("rm", "-f", tmpAuthFile) + }() + + err = node.CopyFromLocal(localAuthFile, tmpAuthFile) + if err != nil { + logger.Errorf("Error copying auth file to node: %s", err) + return false + } + + skopeoCommand := fmt.Sprintf("skopeo delete --authfile %s docker://%s", tmpAuthFile, digestedImage) + fullCommand := fmt.Sprintf("set -a; source /etc/mco/proxy.env; %s", skopeoCommand) + + logger.Infof("Executing command on node: %s", skopeoCommand) + + stdout, stderr, err := node.DebugNodeWithChrootStd("sh", "-c", fullCommand) + if err != nil { + logger.Errorf("Error deleting Quay image via skopeo on node. Stdout: %s, Stderr: %s, Error: %s", stdout, stderr, err) + return false + } + + logger.Infof("Successfully deleted Quay image: %s", digestedImage) + logger.Infof("Skopeo output: %s", stdout) + return true +} + +func verifyMOSBRebuildAfterImageDeletion(mcp *MachineConfigPool, mosc *MachineOSConfig, mosb1, mosb2 *MachineOSBuild, mc *MachineConfig, mcName string) { + exutil.By("Delete MC") + err := mc.Delete() + o.Expect(err).NotTo(o.HaveOccurred(), "Error deleting MC") + logger.Infof("OK!\n") + + exutil.By("Verify MOSB-1 is re-triggered (starts building again)") + o.Eventually(mosb1, "5m", "20s").Should(HaveConditionField("Building", "status", TrueString), + "MOSB-1 should be re-triggered and start building again after image deletion") + logger.Infof("MOSB-1 is re-building\n") + + o.Eventually(mosb1, "35m", "20s").Should(HaveConditionField("Building", "status", FalseString), + "MOSB-1 rebuild should complete") + o.Eventually(mosb1, "10m", "20s").Should(HaveConditionField("Succeeded", "status", TrueString), + "MOSB-1 rebuild should succeed") + logger.Infof("OK!\n") + + exutil.By("Wait for MCP to complete update") + mcp.waitForComplete() + logger.Infof("OK!\n") + + exutil.By("Re-apply MC") + mc.skipWaitForMcp = true + defer mc.DeleteWithWait() + err = mc.Create("-p", "NAME="+mcName, "-p", "POOL="+mcp.GetName(), + "-p", fmt.Sprintf(`KERNEL_ARGS=["%s"]`, "test")) + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating MachineConfig %s", mc.GetName()) + logger.Infof("OK!\n") + + exutil.By("Verify MOSB-2 is NOT triggered again") + o.Consistently(func() bool { + buildingStatus, err := mosb2.Get(`{.status.conditions[?(@.type=="Building")].status}`) + if err != nil || buildingStatus == "" { + return false + } + return buildingStatus == TrueString + }, "2m", "10s").Should(o.BeFalse(), "MOSB-2 should NOT start building again") + logger.Infof("OK!\n") + + exutil.By("Verify MCP starts updating instead of triggering a new/old MOSB") + o.Eventually(mcp, "5m", "20s").Should(HaveConditionField("Updating", "status", TrueString), + "MCP should start updating instead of triggering a new/old MOSB") + logger.Infof("OK!\n") + + exutil.By("Wait for MCP to complete update") + mcp.waitForComplete() + logger.Infof("OK!\n") + + exutil.By("Verify no new MOSB-3 was created during MCP update") + currentMosb, err := mosc.GetCurrentMachineOSBuild() + logger.Infof("Current MOSB: %s\n", currentMosb.GetName()) + logger.Infof("MOSB-1: %s\n", mosb1.GetName()) + logger.Infof("MOSB-2: %s\n", mosb2.GetName()) + o.Expect(err).NotTo(o.HaveOccurred(), "Error getting current MOSB") + o.Expect(currentMosb.GetName()).To(o.Or(o.Equal(mosb1.GetName()), o.Equal(mosb2.GetName())), + "Current MOSB should be either MOSB-1 or MOSB-2, no new MOSB-3 should be triggered") + logger.Infof("OK!\n") +} diff --git a/test/extended-priv/util/pods.go b/test/extended-priv/util/pods.go index 00ef4950eb..61bd2ffb37 100644 --- a/test/extended-priv/util/pods.go +++ b/test/extended-priv/util/pods.go @@ -189,3 +189,28 @@ func GetAllPodsWithLabel(oc *CLI, namespace, label string) ([]string, error) { } return strings.Split(pods, " "), err } + +// AssertAllNonJobPodsToBeReadyWithPollerParams asserts all non-job pods in a namespace are ready within the specified timeout +// Exclude transient Job pods ('job-name') from readiness: build Job pods are short-lived and not expected to be Ready=True, so including them causes false failure +// we validate the Job via logs/conditions separately, and this gate only asserts persistent controller pods (e.g., machine-os-builder) are Ready. +func AssertAllNonJobPodsToBeReadyWithPollerParams(oc *CLI, namespace string, interval, timeout time.Duration) { + ctx := context.Background() + err := wait.PollUntilContextTimeout(ctx, interval, timeout, true, func(_ context.Context) (bool, error) { + + // get the status flag for all pods + // except the ones which are in Complete Status. + // exclude pods that belong to Jobs (pods with 'job-name' label) + // it use 'ne' operator which is only compatible with 4.10+ oc versions + template := "'{{- range .items -}}{{- range .status.conditions -}}{{- if ne .reason \"PodCompleted\" -}}{{- if eq .type \"Ready\" -}}{{- .status}} {{\" \"}}{{- end -}}{{- end -}}{{- end -}}{{- end -}}'" + stdout, err := oc.AsAdmin().Run("get").Args("pods", "-n", namespace, "-l", "!job-name").Template(template).Output() + if err != nil { + e2e.Logf("the err:%v, and try next round", err) + return false, nil + } + if strings.Contains(stdout, "False") { + return false, nil + } + return true, nil + }) + AssertWaitPollNoErr(err, fmt.Sprintf("Some Pods are not ready in NS %s (excluding Job pods)!", namespace)) +} From 61fc8ec6c61f5c46da7d553083da85357ce937d0 Mon Sep 17 00:00:00 2001 From: Prachiti Talgulkar Date: Fri, 24 Jul 2026 19:29:35 +0530 Subject: [PATCH 2/2] Add OCB remaning tc and exclude the failing tc until they are resolved --- test/extended-priv/mco_ocb_longduration.go | 573 +++++---------------- 1 file changed, 132 insertions(+), 441 deletions(-) diff --git a/test/extended-priv/mco_ocb_longduration.go b/test/extended-priv/mco_ocb_longduration.go index 6a45f59314..db6a3b63b1 100644 --- a/test/extended-priv/mco_ocb_longduration.go +++ b/test/extended-priv/mco_ocb_longduration.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "sync" "time" g "github.com/onsi/ginkgo/v2" @@ -25,419 +26,205 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati SkipTestIfOCBIsEnabled(oc) }) - g.It("[PolarionID:79172][OTP] OCB Inherit from global pull secret if baseImagePullSecret field is not specified [Disruptive]", func() { + g.It("[PolarionID:83137][OTP][Skipped:Disconnected] OCB use OutputImage CurrentImagePullSecret [Disruptive]", func() { var ( - infraMcpName = "infra" - ) - - exutil.By("Create custom infra MCP") - // We add no workers to the infra pool, it is not necessary - infraMcp, err := CreateCustomMCP(oc.AsAdmin(), infraMcpName, 0) - defer infraMcp.delete() - o.Expect(err).NotTo(o.HaveOccurred(), "Error creating a new custom pool: %s", infraMcpName) - logger.Infof("OK!\n") - - testContainerFile([]ContainerFile{}, MachineConfigNamespace, infraMcp, nil, true) - }) - - g.It("[PolarionID:83136][OTP][Skipped:Disconnected] Panic Condition for Non-Matching MOSC Resources [Disruptive]", func() { - var ( - infraMcpName = "infra" - // MOSC has to use the same name as the mcp - moscName = infraMcpName - ) - exutil.By("Create New Custom MCP") - defer DeleteCustomMCP(oc.AsAdmin(), infraMcpName) - infraMcp, err := CreateCustomMCP(oc.AsAdmin(), infraMcpName, 1) - o.Expect(err).NotTo(o.HaveOccurred(), "Could not create a new custom MCP") - node := infraMcp.GetNodesOrFail()[0] - logger.Infof("%s", node.GetName()) - logger.Infof("OK!\n") - - exutil.By("Configure OCB functionality for the new infra MCP") - mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, infraMcpName, nil) - defer DisableOCL(mosc) - // remove after this bug is fixed OCPBUGS-36810 - defer func() { - logger.Infof("Configmaps should also be deleted ") - cmList := NewConfigMapList(mosc.GetOC(), MachineConfigNamespace).GetAllOrFail() - for _, cm := range cmList { - if strings.Contains(cm.GetName(), "rendered-") { - o.Expect(cm.Delete()).Should(o.Succeed(), "The ConfigMap related to MOSC has not been removed") - } - } - }() - o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") - logger.Infof("OK!\n") - - exutil.By("Check that a new build has been triggered") - o.Eventually(mosc.GetCurrentMachineOSBuild, "5m", "20s").Should(Exist(), - "No build was created when OCB was enabled") - mosb, err := mosc.GetCurrentMachineOSBuild() - o.Expect(err).NotTo(o.HaveOccurred(), "Error getting MOSB from MOSC") - o.Eventually(mosb.GetJob, "5m", "20s").Should(Exist(), - "No build pod was created when OCB was enabled") - o.Eventually(mosb, "5m", "20s").Should(HaveConditionField("Building", "status", TrueString), - "MachineOSBuild didn't report that the build has begun") - logger.Infof("OK!\n") - - exutil.By("Delete the MCOS and check it is deleted") - o.Expect(mosc.CleanupAndDelete()).To(o.Succeed(), "Error cleaning up %s", mosc) - ValidateMOSCIsGarbageCollected(mosc, infraMcp) - o.Expect(mosb).NotTo(Exist(), "Build is not deleted") - o.Expect(mosc).NotTo(Exist(), "MOSC is not deleted") - logger.Infof("OK!\n") - - exutil.AssertAllPodsToBeReady(oc.AsAdmin(), MachineConfigNamespace) - checkMCCPanic(oc) - }) - - g.It("[PolarionID:78001][OTP][Skipped:Disconnected] The etc-pki-etitlement secret is created automatically for OCB Use custom Containerfile with rhel enablement [Disruptive]", func() { - var ( - entitlementSecret = NewSecret(oc.AsAdmin(), "openshift-config-managed", "etc-pki-entitlement") - containerFileContent = ` - FROM configs AS final - - RUN rm -rf /etc/rhsm-host && \ - rpm-ostree install buildah && \ - ln -s /run/secrets/rhsm /etc/rhsm-host && \ - ostree container commit -` - - checkers = []Checker{ + mcp = GetCompactCompatiblePool(oc.AsAdmin()) + tmpNamespaceName = fmt.Sprintf("tc-%s-mco-ocl-images", GetCurrentTestPolarionIDNumber()) + checkers = []Checker{ CommandOutputChecker{ - Command: []string{"rpm", "-q", "buildah"}, - Matcher: o.ContainSubstring("buildah-"), - ErrorMsg: fmt.Sprintf("Buildah package is not installed after the image was deployed"), - Desc: fmt.Sprintf("Check that buildah is installed"), + Command: []string{"rpm-ostree", "status"}, + Matcher: o.ContainSubstring(fmt.Sprintf("%s/%s/ocb-%s-image", InternalRegistrySvcURL, tmpNamespaceName, mcp.GetName())), + ErrorMsg: fmt.Sprintf("The nodes are not using the expected OCL image stored in the internal registry"), + Desc: fmt.Sprintf("Check that the nodes are using the right OS image"), }, } - - mcp = GetCompactCompatiblePool(oc.AsAdmin()) ) - if !entitlementSecret.Exists() { - g.Skip(fmt.Sprintf("There is no entitlement secret available in this cluster %s. This test case cannot be executed", entitlementSecret)) - } - - testContainerFile([]ContainerFile{{Content: containerFileContent}}, MachineConfigNamespace, mcp, checkers, false) + testContainerFile([]ContainerFile{}, tmpNamespaceName, mcp, checkers, false) }) - g.It("[PolarionID:77498][OTP] OCB Trigger new build when renderedImagePushspec is updated [Disruptive]", func() { + g.It("[PolarionID:79137][OTP][Skipped:Disconnected] OCB Opting into on-cluster builds must respect maxUnavailable setting. Workers.[Disruptive]", func() { + SkipIfCompactOrSNO(oc.AsAdmin()) // This test makes no sense in SNO or Compact var ( - infraMcpName = "infra" - // MOSC resources have to use the same names as the MCP - moscName = infraMcpName + wMcp = NewMachineConfigPool(oc.AsAdmin(), MachineConfigPoolWorker) + // MOSC has to use the same name as the mcp + moscName = wMcp.GetName() + workerNodes = wMcp.GetSortedNodesOrFail() ) - exutil.By("Create custom infra MCP") - // We add no workers to the infra pool, it is not necessary - infraMcp, err := CreateCustomMCP(oc.AsAdmin(), infraMcpName, 0) - defer infraMcp.delete() - o.Expect(err).NotTo(o.HaveOccurred(), "Error creating a new custom pool: %s", infraMcpName) + exutil.By("Configure maxUnavailable if worker pool has more than 2 nodes") + if len(workerNodes) > 2 { + wMcp.SetMaxUnavailable(2) + defer wMcp.RemoveMaxUnavailable() + } + + maxUnavailable := OrFail[int](wMcp.GetMaxUnavailableInt()) + logger.Infof("Current maxUnavailable value %d", maxUnavailable) logger.Infof("OK!\n") - exutil.By("Configure OCB functionality for the new infra MCP") - mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, infraMcpName, nil) - defer mosc.CleanupAndDelete() + exutil.By("Configure OCB functionality for the new worker MCP") + mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, wMcp.GetName(), nil) + defer DisableOCL(mosc) o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") logger.Infof("OK!\n") - ValidateSuccessfulMOSC(mosc, nil) - - exutil.By("Set a new rendered image pull spec") - initialMOSB := OrFail[*MachineOSBuild](mosc.GetCurrentMachineOSBuild()) - initialRIPS := OrFail[string](mosc.GetRenderedImagePushspec()) - o.Expect( - mosc.SetRenderedImagePushspec(strings.ReplaceAll(initialRIPS, "ocb-", "ocb77498-")), - ).NotTo(o.HaveOccurred(), "Error patching %s to set the new renderedImagePullSpec", mosc) + exutil.By("Configure OCB functionality for the new worker MCP") + o.Eventually(wMcp.GetUpdatingStatus, "15m", "15s").Should(o.Equal("True"), + "The worker MCP did not start updating") logger.Infof("OK!\n") - exutil.By("Check that a new build is triggered") - checkNewBuildIsTriggered(mosc, initialMOSB) + exutil.By("Poll the nodes sorted by the order they are updated") + updatedNodes := wMcp.GetSortedUpdatedNodes(maxUnavailable) + for _, n := range updatedNodes { + logger.Infof("updated node: %s created: %s zone: %s", n.GetName(), n.GetOrFail(`{.metadata.creationTimestamp}`), n.GetOrFail(`{.metadata.labels.topology\.kubernetes\.io/zone}`)) + } logger.Infof("OK!\n") - exutil.By("Set the original rendered image pull spec") - o.Expect( - mosc.SetRenderedImagePushspec(initialRIPS), - ).NotTo(o.HaveOccurred(), "Error patching %s to set the new renderedImagePullSpec", mosc) + exutil.By("Wait for the configuration to be applied in all nodes") + wMcp.waitForComplete() logger.Infof("OK!\n") - exutil.By("Check that the initial build is reused") - var currentMOSB *MachineOSBuild - o.Eventually(func() (string, error) { - currentMOSB, err = mosc.GetCurrentMachineOSBuild() - if err != nil || currentMOSB == nil { - return "", err - } - return currentMOSB.GetName(), nil - }, "5m", "20s").Should(o.Equal(initialMOSB.GetName()), - "When the containerfiles were removed and initial MOSC configuration was restored, the initial MOSB was not used") + exutil.By("Check that nodes were updated in the right order") + rightOrder := checkUpdatedLists(workerNodes, updatedNodes, maxUnavailable) + o.Expect(rightOrder).To(o.BeTrue(), "Expected update order %s, but found order %s", workerNodes, updatedNodes) + logger.Infof("OK!\n") + exutil.By("Remove the MachineOSConfig resource") + o.Expect(DisableOCL(mosc)).To(o.Succeed(), "Error cleaning up %s", mosc) logger.Infof("OK!\n") }) - g.It("[PolarionID:77497][OTP] OCB Trigger new build when Containerfile is updated [Disruptive]", func() { + g.It("[PolarionID:83139][OTP][Skipped:Disconnected] OCB build images in many MCPs at the same time [Disruptive]", func() { + SkipIfCompactOrSNO(oc.AsAdmin()) // This test makes no sense in SNO or compact var ( - infraMcpName = "infra" - // MOSC resources have to use the same names as the MCP - moscName = infraMcpName - containerFile = ContainerFile{Content: "RUN touch /etc/test-add-containerfile" + "\n" + ExpirationDockerfileLabel} - containerFileMod = ContainerFile{Content: "RUN touch /etc/test-modified-containerfile" + "\n" + ExpirationDockerfileLabel} - - // We need to test a first MOSC without any container file, so we cannot add the expiration label in the first MOSC - expireImage = false - // We don't configure the pull secret in the MOSC, since it is optional - defaultPullSecret = true + customMCPNames = "infra" + numCustomPools = 5 + moscList = []*MachineOSConfig{} + mcpList = []*MachineConfigPool{} + wg sync.WaitGroup ) - exutil.By("Create custom infra MCP") - // We add no workers to the infra pool, it is not necessary - infraMcp, err := CreateCustomMCP(oc.AsAdmin(), infraMcpName, 0) - defer infraMcp.delete() - o.Expect(err).NotTo(o.HaveOccurred(), "Error creating a new custom pool: %s", infraMcpName) - logger.Infof("OK!\n") - - exutil.By("Configure OCB functionality for the new infra MCP") - mosc, err := createMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, infraMcpName, nil, defaultPullSecret, expireImage) + exutil.By("Create custom MCPS") + for i := 0; i < numCustomPools; i++ { + infraMcpName := fmt.Sprintf("%s-%d", customMCPNames, i) + infraMcp, err := CreateCustomMCP(oc.AsAdmin(), infraMcpName, 0) + defer infraMcp.delete() + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating a new custom pool: %s", infraMcpName) + mcpList = append(mcpList, infraMcp) - defer mosc.CleanupAndDelete() - o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") + } logger.Infof("OK!\n") - ValidateSuccessfulMOSC(mosc, nil) + exutil.By("Checking that all MOSCs were executed properly") + for _, infraMcp := range mcpList { + // MOSCs resources have to use the same name as the MCP + moscName := infraMcp.GetName() - exutil.By("Add new container file") - initialMOSB := OrFail[*MachineOSBuild](mosc.GetCurrentMachineOSBuild()) + mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, infraMcp.GetName(), nil) + defer mosc.CleanupAndDelete() + o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") + moscList = append(moscList, mosc) - o.Expect( - mosc.SetContainerfiles([]ContainerFile{containerFile}), - ).NotTo(o.HaveOccurred(), "Error patching %s to add a container file", mosc) - logger.Infof("OK!\n") + wg.Add(1) + go func() { + defer g.GinkgoRecover() + defer wg.Done() - exutil.By("Check that a new build is triggered when a containerfile is added") - checkNewBuildIsTriggered(mosc, initialMOSB) - logger.Infof("OK!\n") - - exutil.By("Modify the container file") - currentMOSB := OrFail[*MachineOSBuild](mosc.GetCurrentMachineOSBuild()) + ValidateSuccessfulMOSC(mosc, nil) + }() + } - o.Expect( - mosc.SetContainerfiles([]ContainerFile{containerFileMod}), - ).NotTo(o.HaveOccurred(), "Error patching %s to modify an existing container file", mosc) + wg.Wait() logger.Infof("OK!\n") - exutil.By("Check that a new build is triggered when a containerfile is modified") - checkNewBuildIsTriggered(mosc, currentMOSB) + exutil.By("Removing all MOSC resources") + for _, mosc := range moscList { + o.Expect(mosc.CleanupAndDelete()).To(o.Succeed(), "Error cleaning up %s", mosc) + } logger.Infof("OK!\n") - exutil.By("Remove the container files") - o.Expect( - mosc.RemoveContainerfiles(), - ).NotTo(o.HaveOccurred(), "Error patching %s to remove the configured container files", mosc) + exutil.By("Validate that all resources were garbage collected") + for i := 0; i < numCustomPools; i++ { + ValidateMOSCIsGarbageCollected(moscList[i], mcpList[i]) + } logger.Infof("OK!\n") - exutil.By("Check that the initial build is reused") - o.Eventually(func() (string, error) { - currentMOSB, err = mosc.GetCurrentMachineOSBuild() - if err != nil || currentMOSB == nil { - return "", err - } - return currentMOSB.GetName(), nil - }, "5m", "20s").Should(o.Equal(initialMOSB.GetName()), - "When the containerfiles were removed and initial MOSC configuration was restored, the initial MOSB was not used") - + waitForAllMCOPodsReady(oc.AsAdmin(), 10*time.Minute) logger.Infof("OK!\n") }) - g.It("[PolarionID:77576][OTP] In OCB. Create a new MC while a build is running [Disruptive]", func() { - + g.It("[PolarionID:83755][OTP] In OCL check no new image is applied on node after applying ssh/password/file MC .[Disruptive]", g.Label("Exclude: excluded until OCPBUGS-85094 is fixed"), func() { var ( - mcp = GetCompactCompatiblePool(oc.AsAdmin()) - node = mcp.GetSortedNodesOrFail()[0] - moscName = mcp.GetName() - - kArgs = "test" - mcName = "tc-77576-testkargs" - mc = NewMachineConfig(oc.AsAdmin(), mcName, mcp.GetName()) - ) - - exutil.By("Configure OCB functionality for the new worker MCP") - mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, mcp.GetName(), nil) - defer DisableOCL(mosc) - o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") - logger.Infof("OK!\n") - - exutil.By("Check that a new build has been triggered and is building") - var mosb *MachineOSBuild - o.Eventually(func() (*MachineOSBuild, error) { - var err error - mosb, err = mosc.GetCurrentMachineOSBuild() - return mosb, err - }, "5m", "20s").Should(Exist(), - "No build was created when OCB was enabled") - o.Eventually(mosb.GetJob, "5m", "20s").Should(Exist(), - "No build job was created when OCB was enabled") - o.Eventually(mosb, "5m", "20s").Should(HaveConditionField("Building", "status", TrueString), - "MachineOSBuild didn't report that the build has begun") - logger.Infof("OK!\n") - - exutil.By("Create a MC to trigger a new build") - defer mc.DeleteWithWait() - err = mc.Create("-p", "NAME="+mcName, "-p", "POOL="+mcp.GetName(), "-p", fmt.Sprintf(`KERNEL_ARGS=["%s"]`, kArgs)) - o.Expect(err).NotTo(o.HaveOccurred()) - logger.Infof("OK!\n") - - exutil.By("Check that a new build is triggered and the old build is removed") - checkNewBuildIsTriggered(mosc, mosb) - o.Eventually(mosb, "2m", "20s").ShouldNot(Exist(), "The old MOSB %s was not deleted", mosb) - logger.Infof("OK!\n") - - exutil.By("Wait for the configuration to be applied") - mcp.waitForComplete() - logger.Infof("OK!\n") - - exutil.By("Check that the MC was applied") - o.Expect(node.IsKernelArgEnabled(kArgs)).To(o.BeTrue(), - "Kernel argument %s was not enabled in the node %s", kArgs, node.GetName()) - - logger.Infof("OK!\n") - - exutil.By("Remove the MachineOSConfig resource") - o.Expect(DisableOCL(mosc)).To(o.Succeed(), "Error cleaning up %s", mosc) - logger.Infof("OK!\n") - }) + mcp = GetCompactCompatiblePool(oc.AsAdmin()) + node = mcp.GetSortedNodesOrFail()[0] - g.It("[PolarionID:77977][OTP][Skipped:Disconnected] Install extension after OCB is enabled [Disruptive]", func() { + moscName = mcp.GetName() + mcName = fmt.Sprintf("test-ssh-%s", GetCurrentTestPolarionIDNumber()) - var ( - mcp = GetCompactCompatiblePool(oc.AsAdmin()) - moscName = mcp.GetName() // MOSC resources have to use the same name as the MCP - node = mcp.GetSortedNodesOrFail()[0] - mcName = "test-install-extension-" + GetCurrentTestPolarionIDNumber() - applicableExtensions, _ = GetAllApplicableExtensionsToMCPOrFail(mcp) + _, key = GenerateSSHKeyPairOrFail() + user = ign32PaswdUser{Name: "core", SSHAuthorizedKeys: []string{key}} ) exutil.By("Configure OCB functionality for the new worker MCP") mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, mcp.GetName(), nil) defer DisableOCL(mosc) o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") - logger.Infof("OK!\n") + logger.Infof("Applied MOSC!\n") ValidateSuccessfulMOSC(mosc, nil) + logger.Infof("MOSC is applied!\n") - exutil.By("Create a MC") - mc := NewMachineConfig(oc.AsAdmin(), mcName, mcp.GetName()) - defer mc.DeleteWithWait() - mc.parameters = []string{fmt.Sprintf(`EXTENSIONS=%s`, string(MarshalOrFail(applicableExtensions)))} - mc.create() - logger.Infof("OK!\n") - - exutil.By("Wait for the configuration to be applied") - mcp.waitForComplete() - logger.Infof("OK!\n") + exutil.By("Get the image that is currently applied on nodes") + initialImage := OrFail[string](node.GetRpmOstreeStatus(false)) + logger.Infof("Initial image: %s", initialImage) + logger.Infof("Got the initial image!\n") - CheckExtensions(node, applicableExtensions) - - exutil.By("Delete a MC.") - mc.DeleteWithWait() - logger.Infof("OK!\n") - - exutil.By("Remove the MachineOSConfig resource") - o.Expect(DisableOCL(mosc)).To(o.Succeed(), "Error cleaning up %s", mosc) - ValidateMOSCIsGarbageCollected(mosc, mcp) - logger.Infof("OK!\n") - }) - - g.It("[PolarionID:78196][OTP][Skipped:Disconnected] Verify for etc-pki-etitlement secret is removed for OCB rhel enablement [Disruptive]", func() { - - var ( - entitlementSecret = NewSecret(oc.AsAdmin(), "openshift-config-managed", "etc-pki-entitlement") - containerFileContent = ` - FROM configs AS final - RUN rm -rf /etc/rhsm-host && \ - rpm-ostree install buildah && \ - ln -s /run/secrets/rhsm /etc/rhsm-host && \ - ostree container commit - ` - - mcp = GetCompactCompatiblePool(oc.AsAdmin()) - // MOSC resources have to use the same name as the MCP - moscName = mcp.GetName() - ) - if !entitlementSecret.Exists() { - g.Skip(fmt.Sprintf("There is no entitlement secret available in this cluster %s. This test case cannot be executed", entitlementSecret)) - } - - exutil.By("Copy the entitlement secret in MCO namespace") - mcoEntitlementSecret, err := CloneResource(entitlementSecret, "etc-pki-entitlement", MachineConfigNamespace, nil) - defer mcoEntitlementSecret.Delete() - o.Expect(err).NotTo(o.HaveOccurred(), "Error copying %s to the %s namespace", mcoEntitlementSecret, MachineConfigNamespace) - logger.Infof("OK!\n") - - exutil.By("Delete the entitlement secret in the openshift-config-managed namespace") - defer func() { - exutil.By("Recover the entitlement secret in the openshift-config-managed namespace") - recoverSecret, err := CloneResource(mcoEntitlementSecret, "etc-pki-entitlement", "openshift-config-managed", nil) - o.Expect(err).NotTo(o.HaveOccurred(), "Error copying %s to the openshift-config-managed namespace", entitlementSecret) - o.Expect(recoverSecret).To(Exist(), "Unable to recover the entitlement secret in openshift-config-managed namespace") - }() - - entitlementSecret.Delete() - logger.Infof("OK!\n") + exutil.By("Create a new MC to deploy new authorized keys") + mc := NewMachineConfig(oc.AsAdmin(), mcName, mcp.GetName()) + mc.parameters = []string{fmt.Sprintf(`PWDUSERS=[%s]`, MarshalOrFail(user))} + mc.skipWaitForMcp = true - exutil.By("Create the MOSC") - mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, mcp.GetName(), []ContainerFile{{Content: containerFileContent}}) - defer DisableOCL(mosc) - o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") - logger.Infof("OK!\n") + mc.create() + defer mc.DeleteWithWait() + logger.Infof("Created MC!\n") - exutil.By("Check that a new build has been triggered") - o.Eventually(mosc.GetCurrentMachineOSBuild, "5m", "20s").Should(Exist(), - "No build was created when OCB was enabled") + exutil.By("Check that the build is triggered with succeed status and not building") mosb, err := mosc.GetCurrentMachineOSBuild() + logger.Infof("MOSB: %s\n", mosb) o.Expect(err).NotTo(o.HaveOccurred(), "Error getting MOSB from MOSC") - o.Eventually(mosb, "5m", "20s").Should(HaveConditionField("Building", "status", TrueString), - "MachineOSBuild didn't report that the build has begun") - logger.Infof("OK!\n") + o.Expect(mosb).To(HaveConditionField("Building", "status", FalseString), "Build is still building") + o.Expect(mosb).To(HaveConditionField("Succeeded", "status", TrueString), "Build didn't succeed") + logger.Infof("Checked that the build does not take place!\n") - exutil.By("Verify the error is produced in buildPod") - exutil.AssertAllNonJobPodsToBeReadyWithPollerParams(mosc.GetOC(), MachineConfigNamespace, 10*time.Second, 10*time.Minute) + mcp.waitForComplete() logger.Infof("OK!\n") - job, err := mosb.GetJob() - o.Expect(err).NotTo(o.HaveOccurred()) - // Currently this kind of resources are leaked. Until the leak is fixed we need to make sure that this job is removed - // because its pods are in "Error" status and there are other test cases checking that no pod is reporting any error. - // TODO: remove this once the leak is fixed - defer job.Delete() - logger.Infof("OK!\n") + exutil.By("Check that the image is not updated") + o.Expect(OrFail[string](node.GetRpmOstreeStatus(false))).To(o.Equal(initialImage), "Image was updated") + logger.Infof("Image is not updated!\n") - o.Eventually(func() (string, error) { - return job.Logs("-c", "image-build") - }, "5m", "10s").Should(o.ContainSubstring("Found 0 entitlement certificates"), "Error getting the logs") - o.Eventually(job, "15m", "20s").Should(HaveConditionField("Failed", "status", TrueString), "Job didn't fail") - logger.Infof("OK!\n") + exutil.By("Check that all expected keys are present and with the right permissions and owners") + currentMc := OrFail[*MachineConfig](mcp.GetConfiguredMachineConfig()) + initialKeys := OrFail[[]string](currentMc.GetAuthorizedKeysByUserAsList("core")) + checkAuthorizedKeyInNode(node, append(initialKeys, key)) + logger.Infof("MC is configured with the expected keys!\n") - exutil.By("Remove the MachineOSConfig resource") - o.Expect(DisableOCL(mosc)).To(o.Succeed(), "Error cleaning up %s", mosc) - ValidateMOSCIsGarbageCollected(mosc, mcp) - logger.Infof("OK!\n") }) - g.It("[PolarionID:88801][OTP][Skipped:Disconnected] ExternalRegistry OCB Verify new nodes boot directly with OCL image without unnecessary reboots [Disruptive]", func() { - SkipIfCompactOrSNO(oc.AsAdmin()) // This test requires scaling, which doesn't make sense in SNO or Compact - skipTestIfWorkersCannotBeScaled(oc.AsAdmin()) // Skip test if worker node cannot be scaled + g.It("[PolarionID:85843][OTP][Skipped:Disconnected] InternalRegistry OCB Verify new nodes boot directly with OCL image without unnecessary reboots [Disruptive]", func() { + SkipIfCompactOrSNO(oc.AsAdmin()) // This test requires scaling, which doesn't make sense in SNO or Compact + skipTestIfWorkersCannotBeScaled(oc.AsAdmin()) // Skip test if worker node cannot be scaled + SkipTestIfCannotUseInternalRegistry(oc.AsAdmin()) // Skip test if cannot use internal registry var ( mcp = GetCompactCompatiblePool(oc.AsAdmin()) moscName = mcp.GetName() ) - exutil.By("Configure OCB functionality using external registry (Quay)") - mosc, err := CreateMachineOSConfigUsingExternalRegistry(oc.AsAdmin(), moscName, mcp.GetName(), nil, false, false) + exutil.By("Configure OCB functionality for the compatible MCP") + mosc, err := CreateMachineOSConfigUsingInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, mcp.GetName(), nil, true) defer DisableOCL(mosc) o.Expect(err).NotTo(o.HaveOccurred(), "Error creating the MachineOSConfig resource") logger.Infof("OK!\n") @@ -445,105 +232,14 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati ValidateNewNodesBootDirectlyWithOCLImage(oc.AsAdmin(), mosc, mcp) }) - g.It("[PolarionID:85980][OTP][Skipped:Disconnected] Check when MOSB is degraded MCP should be degraded too with right fields updated. [Disruptive]", func() { - var ( - infraMcpName = "infra" - moscName = infraMcpName - // Intentionally uses 'apt' on Alpine (which uses 'apk'), causing build to fail - incorrectContainerFile = "FROM alpine:3.18\nRUN apt update && apt install -y cowsay\n" - correctContainerFile = "FROM configs AS final\nRUN echo \"hello\" > /etc/test.txt\n" - expectedDegradedMessage = "Failed to build OS image" - ) - - exutil.By("Create custom infra MCP") - infraMcp, err := CreateCustomMCP(oc.AsAdmin(), infraMcpName, 1) - defer DeleteCustomMCP(oc.AsAdmin(), infraMcpName) - o.Expect(err).NotTo(o.HaveOccurred(), "Error creating custom MCP: %s", infraMcpName) - node := infraMcp.GetNodesOrFail()[0] - logger.Infof("Infra node: %s\n", node.GetName()) - logger.Infof("OK!\n") - - exutil.By("Create MOSC with incorrect containerfile") - mosc, err := CreateMachineOSConfigUsingExternalOrInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, infraMcpName, - []ContainerFile{{Content: incorrectContainerFile}}) - defer mosc.CleanupAndDelete() - o.Expect(err).NotTo(o.HaveOccurred(), "Error creating MOSC with incorrect containerfile") - logger.Infof("OK!\n") - - exutil.By("Verify MOSB is created and fails") - o.Eventually(mosc.GetCurrentMachineOSBuild, "5m", "20s").Should(Exist(), - "No MOSB was created for the incorrect containerfile") - failedMOSB, err := mosc.GetCurrentMachineOSBuild() - o.Expect(err).NotTo(o.HaveOccurred(), "Error getting MOSB from MOSC") - logger.Infof("MOSB created: %s\n", failedMOSB.GetName()) - o.Eventually(failedMOSB, "20m", "20s").Should(HaveConditionField("Failed", "status", TrueString), - "MOSB should fail due to incorrect containerfile") - logger.Infof("MOSB failed as expected\n") - - exutil.By("Verify MCP is degraded with ImageBuildDegraded condition") - o.Eventually(infraMcp, "5m", "20s").Should(BeDegraded(), "MCP should be degraded when build fails") - o.Eventually(infraMcp, "5m", "20s").Should(HaveConditionField("ImageBuildDegraded", "status", TrueString), - "MCP ImageBuildDegraded should be True when build fails") - o.Eventually(infraMcp, "5m", "20s").Should(HaveConditionField("ImageBuildDegraded", "message", o.ContainSubstring(expectedDegradedMessage)), - "MCP degradation message should indicate OS image build failure") - o.Eventually(infraMcp, "5m", "20s").Should(HaveConditionField("Degraded", "message", o.ContainSubstring("Custom OS image build failed")), - "MCP Degraded message should indicate custom OS image build failed") - logger.Infof("OK!\n") - - exutil.By("Fix the MOSC by updating containerfile") - o.Expect(mosc.SetContainerfiles([]ContainerFile{{Content: correctContainerFile}})).NotTo(o.HaveOccurred(), - "Error updating MOSC with correct containerfile") - logger.Infof("OK!\n") - - exutil.By("Verify new MOSB is created and succeeds") - checkNewBuildIsTriggered(mosc, failedMOSB) - newMOSB, err := mosc.GetCurrentMachineOSBuild() - o.Expect(err).NotTo(o.HaveOccurred(), "Error getting new MOSB") - logger.Infof("New MOSB created: %s\n", newMOSB.GetName()) - - exutil.By("Verify old failed MOSB is deleted") - o.Eventually(failedMOSB, "5m", "20s").ShouldNot(Exist(), - "Old failed MOSB should be garbage collected: %s", failedMOSB.GetName()) - logger.Infof("OK!\n") - - exutil.By("Verify MCP recovers and is no longer degraded") - // Check both ImageBuildDegraded (specific condition) and Degraded (overall status) - // to ensure complete recovery from the failed build state - o.Eventually(infraMcp, "5m", "20s").Should(HaveConditionField("ImageBuildDegraded", "status", FalseString), - "MCP ImageBuildDegraded condition should be False after recovery") - o.Eventually(infraMcp, "5m", "20s").ShouldNot(BeDegraded(), - "MCP Degraded condition should be False after recovery") - o.Eventually(infraMcp, "5m", "20s").Should(HaveConditionField("Updating", "status", TrueString), - "MCP should be updating after successful build") - logger.Infof("OK!\n") - - exutil.By("Wait for MCP to complete update") - infraMcp.waitForComplete() - logger.Infof("OK!\n") - - exutil.By("Verify image is applied to node") - currentImagePullSpec := OrFail[string](mosc.GetStatusCurrentImagePullSpec()) - o.Expect(node.GetCurrentBootOSImage()).To(o.Equal(currentImagePullSpec), - "Node should be using the correct OCL image after recovery") - logger.Infof("Node is using image: %s\n", currentImagePullSpec) - logger.Infof("OK!\n") - - exutil.By("Verify file created in containerfile exists on node") - output, err := node.DebugNodeWithChroot("cat", "/etc/test.txt") - o.Expect(err).NotTo(o.HaveOccurred(), "Error reading test file from node") - o.Expect(output).To(o.ContainSubstring("hello"), - "Test file content should match what was created in containerfile") - logger.Infof("OK!\n") - }) - - g.It("[PolarionID:87176][OTP][Skipped:Disconnected] External Registry In OCB to check when a image is removed the old build is triggered again and the MC should start updating directly. [Disruptive]", func() { + g.It("[PolarionID:82536][OTP][Skipped:Disconnected] Internal Registry In OCB to check when a image is removed the old build is triggered again and the MC should start updating directly. [Disruptive]", g.Label("Exclude: excluded until OCPBUGS-85094 is fixed"), func() { SkipIfCompactOrSNO(oc.AsAdmin()) + SkipTestIfCannotUseInternalRegistry(oc.AsAdmin()) // This test case requires the internal registry to be enabled var ( - infraMcpName = "infra" - mcName = fmt.Sprintf("tc-%s-ext-kernelarg", GetCurrentTestPolarionIDNumber()) - kArgs = "test" - containerFileNoExpiration = "FROM configs AS final\nLABEL maintainer=\"mco@example.com\"\n" + infraMcpName = "infra" + mcName = fmt.Sprintf("tc-%s-int-kernelarg", GetCurrentTestPolarionIDNumber()) + kArgs = "test" ) exutil.By("Create custom infra MCP") @@ -552,10 +248,9 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati o.Expect(err).NotTo(o.HaveOccurred(), "Error creating a new custom pool: %s", infraMcpName) logger.Infof("OK!\n") - exutil.By("Create MOSC for custom MCP using external registry") + exutil.By("Create MOSC for custom MCP using internal registry") moscName := infraMcp.GetName() - mosc, err := CreateMachineOSConfigUsingExternalRegistry(oc.AsAdmin(), moscName, infraMcp.GetName(), - []ContainerFile{{Content: containerFileNoExpiration}}, false, false) + mosc, err := CreateMachineOSConfigUsingInternalRegistry(oc.AsAdmin(), MachineConfigNamespace, moscName, infraMcp.GetName(), nil, false) defer DisableOCL(mosc) o.Expect(err).NotTo(o.HaveOccurred(), "Error creating MOSC") logger.Infof("OK!\n") @@ -584,12 +279,8 @@ var _ = g.Describe("[sig-mco][Suite:openshift/machine-config-operator/longdurati o.Expect(err).NotTo(o.HaveOccurred(), "Error getting MOSB-2") logger.Infof("Second MOSB created: %s\n", mosb2.GetName()) - exutil.By("Wait for MCP to complete update") - infraMcp.waitForComplete() - logger.Infof("OK!\n") - - exutil.By("Delete MOSB-1 image from Quay using automated skopeo deletion") - o.Expect(removeQuayImageUsingSkepo(oc.AsAdmin(), mosb1, infraMcp)).To(o.BeTrue(), "Error deleting Quay image for MOSB-1") + exutil.By("Delete MOSB-1 image from internal registry using oc tag -d") + o.Expect(removeImageStream(oc.AsAdmin(), mosb1)).To(o.BeTrue(), "Error deleting imagestream tag for MOSB-1") logger.Infof("OK!\n") // Common verification after MOSB-1 deletion