Skipped test sometimes runs and fails due to intermittent Cypress issue

[saksham-s-safe]

Current behavior

In one test is marked as skipped.
However, Cypress intermittently executes the skipped test and fails it.
This occurs even though the test does not have the provided tag for execution.

Desired behavior

Cypress should consistently respect the skipped status of the test and never execute it.
The test should not run unless explicitly unskipped or included in the execution scope.

Test code to reproduce

import Actions from "@support/Actions";
import Assertions from "@support/Assertions";
import Reports from "@pageObjects/reports";
import ControlCenter from "@pageObjects/controls/controlCenter";
import { generateString } from "@support/utils";

import messages from "@fixtures/common/messages.json";
import { uiRoutes, apiRoutes } from "@fixtures/common/routes.json";

import {
    Components,
    Platform,
    TestTypes,
    Services,
    Pages
} from "@support/Tags";

import { assetPayload } from "@fixtures/testData/techFindingsData";
import { customDashboardData } from "@fixtures/testData/customDashboard";
import { v8SmartGroupPayload } from "@fixtures/testData/groupTestData";
import { riskScenarioCreationPayload } from "@fixtures/testData/riskScenarioTestData";
import { control } from "@fixtures/testData/controlCentreTestData";

import RiskScenarioList from "@pageObjects/riskScenarios/riskScenarioList";
import Invitations from "@pageObjects/settings/invitations";
import FIQuestionnaire from "@pageObjects/questionnaire/fiq";
import Groups from "@pageObjects/groups/groups";
import NistCSFQuestionnaire from "@pageObjects/questionnaire/nist";
import Settings from "@pageObjects/settings/settings";
import SignIn from "@pageObjects/login/signIn";
import SignUp from "@pageObjects/login/signUp";
import Table from "@pageObjects/table";
import CustomDashboard from "@pageObjects/customDashboard";

const actions = new Actions();
const assertions = new Assertions();
const groups = new Groups();
const fiq = new FIQuestionnaire();
const reports = new Reports();
const controlCenter = new ControlCenter();
const invitationsPage = new Invitations();
const nist = new NistCSFQuestionnaire();
const settings = new Settings();
const loginPage = new SignIn();
const signUpPage = new SignUp();
const table = new Table();
const riskScenarioList = new RiskScenarioList();

const preRequisiteGroupName = `custom-role-smoke-group-pre-req-${generateString()}`;
const preRequisiteRiskScenarioName = `custom-role-smoke-rs-pre-req-${generateString()}`;
const preRequisiteGroupPayload = v8SmartGroupPayload(assetPayload.name);
const customRoleName = `custom-role-smoke-role-${generateString()}`;
const customUserFirstName = `Custom`;
const customUserLastName = `User`;
const customUserPassword = `Safe@123`;
const fiqQuestionnaireName = `Financial Impact Questionnaire`;

const { app: testDataApp, login: loginTestData } = global.testData;
const { v8TenantId } = loginTestData;
const { mtUsername: username, url } = testDataApp;

const customDashboard = new CustomDashboard();
const dashboardName = customDashboardData.dashboard.name;

let assetId: number | null = null;
let preRequisiteGroupId: number | null = null;
let riskScenarioId: number | null = null;
let customRoleUserEmail: string | null = null;
let generatedEmailTag: string | null = null;
let mfaSecretForCustomUser: string | null = null;

describe(
    "To verify the working of custom roles @S5b289a61",
    {
        tags: [
            TestTypes.sanity,
            Components.roles,
            Services.authService,
            Platform.multiTenant,
            Services.authService,
            Pages.roleManagement,
            Pages.invitations
        ]
    },
    () => {
        before(() => {
            cy.createTestEmail().then(args => {
                const { emailId, tag } = args;

                customRoleUserEmail = emailId;
                generatedEmailTag = tag;
                cy.task("log", {
                    message: "\t✔ Generated new email as a part of test data.",
                    username
                });
            });

            cy.mtUILogin({ username, tenantId: v8TenantId });
            cy.addAsset(assetPayload).then((res: any) => {
                assetId = res.body.data.id;

                const groupCreationPayload = {
                    name: preRequisiteGroupName,
                    ...preRequisiteGroupPayload
                };

                cy.addGroup(groupCreationPayload).then(resp => {
                    preRequisiteGroupId = resp.body.data.id;
                    cy.addRiskScenario({
                        ...riskScenarioCreationPayload,
                        attackSurfaceId: resp.body.data.id,
                        name: preRequisiteRiskScenarioName
                    }).then(riskResp => {
                        ({ riskScenarioId } = riskResp.body.data);
                    });
                });
            });
        });

        after(() => {
            cy.mtUILogin({ username, tenantId: v8TenantId });
            cy.getUsers(customRoleUserEmail as string).then(response => {
                const users = response.body.values;
                cy.deleteUser(users[0].peopleId);
            });

            cy.deleteCustomRole(customRoleName);

            cy.deleteAsset(assetId);
            cy.deleteGroup(preRequisiteGroupId);
        });

        it("To verify admin user is able to create new role @T99ece6a3", () => {
            cy.mtUILogin({ username, tenantId: v8TenantId });

            cy.visit(uiRoutes.roleManagement);

            actions.click({
                element: settings.roleManagement().addRoleButton()
            });
            actions.typeText({
                element: settings.roleManagement().roleManagePage.nameInput(),
                value: customRoleName,
                options: {
                    replace: true
                }
            });
            actions.typeText({
                element: settings.roleManagement().roleManagePage.searchGroup(),
                value: preRequisiteGroupName,
                options: {
                    replace: true
                }
            });
            assertions.toHaveText({
                element: settings
                    .roleManagement()
                    .roleManagePage.permissionsTable.columns(0)
                    .name(),
                value: preRequisiteGroupName
            });

            actions.click({
                element: settings
                    .roleManagement()
                    .roleManagePage.permissionsTable.columns(0)
                    .write()
            });

            // Expect that read permission is checked automatically when write permission is checked
            assertions.shouldBeChecked({
                element: settings
                    .roleManagement()
                    .roleManagePage.permissionsTable.columns(0)
                    .read()
            });
            actions.click({
                element: settings.roleManagement().roleManagePage.saveButton()
            });

            assertions.toHaveText({
                element: settings.roleManagement().notification.title(),
                value: messages.roleManagement.notificationTitles.success
            });
            assertions.toHaveText({
                element: settings.roleManagement().notification.description(),
                value: messages.roleManagement.success.create
            });

            // Verify the role is created and visible in the list
            actions.typeText({
                element: settings.roleManagement().searchField(),
                value: customRoleName,
                options: {
                    replace: true
                }
            });

            assertions.toHaveText({
                element: settings.roleManagement().rolesTable.columns().name(),
                value: customRoleName
            });
            assertions.toHaveText({
                element: settings
                    .roleManagement()
                    .rolesTable.columns()
                    .type("Custom"),
                value: messages.roleManagement.roleTypes.custom
            });
        });

        it("To verify admin user is able to assign the new role to another user @Tc303080a", () => {
            cy.mtUILogin({ username, tenantId: v8TenantId });

            cy.sendUserInvite({
                emailId: customRoleUserEmail as string,
                role: customRoleName
            });

            cy.visit(uiRoutes.invitations);

            actions.click({
                element: invitationsPage.invitationsSearchBar()
            });
            actions.typeText({
                element: invitationsPage.invitationsSearchBar(),
                value: customRoleUserEmail as string
            });
            assertions.toHaveText({
                element: invitationsPage.getInviteDetails().toEmail(),
                value: customRoleUserEmail as string
            });
            assertions.toHaveText({
                element: invitationsPage.getInviteDetails().role(),
                value: customRoleName
            });

            cy.getFirstEmail(generatedEmailTag as string).then(email => {
                cy.fetchInvitationCode(email.text).then(text => {
                    cy.wrap(text).as("inviteLink");
                    cy.task(
                        "log",
                        `\t✔ Fetched invitation code for the user - ${text}`
                    );
                });
            });
            cy.get("@inviteLink").then(inviteLink => {
                cy.visit(`${url}?inviteCode=${inviteLink}`);
            });

            actions.typeText({
                element: signUpPage.signUpFormContent().firstName(),
                value: customUserFirstName,
                options: {
                    pressEnter: false
                }
            });
            actions.typeText({
                element: signUpPage.signUpFormContent().lastName(),
                value: customUserLastName,
                options: {
                    pressEnter: false
                }
            });

            actions.click({
                element: signUpPage.nextButton()
            });

            actions.typeText({
                element: signUpPage.createPasswordForm().createPassword(),
                value: customUserPassword,
                options: {
                    replace: true,
                    hideText: true
                }
            });
            actions.typeText({
                element: signUpPage.createPasswordForm().confirmPassword(),
                value: customUserPassword,
                options: {
                    replace: true,
                    hideText: true
                }
            });
            actions.click({
                element: signUpPage
                    .createPasswordForm()
                    .termsOfServiceCheckbox()
            });
            actions.click({
                element: signUpPage.signUpButton()
            });

            // Login to new user
            actions.click({
                element: signUpPage.loginButton()
            });
            actions.click({
                element: loginPage.usernamePage().differentUser()
            });
            assertions.shouldBeVisible({
                element: loginPage.usernamePage().signInForm()
            });

            actions.typeText({
                element: loginPage.usernamePage().emailInput(),
                value: customRoleUserEmail as string
            });
            actions.typeText({
                element: loginPage.passwordPage().passwordInput(),
                value: customUserPassword
            });

            cy.task("log", "\tSetting up 2FA...");
            assertions.shouldBeVisible({
                element: signUpPage.mfa().code()
            });
            signUpPage
                .mfa()
                .qrSecret()
                .then(qrSecret => {
                    const authCode = qrSecret.text();
                    cy.wrap(authCode).as("authCode");
                    mfaSecretForCustomUser = authCode;
                });
            cy.get("@authCode").then(authCode => {
                cy.generate2faCode(authCode).then(qrSecret => {
                    loginPage.mfaPage().mfaCodeInput().click().type(qrSecret);
                });
            });

            cy.task("log", "\t✔ Entered Auth code");
            loginPage.mfaPage().verifyButton().click();
            cy.url({ timeout: 15000 }).should("contain", uiRoutes.dashboard);
            cy.task("log", "\t✔ Logged in successfully as a first time user");

            cy.intercept("GET", `**api/v3/me*`).as("getUserProfile");
            cy.navigateTo(uiRoutes.myProfile);
            cy.wait("@getUserProfile", { timeout: 30 * 1000 });
            settings
                .myProfile()
                .emailId()
                .should("have.value", customRoleUserEmail);
        });

        it.skip("Validate Add dashboard functionality @Tf707d6ae", () => {
            cy.login({
                username: customRoleUserEmail,
                tenantId: v8TenantId,
                mfaSecret: mfaSecretForCustomUser as string,
                password: customUserPassword,
                saveTenantIdInLocalStorage: false
            });

            cy.intercept("GET", `${apiRoutes.riskScenarios}/outputs?*`).as(
                "getRiskScenarios"
            );
            cy.visit(`${uiRoutes.dashboard}/1`);
            cy.reload();
            cy.wait("@getRiskScenarios", { timeout: 30 * 1000 });

            customDashboard.dashboardDropdown().activeDashboardName().click();
            actions.click({
                element: customDashboard.dashboardDropdown().addDashboard()
            });

            cy.task("log", "\tAdding Custom Dashboard...");
            cy.enterValue(
                customDashboard.addDashboard().dashboardName(),
                dashboardName
            );
            actions.click({
                element: customDashboard.addDashboard().createDashboardSave()
            });

            actions.click({
                element: customDashboard.addDashboard().makePublicCheck(),
                force: true
            });

            cy.task("log", "\tAdding a widget to the Custom Dashboard...");
            actions.click({
                element: customDashboard.addWidget().addWidget()
            });
            assertions.shouldBeVisible({
                element: customDashboard.addWidget().widgetTitle(14),
                options: {
                    shouldScroll: true
                }
            });
            customDashboard
                .addWidget()
                .widgetInput(14)
                .click()
                .type(customDashboardData.dashboard.dashboardTextWidget);
            actions.click({
                element: customDashboard.addWidget().saveWidget(14)
            });
            assertions.shouldContainText({
                element: customDashboard.addWidget().addWidgetSuccessMsg(),
                value: customDashboardData.dashboard.widgetNotification
            });
            actions.click({
                element: customDashboard.addWidget().closeWidgetNotification()
            });

            actions.click({
                element: customDashboard.addWidget().closeWidgetDashlet()
            });

            actions.click({
                element: customDashboard.addDashboard().saveDashboard()
            });
            assertions.toHaveText({
                element: customDashboard
                    .addDashboard()
                    .saveDashboardSuccessMsg(),
                value: customDashboardData.dashboard.saveNotification
            });
            actions.click({
                element: customDashboard.addDashboard().closeSaveNotification()
            });

            assertions.shouldBeVisible({
                element: customDashboard.addWidget().widget().dashboardText()
            });
            assertions.toHaveText({
                element: customDashboard
                    .dashboardDropdown()
                    .activeDashboardName(),
                value: dashboardName
            });

            cy.task("log", "\t✔ Custom Dashboard added successfully.");

            //Deleting the dashboard
            cy.slowDown(1000);

            actions.click({
                element: customDashboard.manageDashboard().manage()
            });
            actions.click({
                element: customDashboard.manageDashboard().deleteDashboard()
            });
            actions.click({
                element: customDashboard.deleteDashboard().deleteButton()
            });
            assertions.toHaveText({
                element: customDashboard.deleteDashboard().deleteNotification(),
                value: customDashboardData.dashboard.deleteNotification
            });

            cy.slowDownEnd();

            cy.task("log", "\t✔ Deleted custom dashboard successfully.");
        });

        it("To verify the custom role user is able to download reports @T6cd239b8", () => {
            cy.login({
                username: customRoleUserEmail,
                tenantId: v8TenantId,
                mfaSecret: mfaSecretForCustomUser as string,
                password: customUserPassword,
                saveTenantIdInLocalStorage: false
            });

            cy.intercept("GET", "**/risk-scenarios?page=1&pagelen=10*").as(
                "getRiskScenarios"
            );
            cy.visit(uiRoutes.riskScenarios);
            assertions.shouldBeVisible({
                element: riskScenarioList
                    .allRiskScenarios()
                    .riskScenarioTableBody(),
                options: {
                    shouldScroll: true
                }
            });

            cy.wait("@getRiskScenarios", { timeout: 15000 });

            assertions.toHaveElementLengthGreaterThan({
                element: table.rows().bodyRow("risk-scenarios-table-body"),
                count: 1
            });
            assertions.shouldBeVisible({
                element: reports.icon("risk-scenario")
            });
            actions.click({
                element: reports.icon("risk-scenario")
            });
            assertions.shouldBeVisible({
                element: reports.notification()
            });
            assertions.shouldNotExist({
                element: reports.notification()
            });

            cy.intercept("GET", "**/reports?page=1&pagelen=10*").as(
                "getReports"
            );
            cy.visit(uiRoutes.reports);
            cy.wait("@getReports", { timeout: 15000 });
            cy.checkReportStatus();

            assertions.toHaveText({
                element: reports.status(),
                value: "Success"
            });
            actions.click({ element: reports.download() });
            assertions.shouldBeVisible({
                element: reports.notification()
            });
            assertions.shouldNotExist({
                element: reports.notification()
            });
            assertions.fileShouldContainText({
                element: reports.row(),
                value: `${riskScenarioId},${preRequisiteRiskScenarioName},${preRequisiteGroupName}`,
                timeout: 60000
            });
            cy.task(
                "log",
                `\t✔ Reports downloading verified for risk-scenarios.`
            );
        });

        it("To verify the custom role user is able to assess questionnaire @T3d41364c", () => {
            cy.login({
                username: customRoleUserEmail,
                tenantId: v8TenantId,
                mfaSecret: mfaSecretForCustomUser as string,
                password: customUserPassword,
                saveTenantIdInLocalStorage: false
            });

            cy.navigateTo(uiRoutes.groups);

            assertions.toHaveInnerTextEqual({
                element: groups.groupCreation().groupNameColumn(),
                value: preRequisiteGroupName
            });

            assertions.toHaveLengthEqualTo({
                element: groups.groupCreation().groupListBody(),
                count: 1
            });

            actions.click({
                element: groups.groupCreation().groupNameColumn()
            });

            assertions.toHaveText({
                element: groups.groupDetails().groupHeading(),
                value: preRequisiteGroupName
            });

            actions.click({
                element: groups.groupTabs().questionnaireTab(),
                options: {
                    shouldScroll: false
                }
            });
            assertions.shouldBeVisible({
                element: groups.groupCreation().groupQuestionnaireListBody()
            });
            cy.task("log", `\t✔ Questionnaire list is now visible.`);

            // Fill up some fiq questionnaire questions
            actions.typeText({
                element: groups.groupTabs().questionnaireSearchbar(),
                value: fiqQuestionnaireName
            });
            assertions.toHaveLengthEqualTo({
                element: table.rows().bodyRow("group-questionnaire-list-body"),
                count: 1
            });
            assertions.toHaveText({
                element: groups.groupTabs().questionnaireName(),
                value: fiqQuestionnaireName,
                options: {
                    shouldScroll: true
                }
            });
            actions.click({ element: groups.groupTabs().questionnaireName() });

            actions.typeText({
                element: fiq.getFIQAssessmentInput(0),
                value: "97",
                options: {
                    replace: true,
                    pressEnter: false
                }
            });
            actions.click({
                element: fiq.getFIQAssessmentInput(1)
            });
            assertions.toHaveText({
                element: nist.getSuccessMessage(),
                value: messages.success,
                options: {
                    shouldScroll: true
                }
            });
            fiq.getFIQMessageCloseIcon().click();
            cy.task(
                "log",
                `\t✔ Verified that user is able to edit group and assess FIQ controls.`
            );
        });

        it("To verify the custom role user is able to assess controls @T0a3e9476", () => {
            cy.login({
                username: customRoleUserEmail as string,
                tenantId: v8TenantId,
                mfaSecret: mfaSecretForCustomUser,
                password: customUserPassword,
                saveTenantIdInLocalStorage: false
            });

            cy.visit(`${uiRoutes.groups}/${preRequisiteGroupId}`);
            groups.groupTabs().controlsTab().click();

            cy.task("log", "\t↻ Navigated to Group Control Center page");

            assertions.shouldBeVisible({
                element: controlCenter.table().body(),
                options: {
                    shouldScroll: true
                }
            });

            assertions.shouldBeVisible({
                element: controlCenter.controlInsights().controlInsightsWidget()
            });

            actions.click({
                element: controlCenter.table().search()
            });
            actions.typeText({
                element: controlCenter.table().search(),
                value: control.dscControlId,
                options: {
                    replace: true,
                    pressEnter: true
                }
            });

            assertions.shouldBeVisible({
                element: table
                    .columns()
                    .name("nist-csf-assessment-list-body", 0)
            });
            cy.visit(
                `${uiRoutes.groups}/${preRequisiteGroupId}/controls/${control.dscControlId}?pager=true`
            );
            cy.slowDown(1000);
            assertions.shouldContain({
                element: controlCenter.controlParameter().edit(),
                value: "Edit"
            });
            actions.click({
                element: controlCenter.controlParameter().edit()
            });
            actions.click({
                element: controlCenter.parameterButtonMapping.Capability.up(),
                force: true
            });
            actions.click({
                element: controlCenter.parameterButtonMapping.Coverage.up(),
                force: true
            });
            actions.click({
                element: controlCenter.parameterButtonMapping.Capability.up(),
                force: true
            });
            actions.click({
                element: controlCenter.parameterButtonMapping.Coverage.up(),
                force: true
            });
            actions.click({
                element: controlCenter.parameterButtonMapping.Reliability.up(),
                force: true
            });

            cy.intercept("PATCH", "**/assessment*").as("controlAssessmentCall");

            //Save the changed control
            actions.click({
                element: controlCenter.editControlParameter().save()
            });

            cy.wait("@controlAssessmentCall");
            assertions.toHaveText({
                element: controlCenter
                    .editControlParameter()
                    .notificationSuccess(),
                value: messages.success
            });

            cy.slowDownEnd();
        });

        it("To verify the visibility of about page for custom role @T13fa128c", () => {
            cy.login({
                username: customRoleUserEmail as string,
                tenantId: v8TenantId,
                mfaSecret: mfaSecretForCustomUser,
                password: customUserPassword,
                saveTenantIdInLocalStorage: false
            });
            cy.intercept("GET", `**api/v3/me*`).as("getUserProfile");
            cy.visit(uiRoutes.myProfile);
            cy.wait("@getUserProfile", { timeout: 30 * 1000 });

            const firstName = `firstname${generateString()}`;
            const lastName = `lastname${generateString()}`;

            assertions.shouldBeVisible({
                element: settings.myProfile().currencySelector()
            });

            actions.typeText({
                element: settings.myProfile().firstNameInput(),
                value: firstName,
                options: {
                    replace: true
                }
            });

            actions.typeText({
                element: settings.myProfile().lastNameInput(),
                value: lastName,
                options: {
                    replace: true
                }
            });

            actions.click({
                element: settings.myProfile().saveProfile()
            });
            assertions.shouldBeVisible({
                element: settings.myProfile().notification()
            });
            actions.click({
                element: settings.myProfile().notificationClose()
            });
            cy.task("log", "\t✔ User details updated successfully.");

            actions.click({
                element: settings.settingsLeftNav().aboutSection()
            });

            assertions.toHaveText({
                element: settings.aboutSection().tenantId(),
                value: v8TenantId
            });
        });
    }
);

Cypress Version

Cypress package version: 13.15.0

Node version

v20.11.1

Operating System

linux

[jennifer-shehane]

@saksham-s-safe Are you using @cypress/grep?

Please also ensure you are using the latest versions of the plugin if so, and Cypress.