Make Maestro iOS retries flow-specific

.github/workflow-scripts/__tests__/maestro-ios-test.js

@@ -0,0 +1,62 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+jest.mock('child_process', () => ({
+  execSync: jest.fn(),
+  spawn: jest.fn(),
+}));
+jest.mock('fs', () => ({
+  existsSync: jest.fn(),
+  lstatSync: jest.fn(),
+  readdirSync: jest.fn(),
+}));
+
+const childProcess = require('child_process');
+const fs = require('fs');
+
+const {executeFlows} = require('../maestro-ios');
+
+describe('Maestro iOS runner', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    childProcess.spawn.mockReturnValue({pid: 1, kill: jest.fn()});
+  });
+
+  it('executes each YAML flow separately and skips other files', () => {
+    fs.existsSync.mockReturnValue(true);
+    fs.lstatSync.mockImplementation(path => ({
+      isDirectory: () => path === 'flows/',
+    }));
+    fs.readdirSync.mockReturnValue(['second.yaml', 'image.png', 'first.yml']);
+
+    executeFlows('com.example', 'device-id', 'flows/', 'Hermes');
+
+    expect(childProcess.execSync).toHaveBeenCalledTimes(2);
+    expect(childProcess.execSync.mock.calls[0][0]).toContain(
+      'test "flows/first.yml"',
+    );
+    expect(childProcess.execSync.mock.calls[1][0]).toContain(
+      'test "flows/second.yaml"',
+    );
+  });
+
+  it('retries only the failing flow', () => {
+    fs.existsSync.mockReturnValue(false);
+    childProcess.execSync.mockImplementationOnce(() => {
+      throw new Error('Maestro driver failed');
+    });
+
+    executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes');
+
+    expect(childProcess.execSync).toHaveBeenCalledTimes(2);
+    for (const call of childProcess.execSync.mock.calls) {
+      expect(call[0]).toContain('test "flow.yml"');
+    }
+  });
+});

.github/workflow-scripts/maestro-ios.js

@@ -23,19 +23,6 @@ node maestro-android.js <path to app> <app_id> <maestro_flow> <flavor> <working_
 ==============
 `;
 
-const args = process.argv.slice(2);
-
-if (args.length !== 6) {
-  throw new Error(`Invalid number of arguments.\n${usage}`);
-}
-
-const APP_PATH = args[0];
-const APP_ID = args[1];
-const MAESTRO_FLOW = args[2];
-const JS_ENGINE = args[3];
-const IS_DEBUG = args[4] === 'Debug';
-const WORKING_DIRECTORY = args[5];
-
 const MAX_ATTEMPTS = 5;
 
 function launchSimulator(simulatorName) {
@@ -91,7 +78,7 @@ function startVideoRecording(jsengine, currentAttempt) {
   );
 
   const recordingArgs =
-    `simctl io booted recordVideo video_record_${currentAttempt}.mov`.split(
+    `simctl io booted recordVideo --force video_record_${currentAttempt}.mov`.split(
       ' ',
     );
   const recordingProcess = childProcess.spawn('xcrun', recordingArgs, {
@@ -113,17 +100,12 @@ function stopVideoRecording(recordingProcess) {
   recordingProcess.kill('SIGINT');
 }
 
-function executeTestsWithRetries(
-  appId,
-  udid,
-  maestroFlow,
-  jsengine,
-  currentAttempt,
-) {
+function executeFlowWithRetries(appId, udid, flow, jsengine, currentAttempt) {
   const recProcess = startVideoRecording(jsengine, currentAttempt);
   try {
     const timeout = 1000 * 60 * 10; // 10 minutes
-    const command = `$HOME/.maestro/bin/maestro --udid="${udid}" test "${maestroFlow}" --format junit -e APP_ID="${appId}"`;
+    const command = `$HOME/.maestro/bin/maestro --udid="${udid}" test "${flow}" --format junit -e APP_ID="${appId}"`;
+    console.info(`Executing flow: ${flow} (attempt ${currentAttempt})`);
     console.log(command);
     childProcess.execSync(`MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 ${command}`, {
       stdio: 'inherit',
@@ -132,45 +114,72 @@ function executeTestsWithRetries(
 
     stopVideoRecording(recProcess);
   } catch (error) {
-    // Can't put this in the finally block because it will be executed after the
-    // recursive call of executeTestsWithRetries
     stopVideoRecording(recProcess);
 
     if (currentAttempt < MAX_ATTEMPTS) {
-      executeTestsWithRetries(
-        appId,
-        udid,
-        maestroFlow,
-        jsengine,
-        currentAttempt + 1,
-      );
+      console.info(`Retrying flow: ${flow}`);
+      executeFlowWithRetries(appId, udid, flow, jsengine, currentAttempt + 1);
     } else {
-      console.error(`Failed to execute flow after ${MAX_ATTEMPTS} attempts.`);
+      console.error(
+        `Failed to execute flow ${flow} after ${MAX_ATTEMPTS} attempts.`,
+      );
       throw error;
     }
   }
 }
 
-async function main() {
+function executeFlows(appId, udid, maestroFlow, jsengine) {
+  if (!fs.existsSync(maestroFlow) || !fs.lstatSync(maestroFlow).isDirectory()) {
+    executeFlowWithRetries(appId, udid, maestroFlow, jsengine, 1);
+    return;
+  }
+
+  for (const file of fs.readdirSync(maestroFlow).sort()) {
+    const filePath = `${maestroFlow.replace(/\/$/, '')}/${file}`;
+    if (fs.lstatSync(filePath).isDirectory()) {
+      executeFlows(appId, udid, filePath, jsengine);
+    } else if (file.endsWith('.yml') || file.endsWith('.yaml')) {
+      executeFlowWithRetries(appId, udid, filePath, jsengine, 1);
+    }
+  }
+}
+
+async function main(args = process.argv.slice(2)) {
+  if (args.length !== 6) {
+    throw new Error(`Invalid number of arguments.\n${usage}`);
+  }
+
+  const appPath = args[0];
+  const appId = args[1];
+  const maestroFlow = args[2];
+  const jsengine = args[3];
+  const isDebug = args[4] === 'Debug';
+  const workingDirectory = args[5];
+
   console.info('\n==============================');
   console.info('Running tests for iOS with the following parameters:');
-  console.info(`APP_PATH: ${APP_PATH}`);
-  console.info(`APP_ID: ${APP_ID}`);
-  console.info(`MAESTRO_FLOW: ${MAESTRO_FLOW}`);
-  console.info(`JS_ENGINE: ${JS_ENGINE}`);
-  console.info(`IS_DEBUG: ${IS_DEBUG}`);
-  console.info(`WORKING_DIRECTORY: ${WORKING_DIRECTORY}`);
+  console.info(`APP_PATH: ${appPath}`);
+  console.info(`APP_ID: ${appId}`);
+  console.info(`MAESTRO_FLOW: ${maestroFlow}`);
+  console.info(`JS_ENGINE: ${jsengine}`);
+  console.info(`IS_DEBUG: ${isDebug}`);
+  console.info(`WORKING_DIRECTORY: ${workingDirectory}`);
   console.info('==============================\n');
 
   const simulatorName = 'iPhone 16 Pro';
   launchSimulator(simulatorName);
-  installAppOnSimulator(APP_PATH);
+  installAppOnSimulator(appPath);
   const udid = extractSimulatorUDID();
   bringSimulatorInForeground();
-  await launchAppOnSimulator(APP_ID, udid, IS_DEBUG);
-  executeTestsWithRetries(APP_ID, udid, MAESTRO_FLOW, JS_ENGINE, 1);
+  await launchAppOnSimulator(appId, udid, isDebug);
+  executeFlows(appId, udid, maestroFlow, jsengine);
   console.log('Test finished');
-  process.exit(0);
 }
 
-main();
+if (require.main === module) {
+  main();
+}
+
+module.exports = {
+  executeFlows,
+};