Screen States

Detect and respond to different UI states

Screen state detection is the foundation of robust automation. By identifying what's currently on screen, your automation can respond appropriately to each situation.

The ScreenState Pattern

Define all possible screen states as an enum:

screenStates.ts

TypeScript

export enum ScreenState {
  // System states
  Unknown = "Unknown",
  Crash = "Crash",
  PhoneDialog = "PhoneDialog",
  NoInternet = "NoInternet",
  NotificationShade = "NotificationShade",

  // App states
  SplashScreen = "SplashScreen",
  LoginScreen = "LoginScreen",
  LoginEnterPassword = "LoginEnterPassword",
  HomeScreen = "HomeScreen",
  ProfileScreen = "ProfileScreen",
  SettingsScreen = "SettingsScreen",

  // Dialog states
  PermissionDialog = "PermissionDialog",
  ConfirmDialog = "ConfirmDialog",
  ErrorDialog = "ErrorDialog",

  // Loading states
  Loading = "Loading",

  // Success/failure states
  Success = "Success",
  RateLimited = "RateLimited",
}

Detection Strategies

1. Text-Based Detection

The simplest approach - look for specific text on screen:

Text-based detection

TypeScript

function detectScreenState(screen: AndroidNode): ScreenState {
  const allNodes = getAllNodes(screen);

  // Check for specific text
  if (allNodes.find(n => n.text === "Sign in")) {
    return ScreenState.LoginScreen;
  }

  if (allNodes.find(n => n.text === "Enter your password")) {
    return ScreenState.LoginEnterPassword;
  }

  if (allNodes.find(n => n.text?.toLowerCase() === "home")) {
    return ScreenState.HomeScreen;
  }

  return ScreenState.Unknown;
}

2. ID-Based Detection

More reliable - use resource IDs which don't change with language:

ID-based detection

TypeScript

function detectScreenState(screen: AndroidNode): ScreenState {
  const allNodes = getAllNodes(screen);

  // Check for specific view IDs
  if (findNodesById(screen, "com.example:id/login_form").length) {
    return ScreenState.LoginScreen;
  }

  if (findNodesById(screen, "com.example:id/home_feed").length) {
    return ScreenState.HomeScreen;
  }

  if (findNodesById(screen, "android:id/aerr_close").find(n => n.clickable)) {
    return ScreenState.Crash;
  }

  return ScreenState.Unknown;
}

3. Composite Detection

Combine multiple conditions for accuracy:

Composite detection

TypeScript

function detectScreenState(screen: AndroidNode): ScreenState {
  const allNodes = getAllNodes(screen);

  // Login screen: has email field AND sign-in button
  const hasEmailField = allNodes.find(n =>
    n.className === "android.widget.EditText" &&
    (n.hintText?.toLowerCase()?.includes("email") || n.viewId?.includes("email"))
  );
  const hasSignInButton = allNodes.find(n =>
    n.clickable && n.text?.toLowerCase() === "sign in"
  );

  if (hasEmailField && hasSignInButton) {
    return ScreenState.LoginScreen;
  }

  // Password screen: has password field (isPassword = true)
  const hasPasswordField = allNodes.find(n =>
    n.className === "android.widget.EditText" && n.isPassword
  );

  if (hasPasswordField && !hasEmailField) {
    return ScreenState.LoginEnterPassword;
  }

  return ScreenState.Unknown;
}

4. Package Name Filtering

Filter by app package to avoid false positives from system UI:

Package name filtering

TypeScript

const APP_PACKAGE = "com.example.myapp";

function detectScreenState(screen: AndroidNode): ScreenState {
  const allNodes = getAllNodes(screen);

  // Only consider nodes from our target app
  const appNodes = allNodes.filter(n => n.packageName === APP_PACKAGE);

  // Check for phone dialog (different package)
  if (allNodes.every(n => n.packageName === "com.android.phone")) {
    return ScreenState.PhoneDialog;
  }

  // Check for system crash dialog
  if (allNodes.find(n =>
    n.viewId === "android:id/aerr_close" && n.packageName === "android"
  )) {
    return ScreenState.Crash;
  }

  // Now check app-specific screens
  if (appNodes.find(n => n.viewId?.includes("login"))) {
    return ScreenState.LoginScreen;
  }

  return ScreenState.Unknown;
}

Complete Detection Function

detection.ts

TypeScript

import { ScreenState } from "./screenStates.js";

const APP_PACKAGE = "com.mysocial.app";

export function detectScreenState(screen: AndroidNode): ScreenState {
  const allNodes = getAllNodes(screen);

  // === SYSTEM STATES (check first) ===

  // Crash dialog
  if (findNodesById(screen, "android:id/aerr_close").find(n => n.clickable)) {
    return ScreenState.Crash;
  }

  // Phone dialog
  if (allNodes.every(n => n.packageName === "com.android.phone")) {
    return ScreenState.PhoneDialog;
  }

  // No internet
  if (allNodes.find(n =>
    n.text?.toLowerCase()?.includes("no internet") ||
    n.text?.toLowerCase()?.includes("you're offline")
  )) {
    return ScreenState.NoInternet;
  }

  // Loading (only loading indicator visible)
  if (allNodes.length <= 3 && allNodes.find(n =>
    n.className?.includes("ProgressBar")
  )) {
    return ScreenState.Loading;
  }

  // === APP-SPECIFIC STATES ===

  // Login screen
  if (allNodes.find(n =>
    n.className === "android.widget.EditText" &&
    n.hintText?.toLowerCase()?.includes("email") &&
    n.packageName === APP_PACKAGE
  )) {
    return ScreenState.LoginScreen;
  }

  // Password entry
  if (allNodes.find(n =>
    n.isPassword && n.packageName === APP_PACKAGE
  )) {
    return ScreenState.LoginEnterPassword;
  }

  // Home screen (has bottom navigation with Home selected)
  const homeTab = allNodes.find(n =>
    n.description === "Home" &&
    n.isSelected &&
    n.packageName === APP_PACKAGE
  );
  if (homeTab) {
    return ScreenState.HomeScreen;
  }

  // Profile screen
  if (allNodes.find(n =>
    n.description === "Profile" &&
    n.isSelected &&
    n.packageName === APP_PACKAGE
  )) {
    return ScreenState.ProfileScreen;
  }

  // Permission dialog
  if (allNodes.find(n =>
    n.packageName === "com.android.permissioncontroller" ||
    (n.text?.toLowerCase()?.includes("allow") && n.clickable)
  )) {
    return ScreenState.PermissionDialog;
  }

  return ScreenState.Unknown;
}

Handling Unknown States

Unknown screens will happen. Handle them gracefully:

Unknown state handling

TypeScript

let unknownScreenCount = 0;
const MAX_UNKNOWN_SCREENS = 3;

async function getCurrentScreenState(): Promise<{
  state: ScreenState;
  screen: AndroidNode;
}> {
  let screen = await agent.actions.screenContent();
  let state = detectScreenState(screen);

  // Retry a few times if unknown
  if (state === ScreenState.Unknown) {
    for (let i = 0; i < 3; i++) {
      await sleep(2000);
      screen = await agent.actions.screenContent();
      state = detectScreenState(screen);

      if (state !== ScreenState.Unknown) {
        unknownScreenCount = 0;
        break;
      }
    }
  }

  // Track consecutive unknown screens
  if (state === ScreenState.Unknown) {
    unknownScreenCount++;
    console.warn(`Unknown screen #${unknownScreenCount}`);

    // Store for debugging
    await agent.utils.outOfSteps.storeScreen(
      screen,
      "unknown",
      "Unknown",
      MAX_UNKNOWN_SCREENS - unknownScreenCount,
      ScreenshotRecord.HIGH_QUALITY
    );

    if (unknownScreenCount >= MAX_UNKNOWN_SCREENS) {
      throw new Error("Too many unknown screens");
    }
  } else {
    unknownScreenCount = 0;
  }

  return { state, screen };
}

Stage-Specific Detection

Different stages may need different detection logic:

Stage-aware detection

TypeScript

import { Stage } from "./stages.js";
import { ScreenState } from "./screenStates.js";

export function detectScreenState(
  screen: AndroidNode,
  currentStage: Stage
): ScreenState {
  // Always check system states first
  const systemState = detectSystemStates(screen);
  if (systemState !== ScreenState.Unknown) {
    return systemState;
  }

  // Stage-specific detection
  switch (currentStage) {
    case Stage.Login:
      return detectLoginStates(screen);

    case Stage.NavigateToMessages:
    case Stage.ProcessMessages:
      return detectMessageStates(screen);

    case Stage.NavigateToFeed:
    case Stage.LikePosts:
      return detectFeedStates(screen);

    default:
      return detectCommonStates(screen);
  }
}

function detectSystemStates(screen: AndroidNode): ScreenState {
  // ... crash, phone dialog, no internet
}

function detectLoginStates(screen: AndroidNode): ScreenState {
  // ... login-specific screens
}

function detectMessageStates(screen: AndroidNode): ScreenState {
  // ... message-specific screens
}

Best Practices

Check System States First

Always check for crashes, dialogs, and system UI before app-specific states. These can appear at any time and need immediate handling.

Use Multiple Conditions

Don't rely on a single text match. Combine multiple checks (text + ID + class + package) for reliable detection.

Order Matters

Check more specific states before general ones. For example, check "LoginEnterPassword" before "LoginScreen".

Filter by Package Name

Use package name filtering to avoid false positives from system UI, notifications, or other apps.

Store Unknown Screens

Use agent.utils.outOfSteps.storeScreen() for unknown states. This helps debug what screens you missed.

Automation API

Screen States

The ScreenState Pattern

Detection Strategies

1. Text-Based Detection

2. ID-Based Detection

3. Composite Detection

4. Package Name Filtering

Complete Detection Function

Handling Unknown States

Stage-Specific Detection

Best Practices

Check System States First

Use Multiple Conditions

Order Matters

Filter by Package Name

Store Unknown Screens

Next Steps

Stages →

Error Handling →