Skip to content

Commit 2585b1f

Browse files
gurvinder-dhillongurvinder-dhillon
committed
added page assert tool and improved prompt, added structured output to agent
1 parent 77440a6 commit 2585b1f

File tree

6 files changed

+189
-53
lines changed

6 files changed

+189
-53
lines changed

‎src/auto.ts‎

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { test as base } from '@playwright/test';
2+
import { z } from "zod";
23
import { AutoConfig } from './types';
34
import { sessionManager, context } from './browser';
45
import { createReactAgent } from '@langchain/langgraph/prebuilt';
@@ -21,8 +22,16 @@ import {
2122
browser_choose_file,
2223
browser_go_forward,
2324
browser_assert,
25+
browser_page_assert,
2426
} from './tools';
2527

28+
// Define response schema
29+
const AutoResponseSchema = z.object({
30+
action: z.string().describe("The type of action performed (assert, click, type, etc)"),
31+
error: z.string().describe("Error message if any, empty string if none"),
32+
output: z.string().describe("Raw output from the action")
33+
});
34+
2635
// Extend base test to automatically track page
2736
export const test = base.extend({
2837
page: async ({ page }, use) => {
@@ -51,8 +60,15 @@ const initializeAgent = () => {
5160
- For pressing keys, use the pressKey tool
5261
- For saving PDFs, use the savePDF tool
5362
- For choosing files, use the chooseFile tool
54-
- For verification and assertions, use the assert tool
55-
Return the operation result or content as requested.`;
63+
- While calling the verification and assertion tools, DO NOT assume or make up any expected values. Use the values as provided in the instruction only.
64+
- For verification and assertions like {"isVisible", "hasText", "isEnabled", "isChecked"}, use the browser_assert tool
65+
- For page assertions like {page title, current page url} use the browser_page_assert tools
66+
Return a stringified JSON object with exactly these fields:
67+
{
68+
"action": "<type of action performed>",
69+
"error": "<error message or empty string>",
70+
"output": "<your output message>"
71+
}`;
5672

5773
const agent = createReactAgent({
5874
llm: model,
@@ -73,8 +89,18 @@ const initializeAgent = () => {
7389
browser_choose_file,
7490
browser_assert,
7591
browser_go_forward,
92+
browser_page_assert,
7693
],
7794
stateModifier: prompt,
95+
responseFormat: {
96+
prompt: `Return a stringified JSON object with exactly these fields:
97+
{
98+
"action": "<type of action performed>",
99+
"error": "<error message or empty string>",
100+
"output": "<your output message>"
101+
}`,
102+
schema: AutoResponseSchema
103+
}
78104
});
79105

80106
return { agent };
@@ -87,13 +113,17 @@ export async function auto(
87113
): Promise<any> {
88114
console.log(`[Auto] Processing instruction: "${instruction}"`);
89115

90-
if (config?.page) {
116+
if (config?.page)
117+
{
91118
sessionManager.setPage(config.page);
92119
console.log(`[Auto] Page set from config`);
93-
} else {
94-
try {
120+
} else
121+
{
122+
try
123+
{
95124
sessionManager.getPage();
96-
} catch {
125+
} catch
126+
{
97127
// In standalone mode, create a new page
98128
console.log(`[Auto] No existing page, creating new page`);
99129
await context.createPage();
@@ -107,14 +137,16 @@ export async function auto(
107137
messages: [new HumanMessage(instruction)],
108138
});
109139

110-
console.log('Agent result:', result);
140+
//console.log('Agent result:', result);
111141
// Process agent result
112142
const response = result.messages?.[-1]?.content;
113143
console.log(`[Auto] Agent response:`, response);
114144

115-
if (typeof response === 'string') {
145+
if (typeof response === 'string')
146+
{
116147
// If it's a success message, return null to match original behavior
117-
if (response.startsWith('Successfully')) {
148+
if (response.startsWith('Successfully'))
149+
{
118150
console.log(`[Auto] Detected success message, returning null`);
119151
return null;
120152
}

‎src/tools/browser_assert.ts‎

Lines changed: 44 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import { tool } from '@langchain/core/tools';
22
import { z } from 'zod';
3+
import { test, expect } from '@playwright/test';
34
import { runAndWait } from './utils';
45
import { context } from '../browser/context';
5-
import { expect } from '@playwright/test';
66

77
/**
88
* Schema for assertions with descriptions for the AI model
@@ -27,53 +27,66 @@ const assertSchema = z.object({
2727

2828
export const browser_assert = tool(
2929
async ({ element, ref, assertion, expected }) => {
30-
try {
30+
try
31+
{
3132
console.log(`[Assert Tool] Starting operation:`, {
3233
element,
3334
ref,
3435
assertion,
3536
expected,
3637
});
3738

38-
const result = await runAndWait(
39-
context,
40-
`Asserted "${element}" ${assertion}${expected ? ` equals "${expected}"` : ''}`,
39+
const result = await test.step(
40+
`Assert "${element}" ${assertion}${expected ? ` equals "${expected}"` : ''}`,
4141
async () => {
42-
const locator = context.refLocator(ref);
43-
console.log(`[Assert Tool] Performing assertion`);
42+
return await runAndWait(
43+
context,
44+
`Asserted "${element}" ${assertion}${expected ? ` equals "${expected}"` : ''}`,
45+
async () => {
46+
const locator = context.refLocator(ref);
47+
console.log(`[Assert Tool] Performing assertion`);
4448

45-
switch (assertion.toLowerCase()) {
46-
case 'isvisible':
47-
await expect(locator).toBeVisible();
48-
return 'Element is visible';
49-
case 'hastext':
50-
if (!expected)
51-
throw new Error(
52-
'Expected value required for hasText assertion',
53-
);
54-
await expect(locator).toHaveText(expected);
55-
return `Element has text "${expected}"`;
56-
case 'isenabled':
57-
await expect(locator).toBeEnabled();
58-
return 'Element is enabled';
59-
case 'ischecked':
60-
await expect(locator).toBeChecked();
61-
return 'Element is checked';
62-
default:
63-
throw new Error(
64-
`Unsupported assertion type: ${assertion}`,
65-
);
66-
}
49+
// Create descriptive message for both success and error cases
50+
const message = `${element} should ${assertion}${expected ? ` with text "${expected}"` : ''}`;
51+
52+
switch (assertion.toLowerCase())
53+
{
54+
case 'isvisible':
55+
await expect(locator, message).toBeVisible();
56+
return message;
57+
case 'hastext':
58+
if (!expected)
59+
throw new Error(
60+
'Expected value required for hasText assertion',
61+
);
62+
await expect(locator, message).toHaveText(expected);
63+
return message;
64+
case 'isenabled':
65+
await expect(locator, message).toBeEnabled();
66+
return message;
67+
case 'ischecked':
68+
await expect(locator, message).toBeChecked();
69+
return message;
70+
default:
71+
throw new Error(
72+
`Unsupported assertion type: ${assertion}`,
73+
);
74+
}
75+
},
76+
true,
77+
);
6778
},
68-
true,
6979
);
7080

7181
console.log(`[Assert Tool] Operation completed`);
7282
return result;
73-
} catch (error) {
83+
} catch (error)
84+
{
85+
// Simple error handling using Playwright's built-in error messages
7486
const errorMessage = `Assertion failed: ${error instanceof Error ? error.message : 'Unknown error'}`;
7587
console.error(`[Assert Tool] Error:`, errorMessage);
7688
return errorMessage;
89+
7790
}
7891
},
7992
{

‎src/tools/browser_page_assert.ts‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import { tool } from '@langchain/core/tools';
2+
import { z } from 'zod';
3+
import { test, expect } from '@playwright/test';
4+
import { runAndWait } from './utils';
5+
import { context } from '../browser/context';
6+
7+
/**
8+
* Schema for page-level assertions
9+
*/
10+
const pageAssertSchema = z.object({
11+
assertion: z
12+
.string()
13+
.describe('Type of assertion to perform (e.g., "hasTitle", "hasURL", "isOK")'),
14+
expected: z
15+
.string()
16+
.optional()
17+
.describe('Expected value for title/URL assertions'),
18+
});
19+
20+
export const browser_page_assert = tool(
21+
async ({ assertion, expected }) => {
22+
try
23+
{
24+
console.log(`[Page Assert Tool] Starting operation:`, {
25+
assertion,
26+
expected,
27+
});
28+
29+
const result = await test.step(
30+
`Assert page ${assertion}${expected ? ` equals "${expected}"` : ''}`,
31+
async () => {
32+
return await runAndWait(
33+
context,
34+
`Asserted page ${assertion}${expected ? ` equals "${expected}"` : ''}`,
35+
async () => {
36+
const page = context.existingPage();
37+
console.log(`[Page Assert Tool] Performing assertion`);
38+
39+
// Create descriptive message for both success and error cases
40+
const message = `Page should ${assertion}${expected ? ` "${expected}"` : ''}`;
41+
42+
switch (assertion.toLowerCase())
43+
{
44+
case 'hastitle':
45+
if (!expected)
46+
throw new Error(
47+
'Expected value required for hasTitle assertion',
48+
);
49+
await expect(page, message).toHaveTitle(expected);
50+
return message;
51+
case 'hasurl':
52+
if (!expected)
53+
throw new Error(
54+
'Expected value required for hasURL assertion',
55+
);
56+
await expect(page, message).toHaveURL(expected);
57+
return message;
58+
case 'isok': {
59+
// TODO: Implement response tracking in context
60+
throw new Error('Response assertions not yet implemented');
61+
}
62+
default:
63+
throw new Error(
64+
`Unsupported page assertion type: ${assertion}`,
65+
);
66+
}
67+
},
68+
true,
69+
);
70+
},
71+
);
72+
73+
console.log(`[Page Assert Tool] Operation completed`);
74+
return result;
75+
} catch (error)
76+
{
77+
const errorMessage = `Page assertion failed: ${error instanceof Error ? error.message : 'Unknown error'}`;
78+
console.error(`[Page Assert Tool] Error:`, errorMessage);
79+
return errorMessage;
80+
}
81+
},
82+
{
83+
name: 'page_assert',
84+
description: "Assert conditions on the page or response using Playwright's assertions",
85+
schema: pageAssertSchema
86+
}
87+
);

‎src/tools/browser_snapshot.ts‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { tool } from '@langchain/core/tools';
22
import { z } from 'zod';
33
import { test } from '@playwright/test';
44
import { context } from '../browser/context';
5-
import { captureAriaSnapshot } from './utils';
5+
import { run } from './utils';
66

77
/**
88
* Schema with dummy property to satisfy Gemini's API requirement for non-empty object properties
@@ -16,15 +16,20 @@ const snapshotSchema = z.object({
1616

1717
export const browser_snapshot = tool(
1818
async () => {
19-
try {
19+
try
20+
{
2021
console.log(`[Aria Snapshot] Starting snapshot operation`);
21-
const result =
22-
await test.step(`Capture Accessibility Snapshot`, async () => {
23-
return await captureAriaSnapshot(context);
22+
const result = await test.step(`Capture Accessibility Snapshot`, async () => {
23+
return await run(context, {
24+
callback: async () => { }, // Empty callback since we just want the snapshot
25+
captureSnapshot: true
2426
});
27+
});
28+
2529
console.log(`[Aria Snapshot] Operation completed successfully`);
2630
return result;
27-
} catch (error) {
31+
} catch (error)
32+
{
2833
const errorMessage = `Failed to capture snapshot: ${error instanceof Error ? error.message : 'Unknown error'}`;
2934
console.error(`[Aria Snapshot] Error:`, errorMessage);
3035
return errorMessage;

‎src/tools/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,4 @@ export { browser_save_pdf } from './browser_save_pdf';
1414
export { browser_choose_file } from './browser_choose_file';
1515
export { browser_assert } from './browser_assert';
1616
export { browser_go_forward } from './browser_go_forward';
17+
export { browser_page_assert } from './browser_page_assert';

‎src/tools/utils.ts‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -179,13 +179,11 @@ class PageSnapshot {
179179
`- Page URL: ${page.url()}`,
180180
`- Page Title: ${await page.title()}`
181181
);
182-
lines.push(
183-
`- Page Snapshot`,
184-
'```yaml',
185-
yamlDocument.toString().trim(),
186-
'```',
187-
''
188-
);
182+
lines.push(`- Page Snapshot`);
183+
yamlDocument.toString().trim().split('\n').forEach(line => {
184+
lines.push(` ${line}`); // 4-space indentation
185+
});
186+
lines.push('');
189187
this._text = lines.join('\n');
190188
}
191189

0 commit comments

Comments
 (0)