From dbdac364129bd83b2aafd9f867567b1dc48be20f Mon Sep 17 00:00:00 2001 From: rjambrecic <32619626+rjambrecic@users.noreply.github.com> Date: Tue, 21 May 2024 15:03:56 +0200 Subject: [PATCH 1/4] 692 retrying campaign creation team from scratch (#704) * Update python packages * Retry the team from scratch if something the team is constantly raising exceptions * Retry the team from scratch when max number of messages is reached * Add exponential backoff when retrying the team from scratch * Add exponential backoff to all conversation retries * If the team fails start the new team and add changes_made text to the end of the initial prompt * Update benchamrk with columns success_percentage, success_with_retry_percentage, failed_percentage * Save number of retries to the benchmark csv * Add latest benchmark results * Set AUTOGEN_USE_DOCKER to 'no' --- ...sk-list-2024-05-20T15:53:22-aggregated.csv | 5 ++ ...enchmark-task-list-2024-05-20T15:53:22.csv | 61 ++++++++++++++ ...sk-list-2024-05-20T23:25:19-aggregated.csv | 5 ++ ...enchmark-task-list-2024-05-20T23:25:19.csv | 61 ++++++++++++++ .../captn_agents/backend/benchmarking/base.py | 66 +++++++++------ .../benchmarking/brief_creation_team.py | 6 +- .../benchmarking/campaign_creation_team.py | 4 +- .../backend/benchmarking/end2end.py | 28 ++++--- .../backend/benchmarking/websurfer.py | 7 +- .../backend/teams/_brief_creation_team.py | 6 +- .../backend/teams/_campaign_creation_team.py | 9 +- .../backend/teams/_google_ads_team.py | 2 +- captn/captn_agents/backend/teams/_team.py | 41 ++++++++- .../backend/teams/_weekly_analysis_team.py | 2 +- .../tools/_campaign_creation_team_tools.py | 7 +- .../captn_agents/backend/tools/_functions.py | 1 + .../backend/tools/_google_ads_team_tools.py | 7 +- pyproject.toml | 10 +-- .../backend/benchmarking/test_benchmarking.py | 83 +++++++++++++++---- .../captn_agents/backend/teams/test_team.py | 67 ++++++++++++--- .../test_campaign_creation_team_tools.py | 8 +- 21 files changed, 395 insertions(+), 91 deletions(-) create mode 100644 benchmarking/campaign-creation-benchmark-task-list-2024-05-20T15:53:22-aggregated.csv create mode 100644 benchmarking/campaign-creation-benchmark-task-list-2024-05-20T15:53:22.csv create mode 100644 benchmarking/end2end-benchmark-task-list-2024-05-20T23:25:19-aggregated.csv create mode 100644 benchmarking/end2end-benchmark-task-list-2024-05-20T23:25:19.csv diff --git a/benchmarking/campaign-creation-benchmark-task-list-2024-05-20T15:53:22-aggregated.csv b/benchmarking/campaign-creation-benchmark-task-list-2024-05-20T15:53:22-aggregated.csv new file mode 100644 index 00000000..f3d7ce13 --- /dev/null +++ b/benchmarking/campaign-creation-benchmark-task-list-2024-05-20T15:53:22-aggregated.csv @@ -0,0 +1,5 @@ +url,success_rate,avg_time +https://faststream.airt.ai,1.0,784.6 +https://www.disneystore.eu,1.0,745.62 +https://www.ikea.com/gb/en/,1.0,904.52 +Total,1.0,811.58 diff --git a/benchmarking/campaign-creation-benchmark-task-list-2024-05-20T15:53:22.csv b/benchmarking/campaign-creation-benchmark-task-list-2024-05-20T15:53:22.csv new file mode 100644 index 00000000..65cc5feb --- /dev/null +++ b/benchmarking/campaign-creation-benchmark-task-list-2024-05-20T15:53:22.csv @@ -0,0 +1,61 @@ +,task,url,llm,execution_time,status,success,output +0,campaign_creation,https://www.ikea.com/gb/en/,gpt4,557.7981171607971,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign for the Furniture category, named 'IKEA Furniture Campaign', has been successfully created with an initial ad group focusing on sofas. Here are the details of the work completed:\n\n- **Campaign Name**: IKEA Furniture Campaign\n- **Daily Budget**: 3 EUR\n- **Status**: Enabled\n- **Targeting**: Google Search Network and Google Display Network\n- **Ad Group Name**: Sofas\n- **Ad Group ID**: [401](https://ads.google.com/aw/campaigns?campaignId=713&adGroupId=401)\n- **Ad Copy Headlines**: 'Comfy Sofas', 'Ergonomic Chairs', 'Stylish Tables', 'Spacious Storage', 'Modern Wardrobes', 'Work Desks', 'Outdoor Sets', and others\n- **Ad Copy Descriptions**: 'Find the perfect sofa', 'Chairs for any style', 'Tables for all spaces', 'Solve storage needs'\n- **Keywords**: 'Sofas', 'IKEA Sofas', 'Comfortable Sofas', 'Living Room Furniture', 'Quality Sofas'\n\nWould you like us to create additional ad groups for other furniture categories such as chairs, tables, and wardrobes?"",""smart_suggestions"":{""suggestions"":[""Create ad group for Chairs"",""Create ad group for Tables"",""Create ad group for Wardrobes""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +1,campaign_creation,https://www.disneystore.eu,gpt4,674.6791157722473,DONE,True,"Response from team '123_456': +{""message"":""The campaign \""Disney Spring Highlights 2023\"" has been successfully set up with the following details:\n\n- **Campaign ID**: [957](https://ads.google.com/aw/campaigns?campaignId=957)\n- **Daily Budget**: 3 EUR\n- **Currency**: EUR\n- **Status**: Enabled\n- **Targeting**: Google Search Network and Google Display Network\n\nTwo ad groups have been created focusing on the 'New In Products' and 'Easter Gifts & Toys' sections:\n\n1. **Ad Group for 'New In Products'**:\n - **Ad Group ID**: [503](https://ads.google.com/aw/adgroups?adGroupId=503)\n - **Final URL**: [New In Products](https://www.disneystore.eu/new)\n - **Headlines**: \""Latest Disney Arrivals\"", \""New Marvel Gear\"", \""Pixar's Newest Hits\"", etc.\n - **Descriptions**: \""Explore the latest Disney merchandise and collectibles. Shop now!\"", etc.\n - **Keywords**: \""New Arrivals\"", \""Disney\"", \""Marvel\"", etc.\n\n2. **Ad Group for 'Easter Gifts & Toys'**:\n - **Ad Group ID**: [720](https://ads.google.com/aw/adgroups?adGroupId=720)\n - **Final URL**: [Easter Gifts & Toys](https://www.disneystore.eu/easter?srule=newest)\n - **Headlines**: \""Easter Disney Fun\"", \""Springtime Surprises\"", \""Easter Toy Magic\"", etc.\n - **Descriptions**: \""Fill your Easter with Disney books and soft toys. Shop the joy!\"", etc.\n - **Keywords**: \""Easter\"", \""Disney Gifts\"", \""Toys\"", etc.\n\nThe task of creating the new Google Ads campaign is now complete. If you have any further requests or changes, please let us know. Would you like to review the campaign or request any modifications?"",""smart_suggestions"":{""suggestions"":[""Review campaign"",""Request modifications""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +2,campaign_creation,https://faststream.airt.ai,gpt4,681.3880639076233,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign **FastStream Features & Start** has been successfully created and is now live. Here are the details of the work accomplished:\n\n- Campaign ID: [381](https://ads.google.com/aw/campaigns?campaignId=381)\n- Ad Group ID: [975](https://ads.google.com/aw/adgroups?adGroupId=975)\n- Ad ID: [748](https://ads.google.com/aw/ads?adId=748)\n\nThe ad group **FastStream Features Group** includes ads and keywords specifically tailored to highlight the 'Features' and 'Getting Started' pages of your FastStream website, targeting developers and tech-savvy audiences interested in asynchronous web services.\n\nIf you would like to review the campaign or have any further requests, please let us know. We're here to assist you with any additional changes or optimizations you might need."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +3,campaign_creation,https://www.ikea.com/gb/en/,gpt4,826.8061668872833,DONE,True,"Response from team '123_456': +{""message"":""The Google Ads campaign for the Furniture category, titled 'IKEA Furniture Campaign', has been successfully set up with the following ad groups, each targeting a specific furniture product type:\n\n- **Sofas Ad Group**: Focused on various sofa options offered by IKEA. (Ad Group ID: 245)\n- **Chairs Ad Group**: Highlighting IKEA's range of ergonomic and stylish chairs. (Ad Group ID: 679)\n- **Tables Ad Group**: Showcasing the stylish and versatile tables available at IKEA. (Ad Group ID: 585)\n- **Wardrobes Ad Group**: Featuring modern and spacious wardrobe solutions from IKEA. (Ad Group ID: 789)\n\nEach ad group includes targeted ad copy and keywords designed to attract and direct customers to the relevant product pages on the IKEA website, with the goal of increasing visibility and sales. We have completed the task as per your brief. If you have any further requests or need additional adjustments, please let us know."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +4,campaign_creation,https://www.disneystore.eu,gpt4,602.4507279396057,DONE,True,"Response from team '123_456': +{""message"":""We have successfully created the new Google Ads campaign 'Disney Store EU - New & Easter' with a daily budget of 3 EUR. The campaign is designed to boost sales and increase brand awareness for the 'New In Products' and 'Easter Gifts & Toys' sections of the Disney Store EU website.\n\nHere are the details of the campaign and ad groups:\n\n- Campaign: [Disney Store EU - New & Easter](https://ads.google.com/aw/campaigns?campaignId=728) (ID: 728)\n- Ad Group for 'New In Products': [New In Products](https://ads.google.com/aw/adgroups?adGroupId=384) (ID: 384)\n- Ad Group for 'Easter Gifts & Toys': [Easter Gifts & Toys](https://ads.google.com/aw/adgroups?adGroupId=529) (ID: 529)\n\nEach ad group contains ads with relevant headlines and descriptions, as well as keywords that are set to match exactly with user search queries. The ads direct users to the specific sections of your website to showcase the latest products and seasonal offers.\n\nIf you have any further requests or need additional adjustments, please let us know. Would you like to review the campaign or request any changes?"",""smart_suggestions"":{""suggestions"":[""Review campaign details"",""Request campaign changes""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +5,campaign_creation,https://faststream.airt.ai,gpt4,1330.561810016632,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign **FastStream Features & Start** has been successfully created and is now live. Here are the details of the campaign and ad groups:\n\n- **Campaign Name:** FastStream Features & Start\n- **Daily Budget:** €3 (3,000,000 micros)\n- **Status:** Enabled\n\n**Ad Groups Created:**\n1. **FastStream Features**\n - Focus: 'Features' page\n - Ads and keywords highlight FastStream's capabilities and ease of use.\n2. **FastStream Getting Started**\n - Focus: 'Getting Started' page\n - Ads and keywords guide new users through setup and basic usage.\n\nThe campaign is designed to cater to developers and tech-savvy audiences, emphasizing the features and accessibility of FastStream. If you have any further requests or need to make changes, please start a new chat to discuss your requirements."",""smart_suggestions"":{""suggestions"":[""Review campaign performance"",""Request further optimizations"",""Inquire about additional services""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +6,campaign_creation,https://www.ikea.com/gb/en/,gpt4,1056.8997509479525,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign for the Furniture category, titled 'IKEA Furniture Campaign', has been successfully created with the following ad groups, ads, and keywords:\n\n- **Ad Group: Sofas**\n - Ads with headlines such as 'Comfortable & Stylish Sofas' and descriptions like 'Explore a wide range of sofas. Perfect for any home.'\n - Keywords: 'Sofas', 'Buy Sofas Online'\n\n- **Ad Group: Office Furniture**\n - Ads with headlines such as 'Ergonomic Office Furniture' and descriptions like 'Upgrade your home office with ergonomic chairs and desks.'\n - Keywords: 'Office Furniture', 'Ergonomic Office Chairs UK'\n\n- **Ad Group: Storage Solutions**\n - Ads with headlines such as 'Smart Storage Solutions' and descriptions like 'Maximize your space with IKEA's storage solutions.'\n - Keywords: 'Storage Solutions', 'Wardrobes'\n\n- **Ad Group: Outdoor Furniture**\n - Ads with headlines such as 'Chic Outdoor Furniture' and descriptions like 'Transform your garden with IKEA's outdoor furniture.'\n - Keywords: 'Outdoor Furniture', 'Garden Furniture'\n\n- **Ad Group: Modern & Affordable Furniture**\n - Ads with headlines such as 'Modern Furniture Finds' and descriptions like 'Modernize your home with affordable furniture from IKEA.'\n - Keywords: 'Modern Furniture', 'Affordable Furniture'\n\n- **Ad Group: Unique IKEA Offerings**\n - Ads with headlines such as 'Easy-to-Assemble Furniture' and descriptions like 'Discover the convenience of IKEA's flat-pack furniture.'\n - Keywords: 'Flat-pack Furniture', 'Self-assembly Furniture'\n\nEach ad group is tailored to target specific customer searches and drive traffic to the most relevant product pages on your website. The campaign is set to target the Google Search Network and is currently enabled with a daily budget of 3 EUR.\n\nIf you have any further requests or need additional adjustments, please let us know. Would you like to review the campaign performance metrics or have any other specific requests?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +7,campaign_creation,https://www.disneystore.eu,gpt4,699.6911101341248,DONE,True,"Response from team '123_456': +{""message"":""The Google Ads campaign **Disney Spring Highlights 2023** has been successfully set up with two focused ad groups targeting the 'New In Products' and 'Easter Gifts & Toys' sections of your website.\n\n- The **New In Products** ad group [308](https://ads.google.com/aw/adgroups?adGroupId=308) features ads with headlines like 'Latest Disney Finds' and 'New Marvel Gear', and keywords such as 'New Arrivals' and 'Exclusive Products'.\n- The **Easter Gifts & Toys** ad group [758](https://ads.google.com/aw/adgroups?adGroupId=758) includes ads with headlines such as 'Easter Disney Fun' and 'Springtime Surprises', and keywords like 'Easter' and 'Seasonal Offers'.\n\nThese ad groups are designed to highlight the latest merchandise and seasonal offers, driving traffic to the highlighted sections and aligning with your goals of boosting sales and increasing brand awareness.\n\nWould you like to review the campaign or need assistance with any further optimizations?"",""smart_suggestions"":{""suggestions"":[""Review campaign"",""Further optimizations""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +8,campaign_creation,https://faststream.airt.ai,gpt4,678.8802201747894,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign for FastStream has been successfully created and is now live. Here are the details of the campaign and ad group:\n\n- **Campaign Name**: FastStream Dev Essentials\n- **Campaign ID**: [589](https://ads.google.com/aw/campaigns?campaignId=589)\n- **Daily Budget**: 3 EUR\n- **Status**: Enabled\n- **Targeting**: Google Search Network and Google Display Network\n\nAn ad group named 'FastStream Features Ad Group' with ID [420](https://ads.google.com/aw/adgroups?adGroupId=420) has been set up to highlight the 'Features' and 'Getting Started' pages of FastStream. The ad group includes ads with headlines and descriptions focused on the key features and benefits of FastStream, targeting developers and tech-savvy audiences.\n\nWould you like to review the campaign performance after a certain period or is there anything else we can assist you with at this moment?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +9,campaign_creation,https://www.ikea.com/gb/en/,gpt4,352.10460209846497,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign for the Furniture category has been successfully created and is now live. Here are the details of the work accomplished:\n\n- **Campaign Name**: IKEA Furniture UK\n- **Campaign ID**: [268](https://ads.google.com/aw/campaigns?campaignId=268)\n- **Daily Budget**: 3 EUR\n- **Status**: Enabled\n- **Targeting**: Google Search Network and Google Display Network\n\nWithin this campaign, we have set up an ad group focusing on sofas with the following details:\n\n- **Ad Group Name**: Sofas\n- **Ad Group ID**: [183](https://ads.google.com/aw/adgroups?adGroupId=183)\n- **Final URL**: [IKEA Furniture](https://www.ikea.com/gb/en/cat/furniture-fu001/)\n\nThe ad contains a variety of headlines and descriptions highlighting different aspects of IKEA's furniture offerings, such as comfort, style, and affordability. We have also targeted specific keywords related to sofas to ensure the ads are shown to users with relevant interests.\n\nIf you would like to discuss further optimizations or have any additional requests, please let us know. Would you like to set up ad groups for other furniture product types as well?"",""smart_suggestions"":{""suggestions"":[""Create ad groups for other furniture types"",""Discuss further optimizations"",""No further actions needed""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +10,campaign_creation,https://www.disneystore.eu,gpt4,872.7700967788696,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign \""Disney Store EU - New & Seasonal\"" has been successfully set up with the following ad groups:\n\n1. **New In Products**\n- Ad Group ID: [948](https://ads.google.com/aw/adgroups?adGroupId=948)\n- Focused on showcasing the latest Disney merchandise.\n\n2. **Easter Gifts & Toys**\n- Ad Group ID: [384](https://ads.google.com/aw/adgroups?adGroupId=384)\n- Focused on promoting Easter-themed Disney gifts and toys.\n\nBoth ad groups feature tailored ad creatives and keywords to effectively target your audience and meet your campaign goals of boosting sales and increasing brand awareness.\n\nShould you require any further modifications or wish to discuss additional strategies, please feel free to reach out. Is there anything else we can assist you with today?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +11,campaign_creation,https://faststream.airt.ai,gpt4,989.7051463127136,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign 'FastStream Features & Start' has been successfully set up with two ad groups targeting the 'Features' and 'Getting Started' pages of your website.\n\n**Campaign Details:**\n- Campaign ID: [851](https://ads.google.com/aw/campaigns?campaignId=851)\n- Daily Budget: \n- Currency: EUR\n- Status: Enabled\n- Targeting: Google Search Network and Google Display Network\n\n**Ad Group: Features**\n- Ad Group ID: [348](https://ads.google.com/aw/adgroups?adGroupId=348)\n- Focus: Highlighting FastStream's features\n- Headlines and descriptions emphasize the unified API, Pydantic validation, automatic documentation, and more.\n- Keywords include 'FastStream', 'Asynchronous', 'Event Streams', and others tailored to attract developers and tech-savvy audiences.\n\n**Ad Group: Getting Started**\n- Ad Group ID: [292](https://ads.google.com/aw/adgroups?adGroupId=292)\n- Focus: Showcasing the ease of getting started with FastStream\n- Headlines and descriptions provide quick start guides, installation steps, and basic usage information.\n- Keywords such as 'Getting Started', 'Quick Start', 'Basic Usage', and more are used to guide new users.\n\nThis targeted approach is designed to boost sales by attracting and engaging the right audience. If you have any further requests or need additional adjustments, please let us know."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +12,campaign_creation,https://www.ikea.com/gb/en/,gpt4,1112.5606861114502,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign for IKEA UK, focusing on the Furniture category, has been successfully created and launched. Here's a summary of the accomplished work:\n\n- **Campaign**: A new campaign named 'IKEA Furniture Campaign' with a daily budget of 3 EUR, targeting both the Google Search Network and Google Display Network.\n- **Ad Groups**: Specific ad groups for 'Sofas', 'Chairs', 'Tables', 'Wardrobes', and 'Storage' have been created, each with tailored ad copy and keywords to target the right audience.\n- **Keywords**: Relevant keywords have been added to each ad group to ensure maximum visibility and reach for IKEA's furniture products.\n- **Ad Copy**: Compelling ad copy has been developed, highlighting the unique features and benefits of IKEA's furniture products.\n\nWe will continue to monitor and optimize the campaign to drive traffic, increase conversions, and boost sales of furniture products. Regular performance reports will be shared with you to keep you informed about the campaign's progress.\n\nIs there anything else you would like to review or any additional actions you would like us to take regarding the campaign?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +13,campaign_creation,https://www.disneystore.eu,gpt4,705.7390820980072,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign for Disney Store EU has been successfully set up with the following details:\n\n- **Campaign Name**: Disney Store EU - New & Seasonal\n- **Daily Budget**: 3 EUR\n- **Ad Groups**:\n - **New In Products**: Focused on the latest Disney merchandise with relevant keywords and compelling ad copy.\n - **Easter Gifts & Toys**: Tailored to Easter-themed Disney gifts and toys with targeted keywords and engaging ad creatives.\n\nEach ad group contains a set of ads with headlines and descriptions designed to attract and engage your target audience, driving traffic to the highlighted sections of your website.\n\nThe task of creating the digital campaign is now complete. If you have any further requests or need additional changes, please let us know.\n\nWould you like to review the campaign or request any further modifications?"",""smart_suggestions"":{""suggestions"":[""Review campaign details"",""Request further modifications"",""Confirm completion""],""type"":""oneOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +14,campaign_creation,https://faststream.airt.ai,gpt4,834.3745429515839,DONE,True,"Response from team '123_456': +{""message"":""The Google Ads campaign 'FastStream Async Features & Guide' has been successfully created and is now live. Here are the details of the campaign and ad groups:\n\n- **Campaign ID**: [822](https://ads.google.com/aw/campaigns?campaignId=822)\n- **Ad Group 'FastStream Features Highlight' ID**: [312](https://ads.google.com/aw/adgroups?adGroupId=312)\n - Focuses on the unique features of FastStream.\n - Ads and keywords are tailored to highlight technical capabilities and integrations.\n- **Ad Group 'FastStream Quick Start Guide' ID**: [228](https://ads.google.com/aw/adgroups?adGroupId=228)\n - Emphasizes the ease of getting started with FastStream.\n - Ads and keywords are designed to guide new users through setup and initial use.\n\nBoth ad groups utilize the maximum number of headlines and descriptions, ensuring broad coverage and multiple angles for ad delivery.\n\nIf you have any further requests or need additional adjustments, please let us know. Would you like to review the performance of the campaign after a certain period, or is there anything else we can assist you with at this moment?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +15,campaign_creation,https://www.ikea.com/gb/en/,gpt4,306.390771150589,DONE,True,"Response from team '123_456': +{""message"":""The 'IKEA Furniture Campaign' has been successfully launched with an initial focus on sofas. We've created an ad group named 'IKEA Sofas' that targets users searching for sofas, with compelling ad copy and precise keywords to direct them to the IKEA furniture section.\n\nWould you like us to proceed with creating additional ad groups for other furniture product types such as chairs, tables, and wardrobes? We will follow the same process of developing targeted ad copy and compiling relevant keywords for each category."",""smart_suggestions"":{""suggestions"":[""Create ad groups for chairs"",""Create ad groups for tables"",""Create ad groups for wardrobes""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +16,campaign_creation,https://www.disneystore.eu,gpt4,579.6121151447296,DONE,True,"Response from team '123_456': +{""message"":""The ad groups \""New In Products\"" and \""Easter Gifts & Toys\"" have been successfully created with their respective ads and keywords. Here is a detailed summary of the work accomplished:\n\n- **Campaign Created**: \""Disney Store Spring Highlights\"" with a daily budget of 3 EUR.\n - Campaign ID: [161](https://ads.google.com/aw/campaigns?campaignId=161)\n\n- **Ad Group for 'New In Products'**:\n - Ad Group ID: [507](https://ads.google.com/aw/adgroups?adGroupId=507)\n - Final URL: [New In Products](https://www.disneystore.eu/new)\n - Headlines and descriptions crafted to showcase the latest Disney merchandise.\n - Keywords such as 'New Arrivals', 'Disney', 'Marvel', 'Pixar', 'Star Wars', 'Latest Merchandise', and 'Exclusive Products' were added.\n\n- **Ad Group for 'Easter Gifts & Toys'**:\n - Ad Group ID: [880](https://ads.google.com/aw/adgroups?adGroupId=880)\n - Final URL: [Easter Gifts & Toys](https://www.disneystore.eu/easter?srule=newest)\n - Headlines and descriptions designed to promote Easter-themed Disney gifts and toys.\n - Keywords like 'Easter', 'Disney Gifts', 'Toys', 'Books', 'Soft Toys', 'Seasonal Offers', and 'Themed Merchandise' were included.\n\nThis completes the task of creating a new Google Ads campaign for the Disney Store EU website. If you have any further requests or changes, please let us know.\n\nWould you like to review the campaign or request any modifications?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +17,campaign_creation,https://faststream.airt.ai,gpt4,1085.2395811080933,DONE,True,"Response from team '123_456': +{""message"":""The Google Ads campaign \""FastStream Features & Getting Started\"" has been successfully created and is now live. Here are the details of the campaign and ad groups:\n\n- Campaign: [FastStream Features & Getting Started](https://ads.google.com/aw/campaigns?campaignId=769)\n - Daily Budget: \n - Targeting: Google Search Network and Google Display Network\n\n- Ad Groups:\n - [FastStream Features](https://ads.google.com/aw/adgroups?adGroupId=218): Focused on highlighting the advanced features of FastStream.\n - [FastStream Getting Started](https://ads.google.com/aw/adgroups?adGroupId=195): Aimed at new users for quick and easy setup.\n\nEach ad group contains tailored ads and keywords to attract developers and tech-savvy audiences interested in asynchronous web services and event streaming.\n\nWould you like to review the campaign performance after a certain period, or is there anything else we can assist you with?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +18,campaign_creation,https://www.ikea.com/gb/en/,gpt4,567.9836637973785,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign for the Furniture category, titled **IKEA Furniture Campaign**, has been successfully created and set up with the following ad groups and corresponding details:\n\n- **Sofas Ad Group** (ID: 609): Targeting keywords such as 'Sofas', 'Modern Sofas', and 'Comfortable Sofas'. The ad copy highlights the comfort and style of IKEA's sofa range.\n\n- **Chairs Ad Group** (ID: 833): Focused on keywords like 'Chairs', 'Office Chairs', and 'Dining Chairs'. The ad copy emphasizes the ergonomic design and variety of IKEA's chairs.\n\n- **Tables Ad Group** (ID: 796): Includes keywords such as 'Tables', 'Dining Tables', and 'Coffee Tables'. The ad copy showcases the stylish and functional aspects of IKEA's tables.\n\n- **Wardrobes Ad Group** (ID: 150): With keywords like 'Wardrobes', 'Bedroom Wardrobes', and 'Modern Wardrobes'. The ad copy features the spaciousness and modern design of IKEA's wardrobe solutions.\n\nEach ad group is designed to drive traffic to the furniture product pages on the IKEA website and increase conversions and sales of furniture products. If you have any further requests or need adjustments, please let us know, and we can assist you accordingly. Would you like to review the campaign or request any changes?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +19,campaign_creation,https://www.disneystore.eu,gpt4,851.106076002121,DONE,True,"Response from team '123_456': +{""message"":""Dear client,\n\nWe have successfully launched the \""Disney Spring Highlights 2023\"" campaign, focusing on the 'New In Products' and 'Easter Gifts & Toys' sections of your website. Here are the details:\n\n**Campaign**: Disney Spring Highlights 2023\n- Campaign ID: [830](https://ads.google.com/aw/campaigns?campaignId=830)\n- Daily Budget: 3 EUR\n- Targeting: Google Search Network and Google Display Network\n\n**Ad Groups**:\n1. New In Products\n - Ad Group ID: [779](https://ads.google.com/aw/adgroups?adGroupId=779)\n - Ad ID: [588](https://ads.google.com/aw/ads?adId=588)\n - Keywords: 'New Arrivals', 'Disney', 'Marvel', 'Pixar', 'Star Wars', 'Latest Merchandise', 'Exclusive Products'\n\n2. Easter Gifts & Toys\n - Ad Group ID: [318](https://ads.google.com/aw/adgroups?adGroupId=318)\n - Ad ID: [127](https://ads.google.com/aw/ads?adId=127)\n - Keywords: 'Easter', 'Disney Gifts', 'Toys', 'Books', 'Soft Toys', 'Seasonal Offers', 'Themed Merchandise'\n\nWe have crafted compelling ad creatives with headlines and descriptions that align with your brand and the seasonal themes. The keywords have been carefully selected to target users searching for the latest Disney merchandise and Easter gifts.\n\nWould you like to review the campaign or need assistance with any further optimizations?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +20,campaign_creation,https://faststream.airt.ai,gpt4,642.9695279598236,DONE,True,"Response from team '123_456': +{""message"":""The Google Ads campaign **FastStream Features & Start** has been successfully created and is now live. Here are the details of the work accomplished:\n\n- Campaign: [FastStream Features & Start](https://ads.google.com/aw/campaigns?campaignId=710) with a daily budget of \n3 EUR, targeting both the Google Search Network and Google Display Network.\n\n- Ad Group: [FastStream Features](https://ads.google.com/aw/adgroups?adGroupId=684) focusing on the features of FastStream, with relevant ads and keywords to attract developers and tech-savvy audiences.\n\n- Ad Group: [FastStream Getting Started](https://ads.google.com/aw/adgroups?adGroupId=461) designed to guide new users on how to get started with FastStream, also with tailored ads and keywords.\n\nThe ads are set to run with a variety of headlines and descriptions that highlight the key aspects of FastStream, ensuring maximum visibility and engagement with the target audience.\n\nIf you have any further requests or need additional adjustments, please let us know. Would you like to review the campaign performance after a certain period, or is there anything else we can assist you with right now?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +21,campaign_creation,https://www.ikea.com/gb/en/,gpt4,2361.7059779167175,DONE,True,"Response from team '123_456': +{""message"":""The Google Ads campaign for IKEA UK's Furniture category has been successfully created and launched. The campaign includes ad groups for Sofas, Chairs, Tables, Wardrobes, and Storage Furniture, each with custom ads and keywords to effectively target potential customers.\n\nWe will continue to monitor the campaign's performance and optimize as needed to ensure the best results. If you have any further requests or need additional changes, please feel free to reach out.\n\nThank you for choosing our services for your digital marketing needs."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +22,campaign_creation,https://www.disneystore.eu,gpt4,767.4054679870605,DONE,True,"Response from team '123_456': +{""message"":""The task of creating a new Google Ads campaign for the Disney Store EU website has been successfully completed. Here's a summary of the accomplished work:\n\n- A new campaign titled \""Disney Store EU - New & Seasonal\"" with the ID [727](https://ads.google.com/aw/campaigns?campaignId=727) has been created, focusing on boosting sales and increasing brand awareness for the 'New In Products' and 'Easter Gifts & Toys' sections.\n\n- Two ad groups have been set up within the campaign:\n - \""New In Products\"" ad group with the ID [384](https://ads.google.com/aw/adgroups?adGroupId=384), targeting the latest Disney merchandise with relevant keywords and compelling ad copy.\n - \""Easter Gifts & Toys\"" ad group with the ID [777](https://ads.google.com/aw/adgroups?adGroupId=777), promoting Easter-themed Disney gifts and toys with carefully selected keywords and engaging ad copy.\n\n- Ads and keywords for both ad groups have been created, ensuring that the ad headlines and descriptions are within the character limits and aligned with the client's digital marketing objectives.\n\nThis campaign is designed to highlight the 'New In Products' section with the latest merchandise and promote 'Easter Gifts & Toys' for seasonal offers, as per the client's brief.\n\nDo you have any additional requests or would you like to review the campaign details further?"",""smart_suggestions"":{""suggestions"":[""Review campaign details"",""Discuss further optimizations"",""Start a new task""],""type"":""oneOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +23,campaign_creation,https://faststream.airt.ai,gpt4,436.32006192207336,DONE,True,"Response from team '123_456': +{""message"":""The new ad group 'FastStream Features Ad Group' has been successfully created within the 'FastStream Features & Start' campaign, focusing on the 'Features' page of the FastStream website. The ad group includes ads with headlines and descriptions tailored to highlight the key features of FastStream, and a set of keywords that are relevant to the content of the 'Features' page. The ads are now live and will help to attract developers and tech-savvy audiences interested in asynchronous web services and event streaming.\n\nWould you like to proceed with setting up another ad group for the 'Getting Started' page, or is there anything else you would like to add or modify in the current campaign setup?"",""smart_suggestions"":{""suggestions"":[""Set up 'Getting Started' ad group"",""Review current campaign setup"",""Add more keywords"",""Update ad creatives""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +24,campaign_creation,https://www.ikea.com/gb/en/,gpt4,348.04639506340027,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign for the Furniture category has been successfully created with the following details:\n\n- Campaign Name: IKEA Furniture Campaign\n- Campaign ID: [669](https://ads.google.com/aw/campaigns?campaignId=669)\n\nAdditionally, we have set up an ad group focusing on sofas with relevant keywords and ad copy:\n\n- Ad Group Name: Sofas\n- Ad Group ID: [331](https://ads.google.com/aw/adgroups?adGroupId=331)\n- Final URL: [IKEA Furniture](https://www.ikea.com/gb/en/cat/furniture-fu001/)\n- Headlines and descriptions crafted to highlight IKEA's furniture offerings\n- Keywords targeting customers interested in sofas\n\nWe will continue to monitor the campaign's performance and make necessary optimizations. If you have any further requests or need adjustments, please let us know."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +25,campaign_creation,https://www.disneystore.eu,gpt4,634.7162029743195,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign **Disney Store EU - Spring Specials** has been successfully created with a focus on the 'New In Products' and 'Easter Gifts & Toys' sections of your website. Here are the details of the work accomplished:\n\n- **Campaign**: [Disney Store EU - Spring Specials](https://ads.google.com/aw/campaigns?campaignId=164) with a daily budget of 3 EUR.\n- **Ad Group 1**: [New In Products](https://ads.google.com/aw/adgroups?adGroupId=261) with targeted ads and keywords for the latest Disney merchandise.\n- **Ad Group 2**: [Easter Gifts & Toys](https://ads.google.com/aw/adgroups?adGroupId=496) with ads and keywords for Easter-themed Disney gifts and toys.\n\nBoth ad groups aim to boost sales and increase brand awareness according to your goals. If you wish to make any changes or have further requests, please let us know or start a new chat by clicking on \""New chat\""."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +26,campaign_creation,https://faststream.airt.ai,gpt4,632.0155169963837,DONE,True,"Response from team '123_456': +{""message"":""The ad group \""FastStream Features\"" and its associated ads and keywords have been successfully created within the \""FastStream Features & Start\"" campaign. Here is a summary of the work accomplished:\n\n- **Campaign**: \""FastStream Features & Start\"" with a daily budget of \n3, targeting the Google Search Network and Google Display Network. [718](https://ads.google.com/aw/campaigns?campaignId=718)\n- **Ad Group**: \""FastStream Features\"" focused on the 'Features' page of FastStream. [977](https://ads.google.com/aw/adgroups?adGroupId=977)\n- **Ads**: Multiple headlines and descriptions highlighting the key features of FastStream, such as unified API, Pydantic validation, automatic documentation, and more. [217](https://ads.google.com/aw/ads?adId=217)\n- **Keywords**: A selection of 15 keywords including 'FastStream', 'Asynchronous Web Services', 'Event Streams API', and others, all set to exact match to target the most relevant audience.\n\nThis setup aims to attract developers and tech-savvy users interested in asynchronous web services and event streaming, by emphasizing the capabilities, integrations, and ease of use of FastStream.\n\nWould you like to review the campaign or proceed with any additional steps?"",""smart_suggestions"":{""suggestions"":[""Review the campaign"",""Proceed with additional steps""],""type"":""oneOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +27,campaign_creation,https://www.ikea.com/gb/en/,gpt4,1554.8801400661469,DONE,True,"Response from team '123_456': +{""message"":""The new Google Ads campaign for the Furniture category, named \""IKEA Furniture Campaign,\"" has been successfully created and includes ad groups for Sofas, Chairs, Tables, Wardrobes, Storage Solutions, Office Furniture, and Outdoor Furniture. Each ad group features tailored ad copy and keywords to effectively target potential customers and drive traffic to the relevant product pages on the IKEA UK website.\n\nThe campaign is designed to boost sales and increase brand awareness for IKEA UK's furniture products. If you have any further requests or wish to make updates, please start a new chat to address them. Thank you for choosing our services, and we look forward to contributing to the success of your digital marketing efforts."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +28,campaign_creation,https://www.disneystore.eu,gpt4,1068.017175912857,DONE,True,"Response from team '123_456': +{""message"":""The ad groups \""New In Products\"" and \""Easter Gifts & Toys\"" have been successfully created within the \""Disney Store EU - New & Seasonal\"" campaign.\n\nHere is a summary of the work accomplished:\n\n- Campaign: [Disney Store EU - New & Seasonal](https://ads.google.com/aw/campaigns?campaignId=881)\n - Daily budget: 3 EUR\n - Targeting: Google Search Network and Google Display Network\n\n- Ad Group: [New In Products](https://ads.google.com/aw/adgroups?adGroupId=812)\n - Final URL: [New In Products](https://www.disneystore.eu/new)\n - Headlines and descriptions focused on the latest Disney merchandise.\n - Keywords: New Arrivals, Disney, Marvel, Pixar, Star Wars, Latest Merchandise, Exclusive Products\n\n- Ad Group: [Easter Gifts & Toys](https://ads.google.com/aw/adgroups?adGroupId=729)\n - Final URL: [Easter Gifts & Toys](https://www.disneystore.eu/easter?srule=newest)\n - Headlines and descriptions promoting Easter-themed Disney gifts and toys.\n - Keywords: Easter, Disney Gifts, Toys, Books, Soft Toys, Seasonal Offers, Themed Merchandise\n\nThis completes the task of creating the new Google Ads campaign. If you have any further requests or need additional changes, please let us know.\n\nWould you like to review the campaign or request any further modifications?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +29,campaign_creation,https://faststream.airt.ai,gpt4,534.5563180446625,DONE,True,"Response from team '123_456': +{""message"":""The new ad group 'FastStream Features & Start' has been successfully created within the 'FastStream Features & Start' campaign. It includes ads with headlines and descriptions tailored to highlight the 'Features' and 'Getting Started' pages of FastStream, and a set of keywords specifically chosen to target developers and tech-savvy audiences interested in asynchronous web services and event streaming.\n\nHere is a summary of the accomplished work:\n\n- Campaign: [FastStream Features & Start](https://ads.google.com/aw/campaigns?campaignId=595) (ID: 595)\n- Ad Group: [FastStream Features & Start](https://ads.google.com/aw/adgroups?adGroupId=693) (ID: 693)\n- Ads and Keywords: Created with specific focus on the client's objectives, with each keyword set to 'EXACT' match type for precise targeting.\n\nIf you would like to review the ad creatives or keywords, or if you have any further requests, please let us know. Would you like to make any adjustments or proceed with monitoring the campaign's performance?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" diff --git a/benchmarking/end2end-benchmark-task-list-2024-05-20T23:25:19-aggregated.csv b/benchmarking/end2end-benchmark-task-list-2024-05-20T23:25:19-aggregated.csv new file mode 100644 index 00000000..4c619a83 --- /dev/null +++ b/benchmarking/end2end-benchmark-task-list-2024-05-20T23:25:19-aggregated.csv @@ -0,0 +1,5 @@ +url,success_rate,avg_time +https://faststream.airt.ai,1.0,911.87 +https://www.disneystore.eu,1.0,872.92 +https://www.ikea.com/gb/en/,1.0,2492.2 +Total,1.0,1425.66 diff --git a/benchmarking/end2end-benchmark-task-list-2024-05-20T23:25:19.csv b/benchmarking/end2end-benchmark-task-list-2024-05-20T23:25:19.csv new file mode 100644 index 00000000..1c7f0d93 --- /dev/null +++ b/benchmarking/end2end-benchmark-task-list-2024-05-20T23:25:19.csv @@ -0,0 +1,61 @@ +,task,url,llm,execution_time,status,success,output +0,end2end,https://www.ikea.com/gb/en/,gpt4,948.0930650234222,DONE,True,"Response from team '123_234': +{""message"":""The campaign and all ad groups for IKEA have been successfully set up. Here is a summary of the work accomplished:\n\n- **Campaign Created**: \""IKEA Home Essentials\""\n - Daily Budget: 3 EUR\n - Targeting: Google Search Network and Google Display Network\n - Focus: Promoting IKEA's Furniture, Beds & Mattresses, and Offers\n\n- **Ad Groups Created**:\n 1. \""IKEA Furniture Favorites\""\n - Target Page: [Furniture](https://www.ikea.com/gb/en/cat/furniture-fu001/)\n - Headlines and descriptions focused on various furniture items\n - Keywords: Furniture, Home Decor, Living Room, etc.\n\n 2. \""IKEA Sleep Solutions\""\n - Target Page: [Beds & Mattresses](https://www.ikea.com/gb/en/cat/beds-mattresses-bm001/)\n - Headlines and descriptions focused on beds and mattresses\n - Keywords: Beds, Mattresses, Sleep, etc.\n\n 3. \""IKEA Special Offers\""\n - Target Page: [Offers](https://www.ikea.com/gb/en/offers/)\n - Headlines and descriptions focused on current deals and promotions\n - Keywords: Offers, Discounts, Deals, etc.\n\nThis campaign is designed to increase brand awareness and boost sales by promoting specific product categories and emphasizing IKEA's unique selling points.\n\nIf you wish to make any changes or updates to the campaign, please feel free to start a new chat with us. We look forward to continuing to support your advertising efforts.\n\nWould you like to review the campaign details or have any other requests?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +1,end2end,https://www.disneystore.eu,gpt4,1110.4882988929749,DONE,True,"Response from team '123_234': +{""message"":""The new Google Ads campaign for Disney Store EU, titled **Disney Store EU - New Collections & Offers**, has been successfully created and is now live. The campaign is structured to increase brand awareness and boost sales, with each ad group targeting a specific section of your website to highlight the unique product offerings and promotions.\n\nHere is a brief overview of the campaign setup:\n\n- **Campaign ID**: [156](https://ads.google.com/aw/campaigns?campaignId=156)\n- **Daily Budget**: 3 EUR\n- **Ad Groups**:\n - New In Products ([157](https://ads.google.com/aw/campaigns?campaignId=157))\n - Offers ([780](https://ads.google.com/aw/campaigns?campaignId=780))\n - Uniquely Disney ([691](https://ads.google.com/aw/campaigns?campaignId=691))\n - Characters & Movies ([869](https://ads.google.com/aw/campaigns?campaignId=869))\n - The Little Mermaid 2023 ([594](https://ads.google.com/aw/campaigns?campaignId=594))\n - Ahsoka ([836](https://ads.google.com/aw/campaigns?campaignId=836))\n\nEach ad group features ads with headlines and descriptions tailored to the corresponding website section, along with carefully selected keywords to maximize visibility and engagement.\n\nShould you have any further requests or require adjustments, please feel free to start a new chat with us. We're here to help you succeed with your digital advertising efforts!"",""smart_suggestions"":{""suggestions"":[""Review campaign performance"",""Request further adjustments"",""Start a new chat for additional services""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +2,end2end,https://faststream.airt.ai,gpt4,574.6588299274445,DONE,True,"Response from team '123_234': +{""message"":""The Google Ads campaign for FastStream, titled 'FastStream Async Advantage', has been successfully created with a daily budget of 3 EUR. Additionally, an ad group named 'FastStream Features' has been set up, complete with targeted ads and keywords that highlight the framework's capabilities and advantages.\n\nHere are the details of the work accomplished:\n\n- Campaign: [FastStream Async Advantage](https://ads.google.com/aw/campaigns?campaignId=446)\n- Ad Group: [FastStream Features](https://ads.google.com/aw/adgroups?adGroupId=465)\n\nThe ad group includes ads with headlines such as 'Async Services', 'Unified API', and 'Pydantic Support', and descriptions like 'Build async services fast' and 'Work with multiple brokers'. The keywords selected are directly related to FastStream's features, such as 'Open-source', 'Asynchronous', and 'Code Generation'.\n\nIf you wish to review the campaign or make further adjustments, please let us know. Would you like to discuss any additional digital marketing strategies or optimizations for FastStream?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +3,end2end,https://www.ikea.com/gb/en/,gpt4,975.6940648555756,DONE,True,"Response from team '123_234': +{""message"":""The **IKEA Home Essentials** campaign has been successfully launched with a focus on furniture. We've created an ad group named **IKEA Furniture**, which includes targeted ads and keywords that highlight key products such as sofas, tables, beds, wardrobes, and chairs. The campaign is now live, aiming to increase brand awareness and boost sales for IKEA's furniture category.\n\nWould you like us to proceed with creating additional ad groups for other product categories such as kitchens, smart home devices, and outdoor living?"",""smart_suggestions"":{""suggestions"":[""Create ad groups for kitchens"",""Create ad groups for smart home devices"",""Create ad groups for outdoor living""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +4,end2end,https://www.disneystore.eu,gpt4,1023.163146018982,DONE,True,"Response from team '123_234': +{""message"":""The new Google Ads campaign for Disney Store EU, titled 'Disney Store EU - New Arrivals & Offers', has been successfully created with a daily budget of 3 EUR. The campaign includes three ad groups targeting different sections of your website:\n\n1. **New In Products** - Showcasing the latest merchandise with ad group ID 509.\n2. **Uniquely Disney** - Featuring unique Disney-themed products with ad group ID 501.\n3. **Disney Offers** - Highlighting current promotions and discounts with ad group ID 247.\n\nEach ad group contains tailored ads and keywords to effectively target potential customers. Would you like to review the campaign or proceed with any additional requests?"",""smart_suggestions"":{""suggestions"":[""Review campaign details"",""Make additional requests""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +5,end2end,https://faststream.airt.ai,gpt4,881.5403771400452,DONE,True,"Response from team '123_234': +{""message"":""The Google Ads campaign 'FastStream Awareness' and the ad group 'FastStream Main Features' have been successfully created. Here are the details of the work accomplished:\n\n- **Campaign**: [FastStream Awareness](https://ads.google.com/aw/campaigns?campaignId=292) (ID: 292)\n- **Ad Group**: FastStream Main Features (ID: 515)\n- **Ads and Keywords**: An ad with multiple headlines and descriptions has been created, focusing on FastStream's key features such as 'Pydantic Validation', 'Automatic API Documentation', and 'FastAPI Compatibility'. Relevant keywords have been added to target the appropriate audience.\n\nPlease review the campaign and let us know if there are any adjustments you would like to make or if we can assist you further."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +6,end2end,https://www.ikea.com/gb/en/,gpt4,554.6447670459747,DONE,True,"Response from team '123_234': +{""message"":""The Google Ads campaign for IKEA UK has been successfully set up with the following details:\n\n- Campaign Name: IKEA UK Brand Awareness\n- Campaign ID: [838](https://ads.google.com/aw/campaigns?campaignId=838)\n\n**Ad Groups Created:**\n1. Shop by Room\n - Ad Group ID: [364](https://ads.google.com/aw/adgroups?adGroupId=364)\n - Ad ID: [174](https://ads.google.com/aw/ads?adId=174)\n - Keywords: Furniture, Homeware, Bedroom, Living Room, Kitchen, Bathroom, Modern Design, Home Decor\n\n2. Offers\n - Ad Group ID: [768](https://ads.google.com/aw/adgroups?adGroupId=768)\n - Ad ID: [278](https://ads.google.com/aw/ads?adId=278)\n - Keywords: Discounts, Deals, Offers, Savings, Furniture Sale, Clearance, Limited Time, Special Prices\n\nPlease review the campaign and ad groups. If you require any further adjustments or have additional requests, let us know how we can assist you further."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +7,end2end,https://www.disneystore.eu,gpt4,639.5262379646301,DONE,True,"Response from team '123_234': +{""message"":""The new Google Ads campaign for Disney Store EU has been successfully set up with the following components:\n\n- **Campaign Name**: Disney Store EU - New Products & Offers\n - **Daily Budget**: 3 EUR\n - **Status**: Enabled\n - **Targeting**: Google Search Network and Google Display Network\n - **Focus**: Promoting new products, movies & TV shows, and special offers\n\n- **Ad Groups**:\n - **New In Products**\n - **Final URL**: [New In Products](https://www.disneystore.eu/new)\n - **Headlines and Descriptions**: Tailored to the latest Disney merchandise\n - **Keywords**: New Arrivals, Disney Merchandise, Latest Toys, etc.\n - **Movies & TV Show**\n - **Final URL**: [Movies & TV Show](https://www.disneystore.eu/new/movies-and-tv-show)\n - **Headlines and Descriptions**: Related to popular and upcoming movies and TV shows\n - **Keywords**: Disney Movies, TV Show Merch, Film Toys, etc.\n - **Offers**\n - **Final URL**: [Offers](https://www.disneystore.eu/offers)\n - **Headlines and Descriptions**: Highlighting current promotions and special deals\n - **Keywords**: Disney Offers, Promotions, Special Deals, etc.\n\nThe campaign is designed to increase brand awareness and boost sales by utilizing the latest product collections and current promotions. If you would like to review the campaign or if there are any other specific requests you have, please let us know."",""smart_suggestions"":{""suggestions"":[""Review campaign details"",""Request further changes""],""type"":""oneOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +8,end2end,https://faststream.airt.ai,gpt4,616.9481389522552,DONE,True,"Response from team '123_234': +{""message"":""The Google Ads campaign for FastStream has been successfully created with the following details:\n\n- Campaign Name: FastStream Awareness Campaign\n- Daily Budget: 3 EUR\n- Status: Enabled\n- Targeting: Google Search Network\n- Campaign ID: [988](https://ads.google.com/aw/campaigns?campaignId=988)\n\nWithin this campaign, an ad group named 'FastStream Main Features' has been set up, focusing on the unique selling points of FastStream. The ad group includes ads with headlines and descriptions that highlight FastStream's features and direct users to the FastStream website. Relevant keywords have been added to target developers and companies interested in asynchronous web services and event stream handling.\n\n- Ad Group Name: FastStream Main Features\n- Ad Group ID: [312](https://ads.google.com/aw/adgroups?adGroupId=312)\n\nIf you have any further requests or need additional modifications, please let us know. Would you like to review the campaign performance after a certain period or proceed with any other actions?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +9,end2end,https://www.ikea.com/gb/en/,gpt4,945.9722440242768,DONE,True,"Response from team '123_234': +{""message"":""The ad group 'IKEA Furniture' has been successfully created within the 'IKEA Brand Awareness & Sales Boost' campaign. It includes tailored ads and keywords specifically designed to showcase IKEA's furniture offerings.\n\nAd Group ID: 663\nAd ID: 610\nKeywords: Sofas (981), Chairs (722), Tables (292), Storage (429), Office Furniture (634), Bedroom Sets (996), Living Room Essentials (442), Dining Furniture (920), Children's Furniture (753), Modern Designs (190), Traditional Pieces (817), Space-Saving (228), Functional (209), Stylish (693), Affordable (503).\n\nEach keyword has been set to 'EXACT' match type to target users precisely searching for these terms. The ad copy includes engaging headlines and descriptions that reflect IKEA's brand image and values, emphasizing quality, affordability, and design.\n\nWould you like to proceed with creating additional ad groups for other product categories such as beds & mattresses, outdoor living, and more?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +10,end2end,https://www.disneystore.eu,gpt4,599.8097238540649,DONE,True,"Response from team '123_234': +{""message"":""The Google Ads campaign for The Disney Store EU has been successfully set up with the following components:\n\n- **Campaign**: Disney Store EU - Brand Awareness ([Campaign ID: 898](https://ads.google.com/aw/campaigns?campaignId=898))\n\n- **Ad Groups**:\n - New In Products ([Ad Group ID: 909](https://ads.google.com/aw/adgroups?adGroupId=909))\n - Uniquely Disney ([Ad Group ID: 491](https://ads.google.com/aw/adgroups?adGroupId=491))\n - Movies & TV Show Merchandise ([Ad Group ID: 571](https://ads.google.com/aw/adgroups?adGroupId=571))\n\nEach ad group contains tailored ads and keywords to target potential customers effectively. If you would like to make any changes or have further instructions, please let us know."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +11,end2end,https://faststream.airt.ai,gpt4,1769.260269165039,DONE,True,"Response from team '123_234': +{""message"":""The Google Ads campaign for FastStream has been successfully set up with the following components:\n\n- **Campaign**: \""FastStream Awareness & Sales\"" with a daily budget of 3 EUR, targeting both the Google Search Network and Google Display Network, with the aim of increasing brand awareness and boosting sales.\n\n- **Ad Groups**:\n - \""FastStream Features\"": Focused on the framework's unique features such as the unified API, Pydantic validation, and automatic documentation.\n - \""Getting Started with FastStream\"": Provides a quick start guide and basic usage instructions for new users.\n - \""FastStream Code API Reference\"": Details the comprehensive code API of FastStream, including classes, functions, parameters, and attributes.\n\nEach ad group contains a set of targeted headlines, descriptions, and keywords designed to attract the right audience and achieve your digital marketing objectives.\n\nIf you require any further changes, updates, or have additional requests, please let us know. Would you like to review the campaign details or need assistance with anything else?"",""smart_suggestions"":{""suggestions"":[""Review campaign details"",""Discuss further changes"",""Start a new chat for additional requests""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +12,end2end,https://www.ikea.com/gb/en/,gpt4,1579.4613099098206,DONE,True,"Response from team '123_234': +{""message"":""The new Google Ads campaign for IKEA UK, titled 'IKEA UK Home & Business Solutions', has been successfully created with a focus on offers, furniture, kitchens, and business solutions. Each ad group contains targeted ads and keywords designed to increase brand awareness and boost sales.\n\nHere are the details of the campaign and ad groups:\n\n- **Campaign**: [IKEA UK Home & Business Solutions](https://ads.google.com/aw/campaigns?campaignId=948)\n- **Ad Group - Offers**: [837](https://ads.google.com/aw/adgroups?adGroupId=837)\n- **Ad Group - Furniture**: [849](https://ads.google.com/aw/adgroups?adGroupId=849)\n- **Ad Group - Kitchens**: [254](https://ads.google.com/aw/adgroups?adGroupId=254)\n- **Ad Group - IKEA for Business**: [227](https://ads.google.com/aw/adgroups?adGroupId=227)\n\nWould you like to review the campaign or need assistance with any further adjustments?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +13,end2end,https://www.disneystore.eu,gpt4,644.4932479858398,DONE,True,"Response from team '123_234': +{""message"":""The new Google Ads campaign 'Disney Store EU - Exclusive Merchandise' and the ad group 'Disney New Arrivals' have been successfully created. The campaign is designed to increase brand awareness and boost sales by focusing on the latest and exclusive merchandise available on your website.\n\nHere are the details of the campaign and ad group:\n\n- **Campaign ID**: [818](https://ads.google.com/aw/campaigns?campaignId=818)\n- **Ad Group ID**: [770](https://ads.google.com/aw/adgroups?adGroupId=770)\n\nThe ad group includes targeted ads with headlines and descriptions that highlight the variety of products and special offers, along with keywords that are relevant to your latest merchandise.\n\nWould you like to review the campaign and ad group, or is there anything else you would like to add or modify?"",""smart_suggestions"":{""suggestions"":[""Review campaign details"",""Add more ad groups"",""Adjust campaign budget"",""Update ad copy""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +14,end2end,https://faststream.airt.ai,gpt4,722.7999060153961,DONE,True,"Response from team '123_234': +{""message"":""Dear FastStream team,\n\nWe are pleased to inform you that your Google Ads campaign \""FastStream Awareness & Sales\"" has been successfully launched. Here are the details of the campaign and ad groups:\n\n- **Campaign Name:** FastStream Awareness & Sales\n - **Daily Budget:** 3 EUR\n - **Targeting:** Google Search Network and Google Display Network\n - **Campaign ID:** 265\n\n- **Ad Groups:**\n 1. **Features Ad Group**\n - **Ad Group ID:** 222\n - **Ad ID:** 831\n - **Keywords:** Open-source, Asynchronous, Event Streams, Unified API, Message Brokers, Pydantic Validation, Automatic Documentation, Dependency Injection, Code Generation\n 2. **Kafka Routing Ad Group**\n - **Ad Group ID:** 448\n - **Ad ID:** 593\n - **Keywords:** Kafka, High-throughput, Data Streaming, Publish-Subscribe, Topics, Partitions, Real-time Pipelines\n 3. **RabbitMQ Routing Ad Group**\n - **Ad Group ID:** 853\n - **Ad ID:** 246\n - **Keywords:** RabbitMQ, Message Routing, Flexible Scenarios, Message Prioritization, Complex Logic, Performance\n\nEach ad group has been tailored to highlight the key features and integrations of FastStream, with a focus on increasing brand awareness and boosting sales. We have utilized the maximum number of headlines and descriptions to ensure a wide range of messaging options.\n\nShould you have any further requests or need to make updates, please feel free to start a new chat with us by clicking on the \""New chat\"" button.\n\nBest regards,\nYour Digital Agency Team"",""smart_suggestions"":{""suggestions"":[""Start a new chat for updates"",""Review campaign performance"",""Request additional changes""],""type"":""oneOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +15,end2end,https://www.ikea.com/gb/en/,gpt4,1742.563416004181,DONE,True,"Response from team '123_234': +{""message"":""The Google Ads campaign 'IKEA UK Brand Awareness' has been successfully created and set up with multiple ad groups targeting different categories relevant to IKEA's offerings. Each ad group contains a tailored ad and a set of keywords designed to attract and direct users to specific sections of the IKEA UK website, enhancing brand awareness and boosting sales.\n\nHere is a summary of the ad groups created:\n\n1. **IKEA Offers - Discounts**: Targeting special promotions and discounts.\n2. **IKEA Furniture**: Focused on IKEA's variety of furniture.\n3. **IKEA Kitchens**: Highlighting kitchen solutions.\n4. **IKEA Bedroom**: Promoting bedroom furnishings.\n5. **IKEA Living Room**: Featuring living room furniture and ideas.\n6. **IKEA Smart Home**: Concentrating on smart home devices.\n7. **IKEA New Products**: Showcasing the latest products and collections.\n8. **IKEA for Business**: Catering to business solutions and products.\n\nWould you like to review the details of each ad group or proceed with any additional actions?"",""smart_suggestions"":{""suggestions"":[""Review ad group details"",""Proceed with additional actions""],""type"":""oneOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +16,end2end,https://www.disneystore.eu,gpt4,689.3955490589142,DONE,True,"Response from team '123_234': +{""message"":""The new Google Ads campaign for Disney Store EU has been successfully created with a focus on increasing brand awareness and boosting sales. Here are the details of the campaign and ad groups:\n\n- **Campaign**: Disney Store EU Campaign\n - 366\n - Daily Budget: 3 EUR\n - Status: Enabled\n - Targeting: Google Search Network and Google Display Network\n\n- **Ad Groups**:\n 1. New In Products\n - 520\n 2. Offers\n - 723\n 3. The Little Mermaid 2023\n - 533\n\nEach ad group contains tailored ads and keywords to target the specified areas of your website, ensuring maximum visibility for your latest products, special offers, and The Little Mermaid 2023 merchandise.\n\nIf you have any further requests or need additional changes, please let us know. Would you like to review the campaign or start another project?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +17,end2end,https://faststream.airt.ai,gpt4,788.272088766098,DONE,True,"Response from team '123_234': +{""message"":""The new Google Ads campaign \""FastStream Brand Awareness\"" and its associated ad groups have been successfully created and are now live. Here are the details:\n\n- **Campaign Name:** FastStream Brand Awareness\n- **Daily Budget:** 3 EUR\n- **Currency:** EUR\n- **Status:** Enabled\n- **Targeting:** Google Search Network and Google Display Network\n- **Campaign Focus:** Promoting FastStream's unique features and capabilities\n- **Campaign Link:** [817](https://ads.google.com/aw/campaigns?campaignId=817)\n\n**Ad Groups Created:**\n1. **FastStream Features** - Focusing on the core features of FastStream.\n - Ad Group Link: [480](https://ads.google.com/aw/adgroups?adGroupId=480)\n2. **Kafka Routing** - Highlighting Kafka routing capabilities within FastStream.\n - Ad Group Link: [710](https://ads.google.com/aw/adgroups?adGroupId=710)\n3. **RabbitMQ Routing** - Covering RabbitMQ routing and advantages in FastStream.\n - Ad Group Link: [222](https://ads.google.com/aw/adgroups?adGroupId=222)\n\nEach ad group contains carefully crafted headlines and descriptions to resonate with the target audience and drive the campaign's objectives of increasing brand awareness and boosting sales.\n\nIf you have any further requests or need additional changes, please let us know. Otherwise, we consider this task completed. Thank you for choosing our services, and we look forward to contributing to the success of FastStream."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +18,end2end,https://www.ikea.com/gb/en/,gpt4,1595.799940109253,DONE,True,"Response from team '123_234': +{""message"":""Dear client,\n\nWe have successfully created the \""IKEA Home Furnishing Campaign\"" and the following ad groups with targeted ads and keywords:\n\n1. \""IKEA Offers\"" Ad Group - [ID 382](https://ads.google.com/aw/campaigns?campaignId=382)\n2. \""IKEA Furniture\"" Ad Group - [ID 405](https://ads.google.com/aw/campaigns?campaignId=405)\n3. \""IKEA Kitchens\"" Ad Group - [ID 752](https://ads.google.com/aw/campaigns?campaignId=752)\n4. \""IKEA Smart Home\"" Ad Group - [ID 692](https://ads.google.com/aw/campaigns?campaignId=692)\n5. \""IKEA for Business\"" Ad Group - [ID 918](https://ads.google.com/aw/campaigns?campaignId=918)\n\nEach ad group is tailored to the respective product category, with relevant headlines, descriptions, and keywords to attract and engage potential customers.\n\nPlease review the campaign and ad groups at your convenience. If you have any further requests or need additional changes, feel free to reach out.\n\nWould you like to review the performance metrics or explore additional advertising strategies?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +19,end2end,https://www.disneystore.eu,gpt4,1534.3525519371033,DONE,True,"Response from team '123_234': +{""message"":""The Google Ads campaign and associated ad groups for Disney Store EU have been successfully created. Here are the details:\n\n- **Campaign**: Disney Store EU Awareness\n - **Budget**: 3 EUR per day\n - **Status**: Enabled\n - **Targeting**: Google Search Network and Google Display Network\n - **Campaign ID**: [925](https://ads.google.com/aw/campaigns?campaignId=925)\n\n- **Ad Groups**:\n 1. **New In Products**\n - **Ad Group ID**: [559](https://ads.google.com/aw/adgroups?adGroupId=559)\n - **Final URL**: [https://www.disneystore.eu/new](https://www.disneystore.eu/new)\n - **Headlines and Descriptions**: Focused on the latest Disney merchandise.\n - **Keywords**: New Arrivals, Latest Disney, Marvel, Pixar, Star Wars, New Releases\n\n 2. **Uniquely Disney Collection**\n - **Ad Group ID**: [494](https://ads.google.com/aw/adgroups?adGroupId=494)\n - **Final URL**: [https://www.disneystore.eu/new/uniquely-disney](https://www.disneystore.eu/new/uniquely-disney)\n - **Headlines and Descriptions**: Highlighting exclusive Disney-themed items.\n - **Keywords**: Exclusive Disney, Unique Merchandise, Disney Mugs, Disney Accessories, Disney Toys\n\n 3. **Stitch Attacks Snacks Collection**\n - **Ad Group ID**: [913](https://ads.google.com/aw/adgroups?adGroupId=913)\n - **Final URL**: [https://www.disneystore.eu/toys-and-fancy-dress/collectibles/stitch-attacks-snacks](https://www.disneystore.eu/toys-and-fancy-dress/collectibles/stitch-attacks-snacks)\n - **Headlines and Descriptions**: Featuring Stitch-themed collectible series.\n - **Keywords**: Stitch Collectibles, Disney Snacks, Stitch Toys, Collectible Series, Stitch Merchandise\n\nThis campaign is designed to increase brand awareness and boost sales, focusing on unique selling points such as exclusive 'Uniquely Disney' products and special collections like 'Stitch Attacks Snacks'.\n\nIf you have any further requests or need additional changes, please let us know."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +20,end2end,https://faststream.airt.ai,gpt4,828.155965089798,DONE,True,"Response from team '123_234': +{""message"":""The Google Ads campaign for FastStream has been successfully set up with the following details:\n\n- Campaign Name: FastStream Brand Awareness & Sales\n- Campaign ID: [541](https://ads.google.com/aw/campaigns?campaignId=541)\n\n**Ad Groups Created:**\n1. FastStream Kafka Integration\n - Ad Group ID: [113](https://ads.google.com/aw/adgroups?adGroupId=113)\n - Keywords: Kafka, Routing, Publish-Subscribe, Topics, Partitions, High-Throughput, Real-Time\n2. FastStream RabbitMQ Support\n - Ad Group ID: [543](https://ads.google.com/aw/adgroups?adGroupId=543)\n - Keywords: RabbitMQ, Routing, Message Queues, Flexible Routing, Performance, Complex Scenarios\n3. FastStream NATS Messaging\n - Ad Group ID: [288](https://ads.google.com/aw/adgroups?adGroupId=288)\n - Keywords: NATS, High-Performance, Message Broker, JetStream, RPC, Scalability, Golang\n4. FastStream Redis Broker\n - Ad Group ID: [449](https://ads.google.com/aw/adgroups?adGroupId=449)\n - Keywords: Redis, In-Memory Store, Message Broker, Pub/Sub, List, Streams, Data Structures\n\nEach ad group has targeted ads with headlines and descriptions that highlight the unique benefits of FastStream with the respective message broker. The campaign is now live and ready to increase brand awareness and boost sales for FastStream.\n\nIf you have any further requests or would like to make any adjustments, please let us know."",""smart_suggestions"":{""suggestions"":[""Review campaign performance"",""Adjust campaign budget"",""Create additional ad groups"",""Request further optimizations""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +21,end2end,https://www.ikea.com/gb/en/,gpt4,15159.108757019045,DONE,True,"Response from team '123_234': +{""message"":""The ad group **IKEA UK - Offers** has been successfully created, focusing on IKEA's current promotions and deals. It features tailored ads and keywords designed to highlight the various sale-related themes.\n\nWould you like to proceed with creating the next ad group for Furniture, or do you have any specific instructions or changes for the upcoming ad groups?"",""smart_suggestions"":{""suggestions"":[""Proceed with Furniture ad group"",""Review Offers ad group"",""Provide instructions for upcoming ad groups""],""type"":""oneOf""},""is_question"":true,""status"":""pause"",""terminate_groupchat"":true}" +22,end2end,https://www.disneystore.eu,gpt4,716.4462411403656,DONE,True,"Response from team '123_234': +{""message"":""The Google Ads campaign **Disney Store EU - New Arrivals & Favorites** has been successfully set up with the following ad groups, ads, and keywords:\n\n- **New In Products**\n - Ad Group ID: 302\n - Ad ID: 328\n - Keywords: New Arrivals, Latest Collection, Disney Merchandise, Marvel Gear, Pixar Toys, Star Wars Accessories\n\n- **Official Disney Toys**\n - Ad Group ID: 792\n - Ad ID: 561\n - Keywords: Disney Toys, Soft Toys, Action Figures, Dolls, Play Sets, Board Games, Nursery Toys, Pretend Play\n\n- **Mugs, Cups & Bottles**\n - Ad Group ID: 171\n - Ad ID: 499\n - Keywords: Disney Mugs, Cups, Water Bottles, Kitchenware, Cooking, Dining, Disney Kitchen, Drinkware\n\nEach ad group is designed to highlight the unique selling points of the Disney Store EU and attract potential customers. If you would like to review the ad copy or make any adjustments, please let us know. What would you like to do next?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +23,end2end,https://faststream.airt.ai,gpt4,1148.346246957779,DONE,True,"Response from team '123_234': +{""message"":""The 'FastStream Awareness' Google Ads campaign has been successfully created and includes the following ad groups, each focusing on different aspects of FastStream's features and integrations:\n\n1. **FastStream Features** - Highlighting the core features of FastStream.\n2. **FastStream Tutorial** - Focused on getting started guides and tutorials.\n3. **FastStream Kafka Integration** - Showcasing the integration with Kafka.\n4. **FastStream RabbitMQ Integration** - Emphasizing the benefits of using RabbitMQ.\n5. **FastStream NATS Integration** - Presenting the high-performance integration with NATS.\n6. **FastStream Redis Integration** - Highlighting the advantages of Redis as a broker.\n\nEach ad group contains a set of targeted ads and keywords designed to attract and engage the relevant audience for FastStream. The campaign is now live, and we anticipate it will increase brand awareness and boost sales for FastStream as intended.\n\nDo you have any further requests or adjustments needed for the campaign?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +24,end2end,https://www.ikea.com/gb/en/,gpt4,431.60040497779846,DONE,True,"Response from team '123_234': +{""message"":""The new Google Ads campaign for IKEA UK, titled **IKEA UK Brand & Sales Boost**, has been successfully created and is now live. The campaign includes an ad group focused on the living room category, with targeted keywords and ad copy designed to highlight IKEA's range of affordable and stylish furniture.\n\nYou can view and manage the campaign [here](https://ads.google.com/aw/campaigns?campaignId=578) and the ad group [here](https://ads.google.com/aw/adgroups?adGroupId=844).\n\nPlease review the campaign at your convenience. If you have any further requests or need additional changes, feel free to reach out. Would you like to review the campaign details or discuss additional marketing strategies?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +25,end2end,https://www.disneystore.eu,gpt4,377.00586318969727,DONE,True,"Response from team '123_234': +{""message"":""The new Google Ads campaign for Disney Store EU has been successfully created with the following details:\n\n- Campaign Name: Disney Store EU New Arrivals\n- Daily Budget: 3 EUR\n- Status: Enabled\n- Targeting: Google Search Network and Google Display Network\n\nAdditionally, an ad group named 'New In Products' has been set up with ads and keywords targeting the latest Disney, Marvel, Pixar, and Star Wars products. The ad group is now live and will start showing ads to potential customers.\n\nIf you have any further requests or need additional changes, please let us know. Would you like to create more ad groups for the other relevant pages such as Disney Offers and Stitch Attacks Snacks Collection?"",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +26,end2end,https://faststream.airt.ai,gpt4,745.618319272995,DONE,True,"Response from team '123_234': +{""message"":""The Google Ads campaign for FastStream, titled 'FastStream Features Campaign', has been successfully set up with a focus on increasing brand awareness and boosting sales by highlighting key features of FastStream.\n\n**Campaign Details**:\n- Campaign Name: FastStream Features Campaign\n- Daily Budget: 3 EUR\n- Status: Enabled\n- Targeting: Google Search Network and Google Display Network\n\n**Ad Groups Created**:\n1. FastStream Getting Started\n - [Ad Group ID: 360](https://ads.google.com/aw/campaigns?campaignId=360)\n - [Ad ID: 286](https://ads.google.com/aw/campaigns?campaignId=286)\n - Keywords: FastStream Tutorial, Getting Started, Setup Guide, Installation Instructions, Basic Usage\n\n2. FastStream Kafka Routing\n - [Ad Group ID: 964](https://ads.google.com/aw/campaigns?campaignId=964)\n - [Ad ID: 687](https://ads.google.com/aw/campaigns?campaignId=687)\n - Keywords: Kafka Integration, FastStream Kafka Support, Event Streaming, Kafka Routing, Message Brokers\n\nEach ad group contains tailored ads and keywords to target potential customers interested in specific features of FastStream. The campaign is now live. Please review the campaign and ad groups, and let us know if there are any further adjustments you would like to make or if you have any additional requests."",""smart_suggestions"":{""suggestions"":[""Review campaign and ad groups"",""Request further adjustments"",""Make additional requests""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +27,end2end,https://www.ikea.com/gb/en/,gpt4,989.093908071518,DONE,True,"Response from team '123_234': +{""message"":""The new Google Ads campaign for IKEA, titled \""IKEA Furniture, Kitchens, and Garden\"", has been successfully created with a daily budget of 3 EUR. The campaign includes three ad groups focusing on furniture, kitchens, and garden areas, each with tailored ad copy and keywords to highlight IKEA's diverse product range, customisable solutions, and sustainable and affordable offerings.\n\nHere are the details of the created ad groups with clickable links:\n\n- Furniture Ad Group: 565\n- Kitchens Ad Group: 606\n- Garden Ad Group: 415\n\nWould you like to review the ad copy or make any further adjustments?"",""smart_suggestions"":{""suggestions"":[""Review ad copy"",""Adjust campaign settings"",""Discuss further optimizations""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +28,end2end,https://www.disneystore.eu,gpt4,1394.500629901886,DONE,True,"Response from team '123_234': +{""message"":""The Google Ads campaign for Disney Store EU has been successfully created with the following details:\n\n- Campaign Name: Disney Store EU - New Arrivals & Offers\n- Daily Budget: 3 EUR\n- Status: Enabled\n- Targeting: Google Search Network\n\nAdditionally, an ad group named 'New In Products' has been set up with ads and keywords focused on the latest Disney, Marvel, Pixar, and Star Wars merchandise. The ad group includes carefully crafted headlines, descriptions, and keywords that align with the 'New In Products' theme.\n\nHere are the details of the created ad group:\n\n- Ad Group Name: New In Products\n- Final URL: [https://www.disneystore.eu/new](https://www.disneystore.eu/new)\n- Status: Enabled\n\nHeadlines and descriptions have been crafted to highlight the newest arrivals and entice potential customers. Keywords have been selected to match exactly with user search queries, ensuring high relevance and better chances of ad visibility.\n\nWould you like to proceed with creating additional ad groups for 'Offers' and 'Stitch Attacks Snacks Collection', or is there anything else you would like to adjust in the campaign?"",""smart_suggestions"":{""suggestions"":[""Create 'Offers' ad group"",""Create 'Stitch Attacks Snacks Collection' ad group"",""Adjust campaign settings"",""Review campaign performance""],""type"":""manyOf""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" +29,end2end,https://faststream.airt.ai,gpt4,1043.1329810619354,DONE,True,"Response from team '123_234': +{""message"":""The ad group \""FastStream Features\"" has been successfully created within the \""FastStream Awareness Campaign,\"" along with the associated ads and keywords. Here are the details:\n\n- **Ad Group Name**: FastStream Features\n- **Final URL**: [https://faststream.airt.ai](https://faststream.airt.ai)\n- **Headlines**: FastStream Framework, Kafka & NATS Support, Pydantic Validation, and other relevant headlines.\n- **Descriptions**: Build async web services with ease, Validate with Pydantic, generate docs, and more.\n- **Keywords**: FastStream open-source framework, Kafka routing integration, NATS event streaming, and others.\n\nThis completes the task of creating a new Google Ads campaign for FastStream, focusing on increasing brand awareness and boosting sales by highlighting key features such as Kafka Routing and NATS Integration.\n\nIf you would like to make any changes or updates to the campaign, please feel free to start a new chat with us. Thank you for choosing our services, and we look forward to continuing to support your digital marketing efforts."",""smart_suggestions"":{""suggestions"":[""""],""type"":""""},""is_question"":true,""status"":""completed"",""terminate_groupchat"":true}" diff --git a/captn/captn_agents/backend/benchmarking/base.py b/captn/captn_agents/backend/benchmarking/base.py index e4c05cdc..5f639eae 100755 --- a/captn/captn_agents/backend/benchmarking/base.py +++ b/captn/captn_agents/backend/benchmarking/base.py @@ -34,7 +34,7 @@ def lock_file(path: Path) -> Iterator[None]: lock_path.unlink() -COMMON_COLUMNS = ["execution_time", "status", "success", "output"] +COMMON_COLUMNS = ["execution_time", "status", "success", "output", "retries"] def _add_common_columns_and_save( @@ -219,13 +219,17 @@ def run_test( ) -> Dict[str, Any]: try: time_start = time.time() - output = benchmark(**kwargs) - success = True + output, retry_from_scratch_counters = benchmark(**kwargs) + if retry_from_scratch_counters == 0: + success = "Success" + else: + success = "Success with retry" except Exception as e: traceback.print_stack() traceback.print_exc() output = str(e) - success = False + success = "Failed" + retry_from_scratch_counters = -1 finally: total_time = time.time() - time_start status = "DONE" @@ -235,6 +239,7 @@ def run_test( "status": status, "success": success, "output": output, + "retries": retry_from_scratch_counters, } @@ -250,31 +255,42 @@ def get_random_nan_index(xs: pd.Series) -> Any: def create_ag_report(df: pd.DataFrame, groupby_list: List[str]) -> pd.DataFrame: df = df.dropna(subset=["success"]) - group = df.groupby(groupby_list)["success"] - total = group.count() - success = group.sum() - success_rate = (success / total).rename("success_rate") - - group = df.groupby(groupby_list)["execution_time"] - avg_time = group.mean().rename("avg_time") - - ag_report_df = pd.concat([success_rate, avg_time], axis=1).sort_values( - "success_rate", ascending=True + result = ( + df.groupby(groupby_list) + .agg( + success_percentage=( + "success", + lambda x: sum(x == "Success") / len(x) * 100, + ), + success_with_retry_percentage=( + "success", + lambda x: sum(x == "Success with retry") / len(x) * 100, + ), + failed_percentage=("success", lambda x: sum(x == "Failed") / len(x) * 100), + avg_time=("execution_time", "mean"), + ) + .reset_index() ) - # Calculate total success rate and average time - total_success = success.sum() - count = len(df) - total_time = df["execution_time"].sum() - total_success_rate = total_success / count - total_avg_time = total_time / count - - ag_report_df.loc["Total"] = [total_success_rate, total_avg_time] + total_success_percentage = df["success"].eq("Success").mean() * 100 + total_success_with_retry_percentage = ( + df["success"].eq("Success with retry").mean() * 100 + ) + total_failed_percentage = df["success"].eq("Failed").mean() * 100 + total_avg_execution_time = df["execution_time"].mean() + total_row = { + "success_percentage": total_success_percentage, + "success_with_retry_percentage": total_success_with_retry_percentage, + "failed_percentage": total_failed_percentage, + "avg_time": total_avg_execution_time, + } + for col in groupby_list: + total_row[col] = None + result.loc["Total"] = total_row - # round columns to 2 decimal places - ag_report_df = ag_report_df.round(2) + result.loc[result.index[-1], "url"] = "Total" - return ag_report_df + return result.reset_index(drop=True).round(2) GROUP_BY_DICT = { diff --git a/captn/captn_agents/backend/benchmarking/brief_creation_team.py b/captn/captn_agents/backend/benchmarking/brief_creation_team.py index 3d3f1e70..ac3ac645 100644 --- a/captn/captn_agents/backend/benchmarking/brief_creation_team.py +++ b/captn/captn_agents/backend/benchmarking/brief_creation_team.py @@ -113,7 +113,7 @@ def benchmark_brief_creation( url: str, team_name: str, llm: str = Models.gpt3_5, -) -> str: +) -> Tuple[str, int]: config_list = get_config_list(llm) user_id = 123 @@ -166,7 +166,9 @@ def benchmark_brief_creation( "task" in delegate_task_function_sugestion_function["arguments"] ) # nosec: [B101] - return delegate_task_function_sugestion_function["arguments"] # type: ignore[no-any-return] + return delegate_task_function_sugestion_function[ + "arguments" + ], team.retry_from_scratch_counter finally: poped_team = Team.pop_team(user_id=user_id, conv_id=conv_id) assert isinstance(poped_team, Team) # nosec: [B101] diff --git a/captn/captn_agents/backend/benchmarking/campaign_creation_team.py b/captn/captn_agents/backend/benchmarking/campaign_creation_team.py index a92fd834..734c4d19 100644 --- a/captn/captn_agents/backend/benchmarking/campaign_creation_team.py +++ b/captn/captn_agents/backend/benchmarking/campaign_creation_team.py @@ -182,7 +182,7 @@ def continue_conversation_until_finished( def benchmark_campaign_creation( url: str, llm: str = Models.gpt4, -) -> str: +) -> Tuple[str, int]: try: task = URL_TASK_DICT[url] config_list = get_config_list(llm) @@ -215,7 +215,7 @@ def benchmark_campaign_creation( mock_create_ad_group_ad=mock_create_ad_group_ad, mock_create_ad_group_keyword=mock_create_ad_group_keyword, mock_create_campaign=mock_create_campaign, - ) + ), campaign_creation_team.retry_from_scratch_counter finally: user_id, conv_id = campaign_creation_team.name.split("_")[-2:] success = Team.pop_team(user_id=int(user_id), conv_id=int(conv_id)) diff --git a/captn/captn_agents/backend/benchmarking/end2end.py b/captn/captn_agents/backend/benchmarking/end2end.py index f4957b1c..a8d0cd8f 100644 --- a/captn/captn_agents/backend/benchmarking/end2end.py +++ b/captn/captn_agents/backend/benchmarking/end2end.py @@ -1,6 +1,6 @@ from contextlib import contextmanager from tempfile import TemporaryDirectory -from typing import Any, Iterator +from typing import Any, Iterator, Tuple from unittest.mock import patch from autogen.cache import Cache @@ -49,13 +49,13 @@ def _patch_brief_creation_team_vars( def benchmark_end2end( url: str, llm: str = Models.gpt4, -) -> str: +) -> Tuple[str, int]: config_list = get_config_list(llm) user_id = 123 conv_id = 234 task = _get_task(url) - team = BriefCreationTeam( + brief_creation_team = BriefCreationTeam( task=task, user_id=user_id, conv_id=conv_id, config_list=config_list ) @@ -66,7 +66,9 @@ def benchmark_end2end( with TemporaryDirectory() as cache_dir: with Cache.disk(cache_path_root=cache_dir) as cache: with _patch_brief_creation_team_vars( - team=team, client_system_message=client_system_message, cache=cache + team=brief_creation_team, + client_system_message=client_system_message, + cache=cache, ) as mock_change_the_team_and_start_new_chat: with _patch_campaign_creation_team_vars() as ( mock_create_ad_group, @@ -74,7 +76,7 @@ def benchmark_end2end( mock_create_ad_group_keyword, mock_create_campaign, ): - team.initiate_chat(cache=cache) + brief_creation_team.initiate_chat(cache=cache) continue_conversation_until_finished( user_id=user_id, conv_id=conv_id, @@ -94,12 +96,16 @@ def benchmark_end2end( ) assert isinstance(campaign_creation_team, CampaignCreationTeam) # nosec: [B101] - return run_assertions_and_return_last_message( - campaign_creation_team=campaign_creation_team, - mock_create_ad_group=mock_create_ad_group, - mock_create_ad_group_ad=mock_create_ad_group_ad, - mock_create_ad_group_keyword=mock_create_ad_group_keyword, - mock_create_campaign=mock_create_campaign, + return ( + run_assertions_and_return_last_message( + campaign_creation_team=campaign_creation_team, + mock_create_ad_group=mock_create_ad_group, + mock_create_ad_group_ad=mock_create_ad_group_ad, + mock_create_ad_group_keyword=mock_create_ad_group_keyword, + mock_create_campaign=mock_create_campaign, + ), + brief_creation_team.retry_from_scratch_counter + + campaign_creation_team.retry_from_scratch_counter, ) finally: poped_team = Team.pop_team(user_id=user_id, conv_id=conv_id) diff --git a/captn/captn_agents/backend/benchmarking/websurfer.py b/captn/captn_agents/backend/benchmarking/websurfer.py index 5af4bdc4..1173122f 100644 --- a/captn/captn_agents/backend/benchmarking/websurfer.py +++ b/captn/captn_agents/backend/benchmarking/websurfer.py @@ -1,3 +1,5 @@ +from typing import Tuple + from ..tools._functions import ( get_get_info_from_the_web_page, get_llm_config_gpt_3_5, @@ -17,7 +19,7 @@ def benchmark_websurfer( navigator_llm: str = Models.gpt4, timestamp: str = "2024-01-01T00:00:0", introduce_give_up_after: int = 7, -) -> str: +) -> Tuple[str, int]: llm_configs = { Models.gpt3_5.value: get_llm_config_gpt_3_5(), Models.gpt4.value: get_llm_config_gpt_4(), @@ -42,4 +44,5 @@ def benchmark_websurfer( ): raise AssertionError(last_message) - return last_message + # TODO: Always return 0 as a retry counter + return last_message, 0 diff --git a/captn/captn_agents/backend/teams/_brief_creation_team.py b/captn/captn_agents/backend/teams/_brief_creation_team.py index 2b0a6d84..52d4c994 100644 --- a/captn/captn_agents/backend/teams/_brief_creation_team.py +++ b/captn/captn_agents/backend/teams/_brief_creation_team.py @@ -44,9 +44,6 @@ def __init__( temperature: float = 0.2, config_list: Optional[List[Dict[str, str]]] = None, ): - self.task = task - self.initial_brief = task - recommended_modifications_and_answer_list: List[ Tuple[Dict[str, Any], Optional[str]] ] = [] @@ -58,6 +55,7 @@ def __init__( user_id=user_id, conv_id=conv_id, roles=roles, + task=task, function_map=function_map, work_dir=work_dir, max_round=max_round, @@ -85,7 +83,7 @@ def _add_tools(self) -> None: self.toolbox = create_brief_creation_team_toolbox( user_id=self.user_id, conv_id=self.conv_id, - initial_brief=self.initial_brief, + initial_brief=self.task, ) for agent in self.members: if agent != self.user_proxy: diff --git a/captn/captn_agents/backend/teams/_campaign_creation_team.py b/captn/captn_agents/backend/teams/_campaign_creation_team.py index 845bcb8d..9484a026 100644 --- a/captn/captn_agents/backend/teams/_campaign_creation_team.py +++ b/captn/captn_agents/backend/teams/_campaign_creation_team.py @@ -101,8 +101,6 @@ def __init__( temperature: float = 0.2, config_list: Optional[List[Dict[str, str]]] = None, ): - self.task = task - recommended_modifications_and_answer_list: List[ Tuple[Dict[str, Any], Optional[str]] ] = [] @@ -113,6 +111,7 @@ def __init__( user_id=user_id, conv_id=conv_id, roles=roles, + task=task, function_map=function_map, work_dir=work_dir, max_round=max_round, @@ -319,3 +318,9 @@ def get_brief_template(cls) -> str: Now Let's get all the information from the clients web page and create a detailed plan for the campaign. """ + + def initiate_chat(self, **kwargs: Any) -> None: + if self.toolbox._context.changes_made: # type: ignore[union-attr] + self.initial_message += f"\nThe following modifications have already been done: {self.toolbox._context.changes_made}\n" # type: ignore[union-attr] + self.toolbox._context.changes_made = "" # type: ignore[union-attr] + super().initiate_chat(**kwargs) diff --git a/captn/captn_agents/backend/teams/_google_ads_team.py b/captn/captn_agents/backend/teams/_google_ads_team.py index 1bbfa12c..acd3908f 100644 --- a/captn/captn_agents/backend/teams/_google_ads_team.py +++ b/captn/captn_agents/backend/teams/_google_ads_team.py @@ -80,6 +80,7 @@ def __init__( user_id=user_id, conv_id=conv_id, roles=roles, + task=task, function_map=function_map, work_dir=work_dir, max_round=max_round, @@ -89,7 +90,6 @@ def __init__( use_user_proxy=True, ) self.conv_id = conv_id - self.task = task self.llm_config = GoogleAdsTeam._get_llm_config( seed=seed, temperature=temperature ) diff --git a/captn/captn_agents/backend/teams/_team.py b/captn/captn_agents/backend/teams/_team.py index 3fd8b8c7..11244532 100644 --- a/captn/captn_agents/backend/teams/_team.py +++ b/captn/captn_agents/backend/teams/_team.py @@ -1,4 +1,7 @@ +import datetime import json +import os +import time import traceback from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar @@ -12,6 +15,8 @@ _completions_create_original = autogen.oai.client.OpenAIClient.create +os.environ["AUTOGEN_USE_DOCKER"] = "no" + # WORKAROUND for consistent 500 error code when using openai functions @patch # type: ignore @@ -63,6 +68,8 @@ class Team: "Please continue.", ] * 3 + _MAX_RETRIES_FROM_SCRATCH = 3 + @classmethod def register_team(cls, name: str) -> Callable[[T], T]: def _inner(cls_inner: T) -> T: @@ -133,6 +140,7 @@ def __init__( user_id: int, conv_id: int, roles: List[Dict[str, str]], + task: str = "", function_map: Optional[Dict[str, Callable[[Any], Any]]] = None, work_dir: str = "my_default_workdir", max_round: int = 80, @@ -147,6 +155,7 @@ def __init__( self.user_id = user_id self.conv_id = conv_id self.roles = roles + self.task = task self.initial_message: str self.name: str self.max_round = max_round @@ -164,6 +173,7 @@ def __init__( self.name = Team.construct_team_name(user_id=user_id, conv_id=conv_id) self.user_proxy: Optional[autogen.UserProxyAgent] = None + self.retry_from_scratch_counter = 0 Team._store_team(user_id=user_id, conv_id=conv_id, team=self) @classmethod @@ -361,7 +371,7 @@ def get_last_message(self, add_prefix: bool = True) -> str: return last_message # type: ignore - def retry_func(self) -> None: + def retry_func(self, delay: int = 2) -> None: exception: Optional[Exception] = None for i in range(len(self._retry_messages)): print(f"Retry number {i+1}.") @@ -372,6 +382,7 @@ def retry_func(self) -> None: message = self._retry_messages[i] try: + time.sleep(delay**i) self.manager.send( recipient=self.manager, message=message, @@ -394,11 +405,30 @@ def retry_func(self) -> None: traceback.print_stack() raise exception + @staticmethod + def retry_from_scratch(self: "Team", e: Exception, delay: int = 3) -> None: + print(f"Retry from scratch: {type(e)}, {e}") + # Try the team again from scratch + self.retry_from_scratch_counter += 1 + if self.retry_from_scratch_counter < self._MAX_RETRIES_FROM_SCRATCH: + self.initial_message += ( + f"\nTimestamp: {datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')}" + ) + time.sleep(delay**self.retry_from_scratch_counter) + self.initiate_chat(**self.initiate_chat_kwargs) + else: + raise e + @staticmethod def handle_exceptions(func: Callable[..., None]) -> Callable[..., None]: def wrapper(self: "Team", *args: Any, **kwargs: Any) -> None: try: - return func(self, *args, **kwargs) + delay = kwargs.get("delay", 2) + func(self, *args, **kwargs) + + if len(self.get_messages()) >= self.max_round: + error_message = f"Maximum number of messages reached: {self.max_round}, Retrying the team from scratch." + Team.retry_from_scratch(self, Exception(error_message), delay=delay) except ( openai.APIStatusError, openai.BadRequestError, @@ -407,12 +437,17 @@ def wrapper(self: "Team", *args: Any, **kwargs: Any) -> None: TimeoutError, ) as e: print(f"Handling exception: {type(e)}, {e}") - self.retry_func() + try: + # Try to unstuck the team + self.retry_func(delay=delay) + except Exception as e: + Team.retry_from_scratch(self, e, delay=delay) return wrapper @handle_exceptions def initiate_chat(self, **kwargs: Any) -> None: + self.initiate_chat_kwargs = kwargs self.manager.initiate_chat(self.manager, message=self.initial_message, **kwargs) @handle_exceptions diff --git a/captn/captn_agents/backend/teams/_weekly_analysis_team.py b/captn/captn_agents/backend/teams/_weekly_analysis_team.py index d3ae5d5d..99d22f72 100644 --- a/captn/captn_agents/backend/teams/_weekly_analysis_team.py +++ b/captn/captn_agents/backend/teams/_weekly_analysis_team.py @@ -684,6 +684,7 @@ def __init__( user_id=user_id, conv_id=conv_id, roles=roles, + task=task, function_map=function_map, work_dir=work_dir, max_round=max_round, @@ -691,7 +692,6 @@ def __init__( temperature=temperature, use_user_proxy=True, ) - self.task = task self.llm_config = WeeklyAnalysisTeam._get_llm_config( seed=seed, temperature=temperature ) diff --git a/captn/captn_agents/backend/tools/_campaign_creation_team_tools.py b/captn/captn_agents/backend/tools/_campaign_creation_team_tools.py index 571f88ae..ef7f73b8 100644 --- a/captn/captn_agents/backend/tools/_campaign_creation_team_tools.py +++ b/captn/captn_agents/backend/tools/_campaign_creation_team_tools.py @@ -189,7 +189,7 @@ def _create_ad_group_keywords( ad_group_id=ad_group_id, customer_id=ad_group_with_ad_and_keywords.customer_id, ) - response += f"Keyword: {keyword_response}\n" + response += f"Keyword '{keyword.keyword_text}': {keyword_response}\n" return response @@ -256,7 +256,7 @@ def create_ad_group_with_ad_and_keywords( ) if isinstance(ad_group_response, dict): return ad_group_response - response = f"Ad group: {ad_group_response}\n" + response = f"Ad group '{ad_group_with_ad_and_keywords.ad_group.name}': {ad_group_response}\n" ad_group_id = _get_resource_id_from_response(ad_group_response) ad_group_ad_response = _create_ad_group_ad( @@ -266,7 +266,7 @@ def create_ad_group_with_ad_and_keywords( ad_group_with_ad_and_keywords=ad_group_with_ad_and_keywords, ad_group_id=ad_group_id, ) - response += f"Ad group ad: {ad_group_ad_response}\n" + response += f"Ad group ad with final url - '{ad_group_with_ad_and_keywords.ad_group_ad.final_url}': {ad_group_ad_response}\n" ad_group_keywords_response = _create_ad_group_keywords( user_id=user_id, @@ -276,6 +276,7 @@ def create_ad_group_with_ad_and_keywords( ad_group_id=ad_group_id, ) response += ad_group_keywords_response # type: ignore + context.changes_made += f"\n{response}" return response diff --git a/captn/captn_agents/backend/tools/_functions.py b/captn/captn_agents/backend/tools/_functions.py index 7605832c..5b324385 100644 --- a/captn/captn_agents/backend/tools/_functions.py +++ b/captn/captn_agents/backend/tools/_functions.py @@ -185,6 +185,7 @@ class Context: ] toolbox: Toolbox get_only_non_manager_accounts: bool = False + changes_made: str = "" ask_client_for_permission_description = """Ask the client for permission to make the changes. Use this method before calling any of the modification methods! diff --git a/captn/captn_agents/backend/tools/_google_ads_team_tools.py b/captn/captn_agents/backend/tools/_google_ads_team_tools.py index ee0099cf..04b77708 100644 --- a/captn/captn_agents/backend/tools/_google_ads_team_tools.py +++ b/captn/captn_agents/backend/tools/_google_ads_team_tools.py @@ -266,7 +266,7 @@ def create_campaign( context.recommended_modifications_and_answer_list ) - return google_ads_create_update( + response = google_ads_create_update( user_id=user_id, conv_id=conv_id, recommended_modifications_and_answer_list=recommended_modifications_and_answer_list, @@ -282,6 +282,11 @@ def create_campaign( endpoint="/create-campaign", ) + if isinstance(response, str): + response = f"Campaign '{name}': {response}" + context.changes_made += f"\n{response}" + return response + create_keyword_for_ad_group_description = f"Creates (regular and negative) keywords for Ad Group (AdGroupCriterion). {MODIFICATION_WARNING}" diff --git a/pyproject.toml b/pyproject.toml index 0214f117..8313eb04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,18 +61,18 @@ lint = [ "ruff==0.4.4", "pyupgrade-directories", "bandit==1.7.8", - "semgrep==1.71.0", + "semgrep==1.73.0", "pre-commit==3.7.1", "detect-secrets==1.4.0", ] test-core = [ "coverage[toml]==7.4.4", - "pytest==8.1.1", + "pytest==8.2.1", "pytest-asyncio>=0.23.6", "dirty-equals==0.7.1.post0", - "pyarrow==15.0.2", - "pytest-retry==1.6.2", + "pyarrow==16.1.0", + "pytest-retry==1.6.3", "pytest-repeat==0.9.3", ] @@ -100,7 +100,7 @@ agents = [ "pyautogen[websurfer,websockets]==0.2.27", "pandas>=2.1", "fastcore==1.5.35", - "asyncer==0.0.5", + "asyncer==0.0.7", "pydantic==2.7.1", "markdownify==0.12.1", "tenacity==8.2.3", diff --git a/tests/ci/captn/captn_agents/backend/benchmarking/test_benchmarking.py b/tests/ci/captn/captn_agents/backend/benchmarking/test_benchmarking.py index 3c4bef0c..17c3045f 100644 --- a/tests/ci/captn/captn_agents/backend/benchmarking/test_benchmarking.py +++ b/tests/ci/captn/captn_agents/backend/benchmarking/test_benchmarking.py @@ -1,7 +1,7 @@ import functools from pathlib import Path from tempfile import TemporaryDirectory -from typing import List, Optional +from typing import List, Optional, Set import pandas as pd import pytest @@ -33,7 +33,7 @@ def test_help(self): def benchmark_success(success: bool, *args, **kwargs): print(f"success: {success}") if success: - return "it's ok" + return "it's ok", 0 else: raise RuntimeError("it's not ok") @@ -61,7 +61,14 @@ def run_tests_for_team_success( command: str, aggregated_csv_name: str, success: bool, - columns: List[str] = ["url", "success_rate", "avg_time"], # noqa: B006 + columns: Set[str] = [ # noqa: B006 + "Unnamed: 0", + "url", + "success_percentage", + "success_with_retry_percentage", + "failed_percentage", + "avg_time", + ], no_rows: int = 50, additional_generate_task_table_parameters: Optional[List[str]] = None, ) -> None: @@ -89,9 +96,9 @@ def run_tests_for_team_success( print(tabulate(df, headers="keys", tablefmt="pretty")) assert (df["status"] == "DONE").all() if success: - assert df["success"].all() + assert (df["success"] == "Success").all() else: - assert not df["success"].any() + assert (df["success"] == "Failed").all() aggregated_csv_path = Path(tmp_dir) / aggregated_csv_name assert aggregated_csv_path.exists() @@ -136,7 +143,14 @@ def test_create_ag_report(self): { "url": ["url1", "url2", "url1", "url2", "url1", "url1"], "status": ["DONE", "DONE", "DONE", "DONE", "DONE", "DONE"], - "success": [True, True, False, False, False, False], + "success": [ + "Success", + "Success with retry", + "Failed", + "Failed", + "Failed", + "Failed", + ], "execution_time": [1, 2, 3, 4, 5, 2], } ) @@ -144,14 +158,47 @@ def test_create_ag_report(self): df, ["url"] ) - assert report_ag_df.shape == (3, 2) + assert report_ag_df.shape == (3, 5) + + url1_result = report_ag_df[report_ag_df["url"] == "url1"] + assert url1_result["success_percentage"].values[0] == 25.0 + assert url1_result["failed_percentage"].values[0] == 75.0 + assert url1_result["avg_time"].values[0] == 2.75 + + url2_result = report_ag_df[report_ag_df["url"] == "url2"] + assert url2_result["success_percentage"].values[0] == 0.0 + assert url2_result["success_with_retry_percentage"].values[0] == 50.0 + assert url2_result["failed_percentage"].values[0] == 50.0 + assert url2_result["avg_time"].values[0] == 3.0 + + total_result = report_ag_df[report_ag_df["url"] == "Total"] + assert total_result["success_percentage"].values[0] == 16.67 + assert total_result["success_with_retry_percentage"].values[0] == 16.67 + assert total_result["failed_percentage"].values[0] == 66.67 + assert total_result["avg_time"].values[0] == 2.83 + + def test_create_ag_report_with_multiple_columns_to_group_by(self): + df = pd.DataFrame( + { + "url": ["url1", "url2", "url1", "url2", "url1", "url1"], + "team_name": ["team1", "team1", "team1", "team1", "team2", "team1"], + "status": ["DONE", "DONE", "DONE", "DONE", "DONE", "DONE"], + "success": [ + "Success", + "Success with retry", + "Failed", + "Failed", + "Failed", + "Failed", + ], + "execution_time": [1, 2, 3, 4, 5, 2], + } + ) + report_ag_df = captn.captn_agents.backend.benchmarking.base.create_ag_report( + df, ["url", "team_name"] + ) - assert report_ag_df.loc["url1", "success_rate"] == 0.25 - assert report_ag_df.loc["url1", "avg_time"] == 2.75 - assert report_ag_df.loc["url2", "success_rate"] == 0.5 - assert report_ag_df.loc["url2", "avg_time"] == 3.0 - assert report_ag_df.loc["Total", "success_rate"] == 0.33 - assert report_ag_df.loc["Total", "avg_time"] == 2.83 + assert report_ag_df.shape == (4, 6) class TestBriefCreation: @@ -184,7 +231,15 @@ def test_run_tests_for_brief_creation_success( aggregated_csv_name=self.aggregated_csv_name, success=success, no_rows=100, - columns=["Unnamed: 0", "success_rate", "avg_time"], + columns=[ + "Unnamed: 0", + "url", + "team_name", + "success_percentage", + "success_with_retry_percentage", + "failed_percentage", + "avg_time", + ], ) diff --git a/tests/ci/captn/captn_agents/backend/teams/test_team.py b/tests/ci/captn/captn_agents/backend/teams/test_team.py index a919363d..d06b12dc 100644 --- a/tests/ci/captn/captn_agents/backend/teams/test_team.py +++ b/tests/ci/captn/captn_agents/backend/teams/test_team.py @@ -88,18 +88,18 @@ def test_retry_func(self, number_of_exceptions) -> None: team.manager.send = MagicMock() if number_of_exceptions == 0: - team.retry_func() + team.retry_func(delay=0) team.manager.send.assert_called_once() elif number_of_exceptions > len(Team._retry_messages): team.manager.send.side_effect = httpx.ReadTimeout("Timeout") with pytest.raises(httpx.ReadTimeout): - team.retry_func() + team.retry_func(delay=0) assert team.manager.send.call_count == len(Team._retry_messages) else: team.manager.send.side_effect = [ httpx.ReadTimeout("Timeout") ] * number_of_exceptions + [None] - team.retry_func() + team.retry_func(delay=0) assert team.manager.send.call_count == number_of_exceptions + 1 @pytest.mark.parametrize("raise_exception", [False, True]) @@ -110,6 +110,8 @@ def f(self: Team, message: str, raise_exception: bool) -> None: raise httpx.ReadTimeout("Timeout") team = Team(roles=TestTeam.roles, user_id=123, conv_id=456) + team.groupchat = MagicMock() + team.groupchat.messages = ["message1"] team.retry_func = MagicMock() f(team, "message", raise_exception=raise_exception) @@ -118,32 +120,75 @@ def f(self: Team, message: str, raise_exception: bool) -> None: else: team.retry_func.assert_not_called() - @pytest.mark.parametrize("number_of_exceptions", [0, 2, 10]) + @pytest.mark.parametrize("number_of_exceptions", [0, 2, 10, 30]) def test_initiate_chat(self, number_of_exceptions) -> None: team = Team(roles=TestTeam.roles, user_id=123, conv_id=456) team.initial_message = "Initial message" team.manager = MagicMock() team.manager.initiate_chat = MagicMock() team.manager.send = MagicMock() + team.groupchat = MagicMock() + team.groupchat.messages = ["message1", "message2"] if number_of_exceptions == 0: - team.initiate_chat() + team.initiate_chat(delay=0) team.manager.send.assert_not_called() - elif number_of_exceptions > len(Team._retry_messages): + team.manager.initiate_chat.assert_called_once() + elif ( + number_of_exceptions + > len(Team._retry_messages) * Team._MAX_RETRIES_FROM_SCRATCH + ): team.manager.initiate_chat.side_effect = httpx.ReadTimeout("Timeout") team.manager.send.side_effect = httpx.ReadTimeout("Timeout") with pytest.raises(httpx.ReadTimeout): - team.initiate_chat() - assert team.manager.send.call_count == len(Team._retry_messages) - else: + team.initiate_chat(delay=0) + + expected_call_count = ( + len(Team._retry_messages) * Team._MAX_RETRIES_FROM_SCRATCH + ) + assert team.manager.send.call_count == expected_call_count + assert ( + team.manager.initiate_chat.call_count == Team._MAX_RETRIES_FROM_SCRATCH + ) + elif number_of_exceptions < len(Team._retry_messages): + team.manager.initiate_chat.side_effect = httpx.ReadTimeout("Timeout") + team.manager.send.side_effect = [ + httpx.ReadTimeout("Timeout") + ] * number_of_exceptions + [None] + team.initiate_chat(delay=0) + assert team.manager.send.call_count == number_of_exceptions + 1 + team.manager.initiate_chat.assert_called_once() + elif number_of_exceptions == 10: team.manager.initiate_chat.side_effect = httpx.ReadTimeout("Timeout") team.manager.send.side_effect = [ httpx.ReadTimeout("Timeout") ] * number_of_exceptions + [None] - team.initiate_chat() + team.initiate_chat(delay=0) assert team.manager.send.call_count == number_of_exceptions + 1 + assert team.manager.initiate_chat.call_count == 2 - team.manager.initiate_chat.assert_called_once() + @pytest.mark.parametrize("num_of_messages", [2, 5, 10]) + def test_max_number_of_messages(self, num_of_messages) -> None: + max_round = 5 + team = Team(roles=TestTeam.roles, user_id=123, conv_id=456, max_round=max_round) + team.initial_message = "Initial message" + team.manager = MagicMock() + team.manager.initiate_chat = MagicMock() + team.manager.send = MagicMock() + team.groupchat = MagicMock() + team.groupchat.messages = ["message1"] * num_of_messages + + if num_of_messages < max_round: + team.initiate_chat(delay=0) + team.manager.send.assert_not_called() + team.manager.initiate_chat.assert_called_once() + else: + with pytest.raises(Exception): # noqa: B017 + team.initiate_chat(delay=0) + team.manager.send.assert_not_called() + assert ( + team.manager.initiate_chat.call_count == Team._MAX_RETRIES_FROM_SCRATCH + ) class TestTeamRegistry: diff --git a/tests/ci/captn/captn_agents/backend/tools/test_campaign_creation_team_tools.py b/tests/ci/captn/captn_agents/backend/tools/test_campaign_creation_team_tools.py index 58336b68..300eec84 100644 --- a/tests/ci/captn/captn_agents/backend/tools/test_campaign_creation_team_tools.py +++ b/tests/ci/captn/captn_agents/backend/tools/test_campaign_creation_team_tools.py @@ -119,10 +119,10 @@ def test_create_ad_group_with_ad_and_keywords(self) -> None: context=context, ) - expected_response = f"""Ad group: {side_effect[0]} -Ad group ad: {side_effect[1]} -Keyword: {side_effect[2]} -Keyword: {side_effect[3]} + expected_response = f"""Ad group '{ad_group.name}': {side_effect[0]} +Ad group ad with final url - '{ad_group_ad.final_url}': {side_effect[1]} +Keyword '{keyword1.keyword_text}': {side_effect[2]} +Keyword '{keyword2.keyword_text}': {side_effect[3]} """ print() From 8150c45aad14d9ea0b847eef0db3551e46532797 Mon Sep 17 00:00:00 2001 From: rjambrecic <32619626+rjambrecic@users.noreply.github.com> Date: Wed, 22 May 2024 09:13:35 +0200 Subject: [PATCH 2/4] Propose maximum 3 actions in the weekly analysis (#705) --- .../backend/teams/_weekly_analysis_team.py | 11 ++++++----- captn/captn_agents/backend/tools/_functions.py | 8 ++++++-- .../backend/tools/test_functions.py | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/captn/captn_agents/backend/teams/_weekly_analysis_team.py b/captn/captn_agents/backend/teams/_weekly_analysis_team.py index 99d22f72..8521ffbb 100644 --- a/captn/captn_agents/backend/teams/_weekly_analysis_team.py +++ b/captn/captn_agents/backend/teams/_weekly_analysis_team.py @@ -739,10 +739,11 @@ def _guidelines(self) -> str: 11. The client can NOT see your conversation, he only receives the message which you send him by using the 'send_email' command Here an example on how to use the 'send_email' command: -{ +'proposed_user_actions' must contain a list with MIN 1 and MAX 3 strings. We suggest to use the MAXIMUM number of proposed actions. +{"actions": { "proposed_user_actions": ["Remove 'Free' keyword because it is not performing well", "Increase budget from $10/day to $20/day", "Remove the headline 'New product' and replace it with 'Very New product' in the 'Adgroup 1'", "Select some or all of them"] -} +}} propose_user_actions should NOT be general, but specific. 'proposed_user_actions' BAD EXAMPLES: @@ -793,7 +794,7 @@ def _guidelines(self) -> str: 23. Once you have completed weekly analysis, you must send a summary of the work done to the client by using the 'send_email' command. Here is a list of things which you CAN do after the client responds to your email. -So please recommend some of these changes to the client by using the 'proposed_user_actions' parameter in the 'send_email' command: +So please recommend some of these changes to the client by using the 'actions' parameter in the 'send_email' command: - update the status (ENABLED / PAUSED) of the campaign, ad group and ad - create/update/remove headlines and descriptions in the Ad Copy. Make sure to follow the restrictions for the headlines and descriptions (MAXIMUM 30 characters for headlines and MAXIMUM 90 characters for descriptions) - create/update/remove new keywords @@ -822,8 +823,8 @@ def _commands(self) -> str: Just suggest calling function 'function_name'. All team members have access to the following command: -1. send_email: Send email to the client, params: (proposed_user_actions: List[str]]) -Each message in the 'proposed_user_actions' parameter must contain all information useful to the client, because the client does not see your team's conversation +1. send_email: Send email to the client, params: (actions: Actions(BaseModel)) +Each message in the actions.proposed_user_actions parameter must contain all information useful to the client, because the client does not see your team's conversation As we send this message to the client, pay attention to the content inside it. We are a digital agency and the messages we send must be professional. 2. {GET_INFO_FROM_THE_WEB_COMMAND} diff --git a/captn/captn_agents/backend/tools/_functions.py b/captn/captn_agents/backend/tools/_functions.py index 5b324385..0748e8a9 100644 --- a/captn/captn_agents/backend/tools/_functions.py +++ b/captn/captn_agents/backend/tools/_functions.py @@ -801,13 +801,17 @@ def get_info_from_the_web_page( send_email_description = "Send email to the client." +class Actions(BaseModel): + proposed_user_actions: Annotated[List[str], Len(min_length=1, max_length=3)] + + def send_email( - proposed_user_actions: Annotated[List[str], "List of proposed user actions"], + actions: Actions, ) -> Dict[str, Any]: return_msg = { "subject": "Capt’n.ai Weekly Analysis", "email_content": "", - "proposed_user_action": proposed_user_actions, + "proposed_user_action": actions.proposed_user_actions, "terminate_groupchat": True, } return return_msg diff --git a/tests/ci/captn/captn_agents/backend/tools/test_functions.py b/tests/ci/captn/captn_agents/backend/tools/test_functions.py index 8a6153ce..8132baf0 100644 --- a/tests/ci/captn/captn_agents/backend/tools/test_functions.py +++ b/tests/ci/captn/captn_agents/backend/tools/test_functions.py @@ -11,6 +11,7 @@ AdGroupWithAdAndKeywords, ) from captn.captn_agents.backend.tools._functions import ( + Actions, Summary, WebPageSummary, WebUrl, @@ -18,6 +19,7 @@ _validate_modification_parameters, get_get_info_from_the_web_page, get_webpage_status_code, + send_email, ) @@ -218,3 +220,19 @@ def create_ad_group_with_ad_and_keywords( modification_function_parameters=modification_function_parameters, ) assert expected_output in str(e._excinfo[1]) + + +@pytest.mark.parametrize( + "proposed_user_actions", + [[], ["1"], ["1", "2"], ["1", "2", "3"], ["1", "2", "3", "4"]], +) +def test_send_email_with_proposed_user_actions(proposed_user_actions): + actions_count = len(proposed_user_actions) + if actions_count < 1 or actions_count > 3: + with pytest.raises(ValidationError): + actions = Actions(proposed_user_actions=proposed_user_actions) + else: + actions = Actions(proposed_user_actions=proposed_user_actions) + + result = send_email(actions=actions) + assert result["proposed_user_action"] == proposed_user_actions From 0dd4fb2e1c94ca64d040452ca768cdbc2c452e8e Mon Sep 17 00:00:00 2001 From: rjambrecic <32619626+rjambrecic@users.noreply.github.com> Date: Fri, 24 May 2024 08:12:43 +0200 Subject: [PATCH 3/4] Error handling in the create_ad_group_with_ad_and_keywords function implemented (#709) Co-authored-by: Davor Runje Co-authored-by: Kumaran Rajendhiran --- .../tools/_campaign_creation_team_tools.py | 136 +++++++++++++----- .../test_campaign_creation_team_tools.py | 112 +++++++++++---- 2 files changed, 186 insertions(+), 62 deletions(-) diff --git a/captn/captn_agents/backend/tools/_campaign_creation_team_tools.py b/captn/captn_agents/backend/tools/_campaign_creation_team_tools.py index ef7f73b8..4535e82d 100644 --- a/captn/captn_agents/backend/tools/_campaign_creation_team_tools.py +++ b/captn/captn_agents/backend/tools/_campaign_creation_team_tools.py @@ -4,6 +4,8 @@ from pydantic import BaseModel, Field, StringConstraints from typing_extensions import Annotated +from google_ads.model import RemoveResource + from ....google_ads.client import check_for_client_approval, google_ads_create_update from ..toolboxes import Toolbox from ._functions import ( @@ -96,7 +98,7 @@ class AdGroupWithAdAndKeywords(BaseModel): def _get_resource_id_from_response(response: str) -> str: - return response.split("/")[-1].replace(".", "") + return response.split("/")[-1].split("~")[-1].replace(".", "").strip() def _create_ad_group( @@ -178,6 +180,7 @@ def _create_ad_group_keywords( ], ad_group_with_ad_and_keywords: AdGroupWithAdAndKeywords, ad_group_id: str, + list_of_resources: List[RemoveResource], ) -> Union[Dict[str, Any], str]: response = "" for keyword in ad_group_with_ad_and_keywords.keywords: @@ -189,10 +192,101 @@ def _create_ad_group_keywords( ad_group_id=ad_group_id, customer_id=ad_group_with_ad_and_keywords.customer_id, ) + keyword_id = _get_resource_id_from_response(keyword_response) # type: ignore[arg-type] + list_of_resources.append( + RemoveResource( + customer_id=ad_group_with_ad_and_keywords.customer_id, + parent_id=ad_group_id, + resource_id=keyword_id, + resource_type="ad_group_criterion", + ) + ) response += f"Keyword '{keyword.keyword_text}': {keyword_response}\n" return response +def _remove_resources( + user_id: int, conv_id: int, list_of_resources: List[RemoveResource] +) -> None: + for remove_resource in list_of_resources[::-1]: + remove_response = google_ads_create_update( + user_id=user_id, + conv_id=conv_id, + recommended_modifications_and_answer_list=[], + ad=remove_resource, + endpoint="/remove-google-ads-resource", + already_checked_clients_approval=True, + ) + print(remove_response) + print("Resources removed successfully") + + +def _create_ad_group_with_ad_and_keywords( + ad_group_with_ad_and_keywords: Annotated[ + AdGroupWithAdAndKeywords, "An ad group with an ad and a list of keywords" + ], + context: Context, +) -> Union[Dict[str, Any], str]: + list_of_resources = [] + try: + ad_group_response = _create_ad_group( + user_id=context.user_id, + conv_id=context.conv_id, + recommended_modifications_and_answer_list=context.recommended_modifications_and_answer_list, + ad_group_with_ad_and_keywords=ad_group_with_ad_and_keywords, + ) + if isinstance(ad_group_response, dict): + return ad_group_response + response = f"Ad group '{ad_group_with_ad_and_keywords.ad_group.name}': {ad_group_response}\n" + ad_group_id = _get_resource_id_from_response(ad_group_response) + list_of_resources.append( + RemoveResource( + customer_id=ad_group_with_ad_and_keywords.customer_id, + resource_id=ad_group_id, + resource_type="ad_group", + ) + ) + + ad_group_ad_response = _create_ad_group_ad( + user_id=context.user_id, + conv_id=context.conv_id, + recommended_modifications_and_answer_list=context.recommended_modifications_and_answer_list, + ad_group_with_ad_and_keywords=ad_group_with_ad_and_keywords, + ad_group_id=ad_group_id, + ) + ad_group_ad_id = _get_resource_id_from_response(ad_group_ad_response) # type: ignore[arg-type] + list_of_resources.append( + RemoveResource( + customer_id=ad_group_with_ad_and_keywords.customer_id, + parent_id=ad_group_id, + resource_id=ad_group_ad_id, + resource_type="ad", + ) + ) + + response += f"Ad group ad with final url - '{ad_group_with_ad_and_keywords.ad_group_ad.final_url}': {ad_group_ad_response}\n" + + ad_group_keywords_response = _create_ad_group_keywords( + user_id=context.user_id, + conv_id=context.conv_id, + recommended_modifications_and_answer_list=context.recommended_modifications_and_answer_list, + ad_group_with_ad_and_keywords=ad_group_with_ad_and_keywords, + ad_group_id=ad_group_id, + list_of_resources=list_of_resources, + ) + response += ad_group_keywords_response # type: ignore + context.changes_made += f"\n{response}" + except Exception as e: + _remove_resources( + user_id=context.user_id, + conv_id=context.conv_id, + list_of_resources=list_of_resources, + ) + raise e + + return response + + def create_campaign_creation_team_toolbox( user_id: int, conv_id: int, @@ -230,55 +324,21 @@ def create_ad_group_with_ad_and_keywords( """ # TODO: If any of the creation fails, we should rollback the changes # and return the error message (delete ad_group, other resources will be deleted automatically) - - user_id = context.user_id - conv_id = context.conv_id - recommended_modifications_and_answer_list = ( - context.recommended_modifications_and_answer_list - ) - modification_function_parameters = {} modification_function_parameters["ad_group_with_ad_and_keywords"] = ( ad_group_with_ad_and_keywords.model_dump() ) error_msg = check_for_client_approval( modification_function_parameters=modification_function_parameters, - recommended_modifications_and_answer_list=recommended_modifications_and_answer_list, + recommended_modifications_and_answer_list=context.recommended_modifications_and_answer_list, ) if error_msg: raise ValueError(error_msg) - ad_group_response = _create_ad_group( - user_id=user_id, - conv_id=conv_id, - recommended_modifications_and_answer_list=recommended_modifications_and_answer_list, - ad_group_with_ad_and_keywords=ad_group_with_ad_and_keywords, - ) - if isinstance(ad_group_response, dict): - return ad_group_response - response = f"Ad group '{ad_group_with_ad_and_keywords.ad_group.name}': {ad_group_response}\n" - ad_group_id = _get_resource_id_from_response(ad_group_response) - - ad_group_ad_response = _create_ad_group_ad( - user_id=user_id, - conv_id=conv_id, - recommended_modifications_and_answer_list=recommended_modifications_and_answer_list, + return _create_ad_group_with_ad_and_keywords( ad_group_with_ad_and_keywords=ad_group_with_ad_and_keywords, - ad_group_id=ad_group_id, + context=context, ) - response += f"Ad group ad with final url - '{ad_group_with_ad_and_keywords.ad_group_ad.final_url}': {ad_group_ad_response}\n" - - ad_group_keywords_response = _create_ad_group_keywords( - user_id=user_id, - conv_id=conv_id, - recommended_modifications_and_answer_list=recommended_modifications_and_answer_list, - ad_group_with_ad_and_keywords=ad_group_with_ad_and_keywords, - ad_group_id=ad_group_id, - ) - response += ad_group_keywords_response # type: ignore - context.changes_made += f"\n{response}" - - return response add_shared_functions(toolbox) diff --git a/tests/ci/captn/captn_agents/backend/tools/test_campaign_creation_team_tools.py b/tests/ci/captn/captn_agents/backend/tools/test_campaign_creation_team_tools.py index 300eec84..a7350a7c 100644 --- a/tests/ci/captn/captn_agents/backend/tools/test_campaign_creation_team_tools.py +++ b/tests/ci/captn/captn_agents/backend/tools/test_campaign_creation_team_tools.py @@ -11,10 +11,13 @@ AdGroupCriterionForCreation, AdGroupForCreation, AdGroupWithAdAndKeywords, + _get_resource_id_from_response, + _remove_resources, create_campaign_creation_team_toolbox, ) from captn.captn_agents.backend.tools._functions import Context from captn.google_ads.client import clean_nones +from google_ads.model import RemoveResource from .helpers import check_llm_config_descriptions, check_llm_config_total_tools @@ -52,7 +55,38 @@ def test_llm_config(self) -> None: }, ) - def test_create_ad_group_with_ad_and_keywords(self) -> None: + @pytest.mark.parametrize( + "response", + [ + "Created customers/1212/adGroups/3434.", + "Created customers/7119828439/adGroupCriteria/1212~3434.", + ], + ) + def test_get_resource_id_from_response(self, response) -> None: + resource_id = _get_resource_id_from_response(response) + assert resource_id == "3434", resource_id + + @pytest.mark.parametrize( + "side_effect", + [ + [ + "Created customers/1212/adGroups/3434.", + "Created customers/1212/adGroupAds/3434~5656.", + "Created customers/1212/adGroupCriteria/3434~7878.", + "Created customers/1212/adGroupCriteria/3434~8989.", + ], + [ + "Created customers/1212/adGroups/3434.", + "Created customers/1212/adGroupAds/3434~5656.", + "Created customers/1212/adGroupCriteria/3434~7878.", + ValueError("Error: 400: The keyword text 'keyword1' is invalid."), + "Removed customers/1212/adGroupCriteria/3434~7878.", + "Removed customers/1212/adGroupAds/3434~5656.", + "Removed customers/1212/adGroups/3434.", + ], + ], + ) + def test_create_ad_group_with_ad_and_keywords(self, side_effect) -> None: ad_group_ad = AdGroupAdForCreation( final_url="https://www.example.com", headlines=["headline1", "headline2", "headline3"], @@ -86,17 +120,16 @@ def test_create_ad_group_with_ad_and_keywords(self) -> None: keywords=[keyword1, keyword2], ) - with unittest.mock.patch( - "captn.captn_agents.backend.tools._campaign_creation_team_tools.google_ads_create_update" - ) as mock_google_ads_create_update: - side_effect = [ - "Created customers/1212/adGroups/3434.", - "Created customers/1212/adGroupAds/3434~5656.", - "Created customers/1212/adGroupCriteria/3434~7878.", - "Created customers/1212/adGroupCriteria/3434~8989.", - ] - mock_google_ads_create_update.side_effect = side_effect - + with ( + unittest.mock.patch( + "captn.captn_agents.backend.tools._campaign_creation_team_tools.google_ads_create_update", + side_effect=side_effect, + ) as mock_google_ads_create_update, + unittest.mock.patch( + "captn.captn_agents.backend.tools._campaign_creation_team_tools._remove_resources", + wraps=_remove_resources, + ) as mock_remove_resources, + ): modification_function_params = clean_nones( { "ad_group_with_ad_and_keywords": ad_group_with_ad_and_keywords.model_dump() @@ -114,23 +147,54 @@ def test_create_ad_group_with_ad_and_keywords(self) -> None: "create_ad_group_with_ad_and_keywords" ) - response = create_ad_group_with_ad_and_keywords( - ad_group_with_ad_and_keywords=ad_group_with_ad_and_keywords, - context=context, - ) - - expected_response = f"""Ad group '{ad_group.name}': {side_effect[0]} + if isinstance(side_effect[3], ValueError): + with pytest.raises(ValueError): + create_ad_group_with_ad_and_keywords( + ad_group_with_ad_and_keywords=ad_group_with_ad_and_keywords, + context=context, + ) + + # 4 create calls (the last one failed) + 3 remove calls + assert mock_google_ads_create_update.call_count == 7 + mock_remove_resources.assert_called_once_with( + user_id=1, + conv_id=1, + list_of_resources=[ + RemoveResource( + customer_id="1111", + resource_id="3434", + resource_type="ad_group", + ), + RemoveResource( + customer_id="1111", + parent_id="3434", + resource_id="5656", + resource_type="ad", + ), + RemoveResource( + customer_id="1111", + parent_id="3434", + resource_id="7878", + resource_type="ad_group_criterion", + ), + ], + ) + + else: + response = create_ad_group_with_ad_and_keywords( + ad_group_with_ad_and_keywords=ad_group_with_ad_and_keywords, + context=context, + ) + + expected_response = f"""Ad group '{ad_group.name}': {side_effect[0]} Ad group ad with final url - '{ad_group_ad.final_url}': {side_effect[1]} Keyword '{keyword1.keyword_text}': {side_effect[2]} Keyword '{keyword2.keyword_text}': {side_effect[3]} """ - print() - print(f"{response=}") - print(f"{expected_response=}") - - assert mock_google_ads_create_update.call_count == 4 - assert response == expected_response, response + assert mock_google_ads_create_update.call_count == 4 + assert response == expected_response, response + mock_remove_resources.assert_not_called() class TestContext: From cc773cae9bc620a5835e56dd4ac23b163d08db8b Mon Sep 17 00:00:00 2001 From: rjambrecic <32619626+rjambrecic@users.noreply.github.com> Date: Sun, 26 May 2024 15:12:44 +0200 Subject: [PATCH 4/4] 694 increase ad copy and keywords quality (#710) * wip * wip * Example updates --- .../campaign_creation_team_fixtures.py | 182 +++++++----------- .../backend/teams/_campaign_creation_team.py | 33 +++- .../tools/_brief_creation_team_tools.py | 8 +- .../tools/_campaign_creation_team_tools.py | 56 +++++- .../captn_agents/backend/tools/_functions.py | 27 ++- google_ads/model.py | 35 +++- .../tools/test_brief_creation_team_tools.py | 6 +- .../test_campaign_creation_team_tools.py | 169 ++++++++++++---- tests/ci/google_ads/test_model.py | 73 +++++++ 9 files changed, 408 insertions(+), 181 deletions(-) create mode 100644 tests/ci/google_ads/test_model.py diff --git a/captn/captn_agents/backend/benchmarking/fixtures/campaign_creation_team_fixtures.py b/captn/captn_agents/backend/benchmarking/fixtures/campaign_creation_team_fixtures.py index 0d8aa167..40183968 100644 --- a/captn/captn_agents/backend/benchmarking/fixtures/campaign_creation_team_fixtures.py +++ b/captn/captn_agents/backend/benchmarking/fixtures/campaign_creation_team_fixtures.py @@ -43,73 +43,37 @@ Additional info from the web page: Here is a summary of the information you requested: -IKEA UK offers a wide range of home furnishing products including furniture, beds & mattresses, kitchens, decorations, lighting, textiles, kitchenware & tableware, laundry & cleaning supplies, and bathroom products. The brand is known for its affordable prices, flat-pack delivery, and self-assembly furniture, catering to various styles and preferences. +IKEA UK offers a wide range of home furnishing products including furniture, smart home devices, and business solutions. They provide stylish and affordable furniture for every room, innovative smart home products, and specialized solutions for businesses. Relevant Pages: -1. Furniture: -URL: https://www.ikea.com/gb/en/cat/furniture-fu001/ -Summary: A variety of home furniture for living room, bedroom, office, and more. -Keywords: 'Sofas', 'Chairs', 'Tables', 'Wardrobes', 'Storage', 'Office Furniture', 'Outdoor Furniture' -Headlines: 'Comfy Sofas', 'Ergonomic Chairs', 'Stylish Tables', 'Spacious Storage', 'Modern Wardrobes', 'Work Desks', 'Outdoor Sets' -Descriptions: 'Find the perfect sofa', 'Chairs for any style', 'Tables for all spaces', 'Solve storage needs', 'Wardrobes for order', 'Desks for productivity', 'Enjoy outdoor living' - -2. Beds & Mattresses: -URL: https://www.ikea.com/gb/en/cat/beds-mattresses-bm001/ -Summary: Comfortable beds and mattresses for a good night's sleep. -Keywords: 'Beds', 'Mattresses', 'Bed Frames', "Children's Beds", 'Mattress Toppers', 'Pillows', 'Bedding' -Headlines: 'Dreamy Mattresses', 'Cozy Beds', "Kids' Bed Frames", 'Sleep Well', 'Top Mattress Picks', 'Soft Pillows', 'Quality Bedding' -Descriptions: 'Mattresses for rest', 'Beds for all ages', "Fun children's beds", 'Ensure a restful sleep', 'Topper for comfort', 'Pillows for support', 'Bedding for warmth' - -3. Kitchens: -URL: https://www.ikea.com/gb/en/cat/kitchens-ka001/ -Summary: Complete kitchen solutions and accessories. -Keywords: 'Kitchen Units', 'Cabinets', 'Worktops', 'Kitchen Appliances', 'Sinks', 'Taps', 'Kitchen Storage' -Headlines: 'Modern Kitchens', 'Spacious Cabinets', 'Durable Worktops', 'Appliances for Chefs', 'Stainless Sinks', 'Functional Taps', 'Organized Storage' -Descriptions: 'Design your kitchen', 'Cabinets for storage', 'Worktops for prep', 'Appliances that last', 'Sinks for cleanup', 'Taps that impress', 'Storage solutions' - -4. Decoration: -URL: https://www.ikea.com/gb/en/cat/decoration-de001/ -Summary: Decorative items to personalize your space. -Keywords: 'Candles', 'Vases', 'Picture Frames', 'Mirrors', 'Clocks', 'Ornaments', 'Wall Art' -Headlines: 'Scented Candles', 'Elegant Vases', 'Frame Memories', 'Stylish Mirrors', 'Timely Clocks', 'Decorative Pieces', 'Artistic Wall Art' -Descriptions: 'Candles for ambiance', 'Vases for decor', 'Frames for photos', 'Mirrors for rooms', 'Clocks for time', 'Ornaments for style', 'Wall art for flair' - -5. Lighting: -URL: https://www.ikea.com/gb/en/cat/lighting-li001/ -Summary: Lighting options for every room in your home. -Keywords: 'Ceiling Lights', 'Lamps', 'LED Lighting', 'Smart Lighting', 'Wall Lights', 'Outdoor Lighting', 'Decorative Lights' -Headlines: 'Bright Ceiling', 'Lamps for Tasks', 'LED Solutions', 'Smart Light Tech', 'Wall Illumination', 'Garden Lights', 'Decor Lights' -Descriptions: 'Ceiling lights for all', 'Lamps for reading', 'LED for efficiency', 'Smart lighting control', 'Wall lights for mood', 'Outdoor light options', 'Lights for atmosphere' - -6. Textiles: -URL: https://www.ikea.com/gb/en/cat/textiles-tl001/ -Summary: Soft textiles to enhance comfort and appearance. -Keywords: 'Curtains', 'Rugs', 'Cushions', 'Bed Linen', 'Towels', 'Throws', 'Fabric' -Headlines: 'Elegant Curtains', 'Cozy Rugs', 'Comfy Cushions', 'Soft Bed Linen', 'Absorbent Towels', 'Warm Throws', 'Quality Fabrics' -Descriptions: 'Curtains for privacy', 'Rugs for warmth', 'Cushions for sofas', 'Bed linen for sleep', 'Towels for baths', 'Throws for snuggling', 'Fabrics for projects' - -7. Kitchenware & Tableware: -URL: https://www.ikea.com/gb/en/cat/kitchenware-tableware-kt001/ -Summary: Essentials for cooking and dining. -Keywords: 'Cookware', 'Dinnerware', 'Glassware', 'Utensils', 'Serveware', 'Cutlery', 'Food Storage' -Headlines: 'Cooking Made Easy', 'Dine in Style', 'Glassware Galore', 'Utensils for Chefs', 'Serve with Pride', 'Cutlery Sets', 'Store Fresh Food' -Descriptions: 'Cookware for meals', 'Dinnerware for feasts', 'Glassware for drinks', 'Utensils for prep', 'Serveware for hosting', 'Cutlery for dining', 'Food storage options' - -8. Laundry & Cleaning: -URL: https://www.ikea.com/gb/en/cat/laundry-cleaning-lc001/ -Summary: Products to keep your home clean and organized. -Keywords: 'Laundry Baskets', 'Cleaning Supplies', 'Ironing Boards', 'Waste Bins', 'Organizers', 'Drying Racks', 'Cleaning Accessories' -Headlines: 'Laundry Solutions', 'Clean & Tidy', 'Ironing Ease', 'Waste Disposal', 'Space Organizers', 'Drying Made Simple', 'Cleaning Tools' -Descriptions: 'Baskets for laundry', 'Supplies for cleaning', 'Ironing board options', 'Bins for waste', 'Organizers for order', 'Racks for drying', 'Tools for cleanliness' - -9. Bathroom Products: -URL: https://www.ikea.com/gb/en/cat/bathroom-products-ba001/ -Summary: A range of products for your bathroom needs. -Keywords: 'Bathroom Storage', 'Shower Accessories', 'Bath Mats', 'Bathroom Mirrors', 'Towel Rails', 'Bathroom Lighting', 'Bathroom Decor' -Headlines: 'Storage for Baths', 'Shower Must-Haves', 'Soft Bath Mats', 'Clear Mirrors', 'Towel Hang-Ups', 'Bathroom Brightness', 'Decor Touches' -Descriptions: 'Storage for toiletries', 'Shower accessories', 'Mats for safety', 'Mirrors for clarity', 'Rails for towels', 'Lighting for bathrooms', 'Decor for style' - +1. Offers - Discounts: + URL: https://www.ikea.com/gb/en/offers/ + Summary: Current promotions and discounts on IKEA products + Keywords: 'IKEA Discounts', 'Furniture Sales', 'Home Decor Deals', 'Affordable Furnishings', 'Special Offers' + Headlines: 'Save Big on IKEA Finds', 'Deals on Stylish Furniture', 'Discounted Home Essentials', 'IKEA Sales Event', 'Affordable Home Makeover', 'Exclusive IKEA Offers', 'Budget-Friendly Decor', 'Furnish for Less', 'Home Savings Galore', 'Decor Deals Await', 'IKEA Markdowns', 'Furniture on a Budget', 'Shop IKEA Specials', 'Redecorate for Less', '{Keyword: IKEA Savings}' + Descriptions: 'Discover IKEA discounts on furniture.', 'Upgrade your home for less today.', 'Find deals on IKEA home essentials.', 'Save on decor with IKEA offers.' + +2. Furniture: + URL: https://www.ikea.com/gb/en/cat/furniture-fu001/ + Summary: Wide range of furniture for every room + Keywords: 'Living Room Furniture', 'Bedroom Sets', 'Office Desks', 'IKEA Sofas', 'Dining Tables' + Headlines: 'Chic Sofas & Sectionals', 'Bedroom Furniture Deals', 'Stylish Office Desks', 'Dine in Style with IKEA', 'Modern Living Room Sets', 'Cozy Bedroom Collections', 'Work from Home Solutions', 'Dining Sets for Less', 'Sofas for Every Style', 'Elegant Bedroom Designs', 'Office Furniture Sale', 'Living Room Makeover', 'Dreamy Bedroom Finds', 'IKEA Dining Essentials', '{Keyword: IKEA Furniture}' + Descriptions: "Explore IKEA's furniture range.", 'Furnish your bedroom with IKEA.', 'Find the perfect office desk.', 'Dining solutions for every home.' + +3. Smart Home: + URL: https://www.ikea.com/gb/en/cat/smart-home-hs001/ + Summary: Innovative smart home devices + Keywords: 'Smart Lighting', 'Home Automation', 'IKEA Smart Blinds', 'Wireless Charging', 'Smart Speakers' + Headlines: 'Illuminate with Smart Lights', 'Automate Your Home', 'Smart Blinds for Privacy', 'Charge Wirelessly at Home', 'IKEA Smart Home Tech', 'Lighting at Your Fingertips', 'Home Automation Made Easy', 'Smart Blinds on Sale', 'Tech-Savvy Home Solutions', 'Wireless Charging Furniture', 'Sound of Smart Speakers', 'Smart Home for Less', 'IKEA Tech Upgrades', 'Automated Home Essentials', '{Keyword: Smart Living}' + Descriptions: 'Smart home devices by IKEA.', 'Control lighting with a tap.', 'Smart blinds for modern homes.', "IKEA's wireless charging tech." + +4. IKEA for Business: + URL: https://www.ikea.com/gb/en/ikea-business/ + Summary: Business solutions to save time and money + Keywords: 'Business Furniture', 'Office Design', 'Corporate Discounts', 'Workspace Solutions', 'Commercial Interiors' + Headlines: 'Furnish Your Business', 'Office Design Solutions', 'IKEA Corporate Offers', 'Workspace Furnishing', 'Stylish Commercial Interiors', 'Business Decor Savings', 'Efficient Office Layouts', 'Discounted Business Range', 'Corporate Furniture Deals', 'Workspace Design Ideas', 'Business Essentials Sale', 'Office Makeover with IKEA', 'Commercial Space Upgrades', 'IKEA Business Discounts', '{Keyword: Office Furniture}' + Descriptions: 'IKEA solutions for businesses.', 'Design your office space now.', 'Corporate furniture discounts.', 'Upgrade workspaces with IKEA.' And the task is following: Create a new Google Ads campaign for the Furniture category @@ -145,34 +109,33 @@ Additional info from the web page: Here is a summary of the information you requested: -Disney Store EU features a vast collection of Disney, Marvel, Pixar, and Star Wars merchandise, including new arrivals, seasonal gifts, and exclusive offers. Key selling points include free delivery on orders over 60€, free returns, and multiple click & collect options. +Disney Store EU features a wide range of Disney, Marvel, Pixar, and Star Wars merchandise, including toys, clothing, and collectibles. Relevant Pages: -1. New In Products: -URL: https://www.disneystore.eu/new -Summary: Showcases the latest products across various Disney franchises. -Keywords: 'New Arrivals', 'Disney', 'Marvel', 'Pixar', 'Star Wars', 'Latest Merchandise', 'Exclusive Products' -Headlines: 'Latest Disney Finds', 'New Marvel Gear', "Pixar's Newest", 'Star Wars Arrivals', 'Fresh Disney Magic', 'Just In: Disney', 'Marvel Must-Haves', 'Pixar Picks', 'Star Wars Selection', 'Newest Arrivals', 'Disney Delights', "Marvel's Best", "Pixar's Finest", 'Star Wars Treasures', "Disney's Latest" -Descriptions: 'Explore the latest Disney merchandise and collectibles.', 'Discover new Marvel apparel and accessories.', 'Find the newest Pixar-themed items.', 'Get the latest Star Wars products.' - -2. Easter Gifts & Toys: -URL: https://www.disneystore.eu/easter?srule=newest -Summary: A selection of Easter-themed Disney gifts and toys. -Keywords: 'Easter', 'Disney Gifts', 'Toys', 'Books', 'Soft Toys', 'Seasonal Offers', 'Themed Merchandise' -Headlines: 'Easter Disney Fun', 'Springtime Surprises', 'Easter Toy Magic', 'Disney Easter Gifts', 'Bunny-Approved Toys', 'Easter Book Finds', 'Soft Toy Delights', 'Easter Basket Joy', 'Disney Spring Gifts', 'Easter Plush Pals', 'Festive Disney Toys', 'Easter Reads', 'Soft Toy Wonders', 'Easter Treasures', 'Spring Disney Picks' -Descriptions: 'Fill your Easter with Disney books and soft toys.', 'Discover the perfect Disney Easter gifts for kids.', 'Hop into Easter with Disney-themed toys and books.', 'Celebrate Easter with delightful Disney soft toys.' - -3. Offers: -URL: https://www.disneystore.eu/offers -Summary: Current discounts and promotions on Disney merchandise. -Keywords: 'Disney Offers', 'Discounts', 'Promotions', 'Deals', 'Special Offers', 'Savings', 'Exclusive Deals' -Headlines: 'Disney Deals', 'Marvel Discounts', 'Pixar Promos', 'Star Wars Savings', 'Special Offers', 'Disney Bargains', 'Marvel Offers', 'Pixar Savings', 'Star Wars Deals', 'Exclusive Discounts', 'Disney Promotions', 'Marvel Savings', 'Pixar Deals', 'Star Wars Offers', 'Disney Specials' -Descriptions: 'Save on Disney favorites with exclusive offers.', 'Get discounts on Marvel, Pixar, and Star Wars gear.', 'Discover promotions on a wide range of Disney products.', 'Enjoy special deals on Disney merchandise.' - +1. New Arrivals: + URL: https://www.disneystore.eu/new + Summary: Latest Disney merchandise arrivals + Keywords: 'New Disney Toys', 'Latest Marvel Gear', 'Pixar New Releases', 'Star Wars Merchandise', 'Disney Clothing', 'Disney Accessories', 'Disney Collectibles', 'New Disney Products', 'Marvel New Arrivals', 'Pixar Latest', 'Star Wars New Gear', 'Disney Store Exclusives', 'New Disney Plush', 'Disney Figures', 'Disney Apparel' + Headlines: 'New Disney Magic', 'Marvel Gear Unveiled', "Pixar's Latest Hits", 'New Star Wars Gear', 'Disney Style Arrivals', 'Accessorize with Disney', 'Collect Disney Treasures', 'Fresh Disney Finds', "Marvel's Newest", "Pixar's Finest Selection", 'Star Wars Essentials', 'Exclusive Disney Items', '{Keyword:Disney Plush}', 'Disney Figurines', 'Wear Disney Fashion' + Descriptions: 'Discover the latest Disney toys', 'Suit up with new Marvel gear', 'Explore new Pixar releases', 'Get the latest Star Wars merch' + +2. Special Offers: + URL: https://www.disneystore.eu/offers + Summary: Discounts and promotions on Disney products + Keywords: 'Disney Deals', 'Marvel Discounts', 'Pixar Promotions', 'Star Wars Offers', 'Disney Sale', 'Disney Bargains', 'Marvel Specials', 'Pixar Savings', 'Star Wars Deals', 'Disney Discount Code', 'Marvel Sale Items', 'Pixar Offer Codes', 'Star Wars Savings', 'Disney Store Promos', 'Disney Clearance' + Headlines: 'Disney Deals Galore', 'Marvel Markdowns', 'Pixar Promos', 'Star Wars Specials', 'Disney Sale Now On', 'Bargain Disney Buys', "Marvel's Best Deals", 'Save on Pixar', 'Star Wars Discounts', 'Use Disney Code', 'Marvel Sale Picks', "Pixar's Best Offers", 'Star Wars Price Cuts', 'Disney Store Savings', '{Keyword:Disney Clearance}' + Descriptions: 'Save big on Disney favorites', 'Discounted Marvel merchandise', 'Special Pixar promotions', 'Exclusive Star Wars offers' + +3. Magical Offers: + URL: https://www.disneystore.eu/offers/magical-offers + Summary: Exclusive deals and free gifts with purchase + Keywords: 'Disney Magical Deals', 'Marvel Free Gifts', 'Pixar Exclusive Offers', 'Star Wars Bonus', 'Disney Promo Gifts', 'Disney Code Savings', 'Marvel Gift Offers', 'Pixar Special Deals', 'Star Wars Freebies', 'Disney Gift Promotions', 'Marvel Exclusive Deals', 'Pixar Bonus Gifts', 'Star Wars Promotions', 'Disney Store Codes', 'Disney Special Offers' + Headlines: 'Magical Disney Deals', 'Marvel Gift with Purchase', "Pixar's Exclusive Deals", 'Star Wars Bonuses', 'Disney Gift Offers', 'Save with Disney Codes', "Marvel's Special Gifts", 'Pixar Deal Bonanza', 'Star Wars Free Gift', 'Disney Promo Specials', 'Marvel Deals', "Pixar's Gift Offers", 'Star Wars Promo', 'Disney Store Specials', '{Keyword:Disney Offers}' + Descriptions: 'Exclusive Disney offers await', 'Free Marvel gift specials', 'Pixar deals with bonuses', 'Star Wars promotions' And the task is following: -Create a new Google Ads campaign focusing on the 'New In Products' and 'Easter Gifts & Toys' sections of the Disney Store EU website. +Create a new Google Ads campaign focusing on the 'New In Products' and 'Magical Offers' sections of the Disney Store EU website. """ CAMPAIGN_CREATION_FASTSTREAM = """Here is the customer brief: @@ -188,31 +151,30 @@ Additional info from the web page: Here is a summary of the information you requested: -FastStream is an open-source framework for building asynchronous web services that interact with event streams, offering a unified API for multiple message brokers and features like Pydantic validation, automatic documentation, and a powerful DI system. +FastStream is an open-source framework for building asynchronous web services that interact with event streams. It offers features such as multiple broker support, Pydantic validation, automatic AsyncAPI documentation, and compatibility with HTTP frameworks like FastAPI. Relevant Pages: -1. Features - FastStream: -URL: https://faststream.airt.ai/latest/faststream/ -Summary: Details the features of FastStream, including multiple brokers support, Pydantic validation, and automatic AsyncAPI documentation. -Keywords: 'FastStream', 'Asynchronous', 'Event Streams', 'API', 'Message Brokers', 'Pydantic', 'Documentation', 'Dependency Injection', 'Testing', 'Extensible', 'Integrations', 'Code Generation', 'GPT', 'Llama' -Headlines: 'Unified API', 'Pydantic Support', 'Auto Documentation', 'DI System', 'In-Memory Tests', 'HTTP Integrations', 'Code Generation', 'Multiple Brokers', 'Testable Services', 'Extensible Framework', 'FastAPI Compatible', 'Modern Microservices', 'Async Services', 'Streamline Workflow', 'Community Driven' -Descriptions: 'Build async web services', 'Validate messages easily', 'Generate docs automatically', 'Efficient dependency management' - -2. Getting Started - FastStream: -URL: https://faststream.airt.ai/latest/getting-started/ -Summary: Provides a quick start guide and basic usage instructions for FastStream. -Keywords: 'Getting Started', 'FastStream', 'Quick Start', 'Basic Usage', 'Installation', 'Tutorial', 'Setup Guide', 'Event Streaming', 'Message Brokers', 'Python', 'Framework', 'Asynchronous', 'Web Services', 'Documentation' -Headlines: 'Quick Start', 'Install Guide', 'Basic Usage', 'Setup Tutorial', 'FastStream Setup', "Beginner's Guide", 'Quick Installation', 'Usage Instructions', 'Service Creation', 'Run Applications', 'Project Setup', 'Python Environment', 'Development Guide', 'Streaming Basics', 'Broker Containers' -Descriptions: 'Start with FastStream', 'Easy installation steps', 'Learn basic FastStream usage', 'Set up your project quickly' - -3. Reference - Code API - FastStream: -URL: https://faststream.airt.ai/latest/api/ -Summary: Contains the reference code API for FastStream, detailing classes, functions, and components. -Keywords: 'Code API', 'Reference', 'FastStream', 'Documentation', 'Classes', 'Functions', 'Components', 'Framework', 'Asynchronous', 'Web Services', 'API Reference', 'Python', 'Developers', 'Technical Details', 'Source Code' -Headlines: 'API Reference', 'Code Documentation', 'Framework Components', 'Class Details', 'Function Guide', 'API Components', 'Technical Reference', 'Source Code', 'Python Classes', 'Framework Functions', 'Developer Guide', 'Code Insights', 'API Overview', 'Component Specs', 'Code Examples' -Descriptions: 'Detailed API documentation', 'Explore FastStream components', 'Technical reference for devs', "Guide to framework's API" - +1. FastStream Main Page: + URL: https://faststream.airt.ai/ + Summary: Overview of FastStream's capabilities and features for building asynchronous web services. + Keywords: 'Asynchronous Web Services', 'Event Stream Framework', 'Message Queue Automation', 'Microservices Development', 'Broker Support', 'Pydantic Validation', 'AsyncAPI Documentation', 'HTTP Framework Compatibility', 'Dependency Injection System', 'In-Memory Testing', 'Framework Extensibility', 'Automatic Code Generation', 'Data-Centric Microservices', 'FastAPI Integration', 'Open-Source Framework' + Headlines: 'Async Services Simplified', 'Queue Automation Made Easy', 'Microservices Development', 'Broker Integration Support', 'Pydantic Message Validation', 'Generate AsyncAPI Docs', 'Catch Errors Before Runtime', 'Efficient Dependency Injection', 'Faster CI/CD with In-Memory Tests', 'Extend Framework Capabilities', 'Compatible with FastAPI', 'Auto Code Generation Ready', '{Keyword:Open-Source Framework}', 'Data-Centric Service Design', 'Streamline Your Workflow' + Descriptions: 'Build async services efficiently.', 'Automate queues with ease.', 'Develop microservices faster.', 'Seamless broker compatibility.' + +2. FastStream Kafka Support: + URL: https://faststream.airt.ai/kafka/kafka/ + Summary: Information on FastStream's support for Kafka, including features and integration. + Keywords: 'Kafka Integration', 'FastStream Kafka API', 'Kafka Message Queues', 'Kafka Asynchronous Services', 'Kafka Broker Support', 'Kafka Pydantic Validation', 'Kafka AsyncAPI Docs', 'Kafka Dependency Injection', 'Kafka In-Memory Testing', 'Kafka Framework Extensibility', 'Kafka Code Generation', 'Kafka Microservices Development', 'Kafka Open-Source Framework', 'Kafka FastAPI Compatibility', 'Kafka Workflow Streamlining' + Headlines: 'Integrate Kafka Seamlessly', 'Kafka Queue Management', 'Async Kafka Microservices', 'Kafka Broker API Support', 'Validate Kafka Messages', 'Kafka AsyncAPI Generation', 'Error-Free Kafka Services', 'Kafka Dependency Management', 'Test Kafka In-Memory', 'Extend Kafka Capabilities', 'Kafka with FastAPI', 'Kafka Code Gen Optimized', '{Keyword:Kafka Framework}', 'Kafka Service Acceleration', 'Streamline Kafka Workflows' + Descriptions: 'Seamless Kafka integration.', 'Manage Kafka queues easily.', 'Accelerate Kafka service dev.', 'Full Kafka broker compatibility.' + +3. FastStream RabbitMQ Support: + URL: https://faststream.airt.ai/rabbit/ + Summary: Details on FastStream's RabbitMQ support, including API features and usage. + Keywords: 'RabbitMQ Integration', 'FastStream RabbitMQ API', 'RabbitMQ Message Queues', 'RabbitMQ Asynchronous Services', 'RabbitMQ Broker Support', 'RabbitMQ Pydantic Validation', 'RabbitMQ AsyncAPI Docs', 'RabbitMQ Dependency Injection', 'RabbitMQ In-Memory Testing', 'RabbitMQ Framework Extensibility', 'RabbitMQ Code Generation', 'RabbitMQ Microservices Development', 'RabbitMQ Open-Source Framework', 'RabbitMQ FastAPI Compatibility', 'RabbitMQ Workflow Streamlining' + Headlines: 'RabbitMQ Integration Easy', 'Manage RabbitMQ Queues', 'Async RabbitMQ Services', 'RabbitMQ Broker API Ready', 'Validate RabbitMQ Messages', 'Generate RabbitMQ AsyncAPI', 'Error-Free RabbitMQ Dev', 'RabbitMQ Dependency Injection', 'Test RabbitMQ In-Memory', 'Extend RabbitMQ Framework', 'RabbitMQ with FastAPI', 'Optimized RabbitMQ Code Gen', '{Keyword:RabbitMQ Framework}', 'RabbitMQ Dev Acceleration', 'Streamline RabbitMQ Workflow' + Descriptions: 'Easy RabbitMQ integration.', 'Simplify RabbitMQ management.', 'Develop RabbitMQ services fast.', 'Compatible with RabbitMQ brokers.' And the task is following: -Create a new Google Ads campaign focusing on the 'Features' and 'Getting Started' pages of the FastStream website.""" +Create a new Google Ads campaign focusing on the 'Features' and 'Tutorial' pages of the FastStream website.""" diff --git a/captn/captn_agents/backend/teams/_campaign_creation_team.py b/captn/captn_agents/backend/teams/_campaign_creation_team.py index 9484a026..d3be606c 100644 --- a/captn/captn_agents/backend/teams/_campaign_creation_team.py +++ b/captn/captn_agents/backend/teams/_campaign_creation_team.py @@ -21,19 +21,35 @@ ad_group_ad = AdGroupAdForCreation( final_url="https://www.example.com", - headlines=["headline1", "headline2", "headline3"], - descriptions=["description1", "description2"], + headlines=[ + "headline1", + "headline2", + "headline3", + "headline4", + "headline5", + "headline6", + "headline7", + "headline8", + "headline9", + "headline10", + "headline11", + "headline12", + "headline13", + "{KeyWord:your default text1}", + "{KeyWord:your default text2}", + ], + descriptions=["description1", "description2", "description3", "description4"], status="ENABLED", ) keyword1 = AdGroupCriterionForCreation( keyword_text="keyword1", - keyword_match_type="EXACT", + keyword_match_type="BROAD", status="ENABLED", ) keyword2 = AdGroupCriterionForCreation( keyword_text="keyword2", - keyword_match_type="EXACT", + keyword_match_type="BROAD", status="ENABLED", ) @@ -186,9 +202,12 @@ def _guidelines(self) -> str: 32. Ad Copy headlines can have MAXIMUM 30 characters and descriptions can have MAXIMUM 90 characters, NEVER suggest headlines/descriptions which exceed that length or you will be penalized! 33. If the client sends you invalid headline/description, do not try to modify it yourself! Explain the problems to him and suggest valid headline/description. 34. Ad rules: -- MINIMUM 3 and MAXIMUM 15 headlines. -- MINIMUM 2 and MAXIMUM 4 descriptions. -It is recommended to use the MAXIMUM number of headlines and descriptions. So if not explicitly told differently, suggest adding 15 headlines and 4 descriptions! +- Ad MUST contain 15 headlines and 4 descriptions +- Do NOT create general keywords. Keywords must be specific so they attract the right customers. +- Use keyword insertion in the headlines! +- headlines MUST be UNIQUE, do NOT repeat the same or similar phrases +- descriptions MUST be UNIQUE, do NOT repeat the same or similar phrases +- Use longer descriptions to provide more information about the product (but MAX 90 characters) 35. When replying to the client, try to finish the message with a question, that way you will navigate the client what to do next 36. Before setting any kind of budget, check the default currency from the customer table and convert the budget to that currency. You can use the following query for retrieving the local currency: SELECT customer.currency_code FROM customer WHERE customer.id = '1212121212' diff --git a/captn/captn_agents/backend/tools/_brief_creation_team_tools.py b/captn/captn_agents/backend/tools/_brief_creation_team_tools.py index 249d6d2f..5dc70ece 100644 --- a/captn/captn_agents/backend/tools/_brief_creation_team_tools.py +++ b/captn/captn_agents/backend/tools/_brief_creation_team_tools.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import Dict, Optional, Type +from typing import Dict, Type from pydantic import BaseModel, Field from typing_extensions import Annotated @@ -22,7 +22,7 @@ class Context: conv_id: int initial_brief: str function_call_counter: Dict[str, bool] = field(default_factory=dict) - get_info_from_web_page_result: Optional[str] = None + get_info_from_web_page_result: str = "" class DelegateTask(BaseModel): @@ -129,7 +129,7 @@ def delagate_task( context: Context, ) -> str: get_info_from_web_page_result = context.get_info_from_web_page_result - if get_info_from_web_page_result is None: + if get_info_from_web_page_result == "": return "You need to scrape the web page first by using the get_info_from_the_web_page command." user_id = context.user_id @@ -165,7 +165,7 @@ def get_info_from_the_web_page( result = _get_info_from_the_web_page_original(url) if LAST_MESSAGE_BEGINNING in result: - context.get_info_from_web_page_result = result + context.get_info_from_web_page_result += result + "\n\n" result += """\n\nPlease use the rely_to_client to present what you have found on the web page to the client. diff --git a/captn/captn_agents/backend/tools/_campaign_creation_team_tools.py b/captn/captn_agents/backend/tools/_campaign_creation_team_tools.py index 4535e82d..cecb18ff 100644 --- a/captn/captn_agents/backend/tools/_campaign_creation_team_tools.py +++ b/captn/captn_agents/backend/tools/_campaign_creation_team_tools.py @@ -1,10 +1,10 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Union from annotated_types import Len -from pydantic import BaseModel, Field, StringConstraints +from pydantic import BaseModel, Field, field_validator from typing_extensions import Annotated -from google_ads.model import RemoveResource +from google_ads.model import RemoveResource, check_max_string_length_for_each_item from ....google_ads.client import check_for_client_approval, google_ads_create_update from ..toolboxes import Toolbox @@ -39,12 +39,22 @@ class AdGroupAdForCreation(AdBase): str, Field(..., description=properties_config["final_url"]["description"]) ] headlines: Annotated[ - List[Annotated[str, StringConstraints(max_length=30)]], - Len(min_length=3, max_length=15), + List[ + Annotated[ + str, + "Maximum 30 characters. If keyword insertion is used, '{KeyWord' and '}' are NOT included in the 30 characters.", + ] + ], + Len(min_length=15, max_length=15), ] descriptions: Annotated[ - List[Annotated[str, StringConstraints(max_length=90)]], - Len(min_length=2, max_length=4), + List[ + Annotated[ + str, + "Maximum 90 characters. If keyword insertion is used, '{KeyWord' and '}' are NOT included in the 90 characters.", + ] + ], + Len(min_length=4, max_length=4), ] path1: Optional[ Annotated[ @@ -57,6 +67,37 @@ class AdGroupAdForCreation(AdBase): ] ] = None + @classmethod + def validate_field( + cls, + field_name: str, + field: List[str], + max_string_length: int, + ) -> Optional[List[str]]: + error_message = check_max_string_length_for_each_item( + field_name=field_name, field=field, max_string_length=max_string_length + ) + + if error_message: + raise ValueError(error_message) + return field + + @field_validator("headlines") + def headlines_validator(cls, headlines: List[str]) -> Optional[List[str]]: + return cls.validate_field( + field_name="headlines", + field=headlines, + max_string_length=30, + ) + + @field_validator("descriptions") + def descriptions_validator(cls, descriptions: List[str]) -> Optional[List[str]]: + return cls.validate_field( + field_name="descriptions", + field=descriptions, + max_string_length=90, + ) + class AdGroupCriterionForCreation(AdBase): ad_group_id: Optional[ @@ -307,7 +348,8 @@ def create_campaign_creation_team_toolbox( @toolbox.add_function("Create an ad group with a single ad and a list of keywords") def create_ad_group_with_ad_and_keywords( ad_group_with_ad_and_keywords: Annotated[ - AdGroupWithAdAndKeywords, "An ad group with an ad and a list of keywords" + AdGroupWithAdAndKeywords, + "An ad group with an ad (15 headlines and 4 descriptions) and a list of keywords", ], # the context will be injected by the toolbox context: Context, diff --git a/captn/captn_agents/backend/tools/_functions.py b/captn/captn_agents/backend/tools/_functions.py index 0748e8a9..c09a8560 100644 --- a/captn/captn_agents/backend/tools/_functions.py +++ b/captn/captn_agents/backend/tools/_functions.py @@ -95,12 +95,18 @@ class Summary(BaseModel): "Quiet Operation", ], headlines=[ - "Cool Comfort, Anywhere", - "Beat the Heat with Ease", - "Stay Chill All Summer", - "Your Cooling Solution", - "Ultimate Climate Control", - "AC: Your Hot Weather Ally", + "Cool Comfort for Your Home", + "Efficient Cooling Solutions", + "Smart Air Conditioning", + "Whisper-Quiet Operation", + "Energy-Efficient Cooling", + "Stay Cool and Comfortable", + "{Keyword:Stay Cool}", + "{Keyword:Smart Cooling}", + "Home Comfort Solutions", + "Cool Your Space Efficiently", + "Save money", + "ECO-Friendly", ], descriptions=[ "Powerful cooling for any space", @@ -130,6 +136,8 @@ class Summary(BaseModel): "Smart Refrigeration", "Stylish Cooling", "Space-Saving Solutions", + "{Keyword:Fresh Food}", + "{Keyword:Smart Cooling}", ], descriptions=[ "Keep your food fresh longer", @@ -525,6 +533,12 @@ def _create_web_surfer_navigator_system_message(task_guidelines: str) -> str: VERY IMPORTANT: - each headline must have LESS than 30 characters and each description must have LESS than 90 characters - generate EXACTLY 15 headlines and 4 descriptions for EACH link which you think is relevant for the Google Ads campaign! +- Do NOT create general keywords. Keywords must be specific so they attract the right customers. +- Include popular keywords in your headlines! +- headlines MUST be UNIQUE, do NOT repeat the same or similar phrases +- use at least ONE keyword insertion in the headlines list e.g. '{{Keyword: Your default text here}}' +- descriptions MUST be UNIQUE, do NOT repeat the same or similar phrases +- Use longer descriptions to provide more information about the product (but MAX 90 characters) - if not explicitly told, do NOT include links like 'About Us', 'Contact Us' etc. in the summary. We are interested ONLY in the products/services which the page is offering. - NEVER include in the summary links which return 40x error! @@ -589,6 +603,7 @@ def get_webpage_status_code(url: str) -> Optional[int]: AFTER visiting the home page, create a step-by-step plan BEFORE visiting the other pages. You can click on MAXIMUM 10 links. Do NOT try to click all the links on the page, but only the ones which are most relevant for the task (MAX 10)! When clicking on a link, add a comment "Click no. X (I can click MAX 10 links, but I will click only the most relevant ones, once I am done, I need to generate JSON-encoded string)" to the message. +Make sure you use keyword insertion in the headlines and provide unique headlines and descriptions for each link. """ _task_guidelines = "Please provide a summary of the website, including the products/services offered and any unique selling points." diff --git a/google_ads/model.py b/google_ads/model.py index fbf065cd..167d152f 100644 --- a/google_ads/model.py +++ b/google_ads/model.py @@ -24,6 +24,27 @@ class AdGroup(Campaign): cpc_bid_micros: Optional[int] = None +def _remove_keyword_insertion_chars(text: str) -> str: + text = text.lower() + if "{keyword:" in text and text[-1] == "}": + text = text.replace("{keyword:", "") + # remove last character + text = text[:-1].strip() + return text + + +def check_max_string_length_for_each_item( + field_name: str, field: List[str], max_string_length: int +) -> str: + error_message = "" + for item in field: + item_without_keyword_insertion_chars = _remove_keyword_insertion_chars(item) + if len(item_without_keyword_insertion_chars) > max_string_length: + error_message += f"Each {field_name} must be less than {max_string_length} characters!\nLength of {item} is {len(item)}" + + return error_message + + class AdGroupAd(AdGroup): ad_id: Optional[str] = None final_url: Optional[str] = None @@ -40,17 +61,19 @@ def validate_field( min_list_length: int, max_string_length: int, ) -> Optional[List[str]]: + error_message = "" if field is not None: if len(field) < min_list_length: - raise ValueError( + error_message += ( f"{field_name} must have at least {min_list_length} items!" ) - for item in field: - if len(item) > max_string_length: - raise ValueError( - f"Each {field_name} must be less than {max_string_length} characters!\nLength of {item} is {len(item)}" - ) + error_message += check_max_string_length_for_each_item( + field_name=field_name, field=field, max_string_length=max_string_length + ) + + if error_message: + raise ValueError(error_message) return field @field_validator("headlines") diff --git a/tests/ci/captn/captn_agents/backend/tools/test_brief_creation_team_tools.py b/tests/ci/captn/captn_agents/backend/tools/test_brief_creation_team_tools.py index aafc05ab..f0f7f118 100644 --- a/tests/ci/captn/captn_agents/backend/tools/test_brief_creation_team_tools.py +++ b/tests/ci/captn/captn_agents/backend/tools/test_brief_creation_team_tools.py @@ -78,7 +78,7 @@ def test_change_the_team_and_start_new_chat(self) -> None: "get_info_from_web_page_result", [ "My web page info.", - None, + "", ], ) def test_delagate_task(self, get_info_from_web_page_result: Optional[str]) -> None: @@ -116,7 +116,7 @@ def test_delagate_task(self, get_info_from_web_page_result: Optional[str]) -> No context=context, ) - if get_info_from_web_page_result is None: + if get_info_from_web_page_result == "": assert isinstance(response, str) mock_initiate_chat.assert_not_called() mock_get_last_message.assert_not_called() @@ -126,6 +126,6 @@ def test_delagate_task(self, get_info_from_web_page_result: Optional[str]) -> No team_response = TeamResponse.model_validate_json(response) print(f"team_response: {team_response}") finally: - if get_info_from_web_page_result is not None: + if get_info_from_web_page_result != "": poped_team = Team.pop_team(user_id=12345, conv_id=67890) assert isinstance(poped_team, GoogleAdsTeam) diff --git a/tests/ci/captn/captn_agents/backend/tools/test_campaign_creation_team_tools.py b/tests/ci/captn/captn_agents/backend/tools/test_campaign_creation_team_tools.py index a7350a7c..4ad19ae9 100644 --- a/tests/ci/captn/captn_agents/backend/tools/test_campaign_creation_team_tools.py +++ b/tests/ci/captn/captn_agents/backend/tools/test_campaign_creation_team_tools.py @@ -3,14 +3,15 @@ import pytest from autogen.agentchat import AssistantAgent, UserProxyAgent +from pydantic import ValidationError from captn.captn_agents.backend.config import Config +from captn.captn_agents.backend.teams._campaign_creation_team import ( + ad_group_with_ad_and_keywords, +) from captn.captn_agents.backend.toolboxes.base import Toolbox from captn.captn_agents.backend.tools._campaign_creation_team_tools import ( AdGroupAdForCreation, - AdGroupCriterionForCreation, - AdGroupForCreation, - AdGroupWithAdAndKeywords, _get_resource_id_from_response, _remove_resources, create_campaign_creation_team_toolbox, @@ -87,38 +88,10 @@ def test_get_resource_id_from_response(self, response) -> None: ], ) def test_create_ad_group_with_ad_and_keywords(self, side_effect) -> None: - ad_group_ad = AdGroupAdForCreation( - final_url="https://www.example.com", - headlines=["headline1", "headline2", "headline3"], - descriptions=["description1", "description2"], - status="ENABLED", - ) - - keyword1 = AdGroupCriterionForCreation( - keyword_text="keyword1", - keyword_match_type="EXACT", - status="ENABLED", - ) - keyword2 = AdGroupCriterionForCreation( - keyword_text="keyword2", - keyword_match_type="EXACT", - status="ENABLED", - ) - - ad_group = AdGroupForCreation( - name="ad_group", - status="ENABLED", - ad_group_ad=ad_group_ad, - keywords=[keyword1, keyword2], - ) - - ad_group_with_ad_and_keywords = AdGroupWithAdAndKeywords( - customer_id="1111", - campaign_id="1212", - ad_group=ad_group, - ad_group_ad=ad_group_ad, - keywords=[keyword1, keyword2], - ) + ad_group = ad_group_with_ad_and_keywords.ad_group + ad_group_ad = ad_group_with_ad_and_keywords.ad_group_ad + keyword1 = ad_group_with_ad_and_keywords.keywords[0] + keyword2 = ad_group_with_ad_and_keywords.keywords[1] with ( unittest.mock.patch( @@ -161,18 +134,18 @@ def test_create_ad_group_with_ad_and_keywords(self, side_effect) -> None: conv_id=1, list_of_resources=[ RemoveResource( - customer_id="1111", + customer_id="2222", resource_id="3434", resource_type="ad_group", ), RemoveResource( - customer_id="1111", + customer_id="2222", parent_id="3434", resource_id="5656", resource_type="ad", ), RemoveResource( - customer_id="1111", + customer_id="2222", parent_id="3434", resource_id="7878", resource_type="ad_group_criterion", @@ -213,3 +186,123 @@ def test_context_objects_are_not_coppies(self): actual = context.recommended_modifications_and_answer_list expected = [("question", "answer")] assert actual == expected, actual + + +class TestAdGroupAdForCreation: + @pytest.mark.parametrize( + "headlines, descriptions, expected", + [ + ( + [ + "H1", + "H2", + "H3", + "H4", + "H5", + "H6", + "H7", + "H8", + "H9", + "H10", + "H11", + "H12", + "H13", + "H14", + "H15", + ], + ["D1", "D2", "D3", "D4"], + None, + ), + ( + ["H1", "H2", "H3", "H4"], + ["D1", "D2", "D3", "D4"], + ValidationError, + ), + ( + [ + "H1", + "H2", + "H3", + "H4", + "H5", + "H6", + "H7", + "H8", + "H9", + "H10", + "H11", + "H12", + "H13", + "H14", + "H15", + ], + ["D1", "D2", "D3"], + ValidationError, + ), + ], + ) + def test_minimum_number_of_headlines_and_descriptions( + self, headlines, descriptions, expected + ): + if expected == ValidationError: + with pytest.raises(ValidationError): + AdGroupAdForCreation( + customer_id="2222", + headlines=headlines, + descriptions=descriptions, + final_url="https://www.example.com", + status="ENABLED", + ) + else: + AdGroupAdForCreation( + customer_id="2222", + headlines=headlines, + descriptions=descriptions, + final_url="https://www.example.com", + status="ENABLED", + ) + + @pytest.mark.parametrize( + "headline, expected", + [ + ("A" * 31, ValueError), + ("{KeyWord: " + "A" * 30 + "}", None), + ("{KeyWord: " + "A" * 31 + "}", ValueError), + ], + ) + def test_maximum_headline_string_length(self, headline, expected): + descriptions = ["D1", "D2", "D3", "D4"] + headlines = [ + "H1", + "H2", + "H3", + "H4", + "H5", + "H6", + "H7", + "H8", + "H9", + "H10", + "H11", + "H12", + "H13", + "H14", + ] + headlines.append(headline) + if expected == ValueError: + with pytest.raises(ValueError): + AdGroupAdForCreation( + customer_id="2222", + headlines=headlines, + descriptions=descriptions, + final_url="https://www.example.com", + status="ENABLED", + ) + else: + AdGroupAdForCreation( + customer_id="2222", + headlines=headlines, + descriptions=descriptions, + final_url="https://www.example.com", + status="ENABLED", + ) diff --git a/tests/ci/google_ads/test_model.py b/tests/ci/google_ads/test_model.py new file mode 100644 index 00000000..d8b85e8f --- /dev/null +++ b/tests/ci/google_ads/test_model.py @@ -0,0 +1,73 @@ +import pytest +from pydantic import ValidationError + +from google_ads.model import AdGroupAd, _remove_keyword_insertion_chars + + +class TestAdGroupAd: + @pytest.mark.parametrize( + "headlines, descriptions, expected", + [ + ( + ["Headline 1", "Headline 2", "Headline 3"], + ["Description 1", "Description 2"], + None, + ), + ( + ["Headline 1", "Headline 2"], + ["Description 1", "Description 2"], + ValidationError, + ), + ( + ["Headline 1", "Headline 2", "Headline 3"], + ["Description 1"], + ValidationError, + ), + ], + ) + def test_minimum_number_of_headlines_and_descriptions( + self, headlines, descriptions, expected + ): + if expected == ValidationError: + with pytest.raises(ValidationError): + AdGroupAd( + customer_id="2222", headlines=headlines, descriptions=descriptions + ) + else: + AdGroupAd( + customer_id="2222", headlines=headlines, descriptions=descriptions + ) + + @pytest.mark.parametrize( + "headline, expected", + [ + ("{KeyWord: 1}", "1"), + ("{keyWord: abc}", "abc"), + ("keyword1", "keyword1"), + ], + ) + def test_remove_keyword_insertion_chars(self, headline, expected): + assert _remove_keyword_insertion_chars(headline) == expected + + @pytest.mark.parametrize( + "headlines, expected", + [ + (["Headline 1", "h" * 31, "A" * 31], ValueError), + (["Headline 1", "Headline 2", "{KeyWord: " + "A" * 30 + "}"], None), + (["Headline 1", "Headline 2", "{KeyWord: " + "A" * 31 + "}"], ValueError), + ], + ) + def test_maximum_headline_string_length(self, headlines, expected): + if expected == ValueError: + with pytest.raises(ValueError): + AdGroupAd( + customer_id="2222", + headlines=headlines, + descriptions=["Description 1", "Description 2"], + ) + else: + AdGroupAd( + customer_id="2222", + headlines=headlines, + descriptions=["Description 1", "Description 2"], + )