Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import json
import logging

from playwright.sync_api import sync_playwright

import agentql
from playwright.sync_api import sync_playwright

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)
Expand Down Expand Up @@ -33,7 +32,7 @@

# limit the total number of books to 50
if len(response["books"]) + len(books) > 50:
books.extend(response["books"][:50 - len(books)])
books.extend(response["books"][: 50 - len(books)])
else:
books.extend(response["books"])

Expand All @@ -44,5 +43,5 @@
if pagination_info.has_next_page:
pagination_info.navigate_to_next_page()

with open(f"./books.json", "w") as f:
with open("./books.json", "w", encoding="utf-8") as f:
json.dump(books, f, indent=4)
5 changes: 2 additions & 3 deletions examples/python/collect_paginated_news_headlines/main.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import json
import logging

from playwright.sync_api import sync_playwright

import agentql

# import paginate tool from agentql tools
from agentql.tools.sync_api import paginate
from playwright.sync_api import sync_playwright

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)
Expand All @@ -29,6 +28,6 @@
paginated_data = paginate(page, QUERY, 3)

# save the aggregateddata to a json file
with open("./hackernews_paginated_data.json", "w") as f:
with open("./hackernews_paginated_data.json", "w", encoding="utf-8") as f:
json.dump(paginated_data, f, indent=4)
log.debug("Paginated data has been saved to hackernews_paginated_data.json")
8 changes: 3 additions & 5 deletions examples/python/compare_product_prices/async_main.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,19 +48,17 @@ async def get_price_across_websites():
headless=False
) as browser, await browser.new_context() as context:
# Open multiple tabs in the same browser context to fetch prices concurrently
(bestbuy_price, ebay_price, telquest_price) = await asyncio.gather(
bestbuy_price, ebay_price, telquest_price = await asyncio.gather(
fetch_price(context, BESTBUY_URL),
fetch_price(context, EBAY_URL),
fetch_price(context, TELQUEST_URL),
)

print(
f"""
print(f"""
Price at BestBuy: {bestbuy_price}
Price at Target: {ebay_price}
Price at Telquest: {telquest_price}
"""
)
""")


if __name__ == "__main__":
Expand Down
3 changes: 1 addition & 2 deletions examples/python/humanlike-antibot/main.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import random
import time

from playwright.sync_api import ElementHandle, Page, sync_playwright

import agentql
from playwright.sync_api import ElementHandle, Page, sync_playwright


def random_mouse_movement(page: Page):
Expand Down
3 changes: 1 addition & 2 deletions examples/python/maps_scraper/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@

import os

from playwright.sync_api import sync_playwright

import agentql
from playwright.sync_api import sync_playwright

# Set the URL to the Google Maps search for "boba tea" near Palo Alto
URL = "https://www.google.com/maps/search/boba+tea/@37.4400289,-122.1653309,14z/data=!3m1!4b1?entry=ttu&g_ep=EgoyMDI1MDIxMS4wIKXMDSoASAFQAw%3D%3D"
Expand Down
15 changes: 6 additions & 9 deletions examples/python/news-aggregator/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,8 @@
import logging
import os

from playwright.async_api import BrowserContext, async_playwright

import agentql
from playwright.async_api import BrowserContext, async_playwright

# Set up logging
logging.basicConfig(level=logging.INFO)
Expand Down Expand Up @@ -46,12 +45,10 @@ async def main():
headless=True
) as browser, await browser.new_context() as context:
# Open multiple tabs in the same browser context to fetch data concurrently
await asyncio.gather(
*(fetch_data(context, url) for url in WEBSITE_URLS)
)

# Update progress
log.info("All done! CSV is here: %s...", CSV_FILE_PATH)
await asyncio.gather(*(fetch_data(context, url) for url in WEBSITE_URLS))

# Update progress
log.info(f"All done! CSV is here: {CSV_FILE_PATH}...")


async def fetch_data(context: BrowserContext, session_url):
Expand Down Expand Up @@ -82,7 +79,7 @@ async def fetch_data(context: BrowserContext, session_url):
file.writelines(new_lines)

# Update progress
log.info("Fetched items from %s...", session_url)
log.info(f"Fetched items from {session_url}...")


if __name__ == "__main__":
Expand Down
7 changes: 3 additions & 4 deletions examples/python/news-aggregator/main_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@
import logging
import os

from playwright.sync_api import BrowserContext, sync_playwright

import agentql
from playwright.sync_api import BrowserContext, sync_playwright

# Set up logging
logging.basicConfig(level=logging.INFO)
Expand Down Expand Up @@ -43,7 +42,7 @@ def main():
# Process URLs sequentially in synchronous version
for url in WEBSITE_URLS:
fetch_data(context, url)
log.info("All done! CSV is here: %s", CSV_FILE_PATH)
log.info(f"All done! CSV is here: {CSV_FILE_PATH}")


def fetch_data(context: BrowserContext, session_url):
Expand Down Expand Up @@ -73,7 +72,7 @@ def fetch_data(context: BrowserContext, session_url):
with open(CSV_FILE_PATH, "a", encoding="utf-8") as file:
file.writelines(new_lines)

log.info("Fetched items from %s", session_url)
log.info(f"Fetched items from {session_url}")


if __name__ == "__main__":
Expand Down