Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 646440eba3 | |||
| 53e5579326 | |||
| 29a1630d0f | |||
| 171f4ab2ae |
@@ -65,10 +65,31 @@ jobs:
|
||||
cd core
|
||||
pytest tests/ -v
|
||||
|
||||
test-tools:
|
||||
name: Test Tools
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies and run tests
|
||||
run: |
|
||||
cd tools
|
||||
uv sync --extra dev
|
||||
uv pip install --python .venv/bin/python -e ../core
|
||||
uv run --extra dev pytest tests/ -v
|
||||
|
||||
validate:
|
||||
name: Validate Agent Exports
|
||||
runs-on: ubuntu-latest
|
||||
needs: [lint, test]
|
||||
needs: [lint, test, test-tools]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
|
||||
+20
-22
@@ -44,7 +44,7 @@ Aden Agent Framework is a Python-based system for building goal-driven, self-imp
|
||||
Ensure you have installed:
|
||||
|
||||
- **Python 3.11+** - [Download](https://www.python.org/downloads/) (3.12 or 3.13 recommended)
|
||||
- **pip** - Package installer for Python (comes with Python)
|
||||
- **uv** - Python package manager ([Install](https://docs.astral.sh/uv/getting-started/installation/))
|
||||
- **git** - Version control
|
||||
- **Claude Code** - [Install](https://docs.anthropic.com/claude/docs/claude-code) (optional, for using building skills)
|
||||
|
||||
@@ -52,7 +52,7 @@ Verify installation:
|
||||
|
||||
```bash
|
||||
python --version # Should be 3.11+
|
||||
pip --version # Should be latest
|
||||
uv --version # Should be latest
|
||||
git --version # Any recent version
|
||||
```
|
||||
|
||||
@@ -128,8 +128,12 @@ hive/ # Repository root
|
||||
│
|
||||
├── .github/ # GitHub configuration
|
||||
│ ├── workflows/
|
||||
│ │ ├── ci.yml # Runs on every PR
|
||||
│ │ └── release.yml # Runs on tags
|
||||
│ │ ├── ci.yml # Lint, test, validate on every PR
|
||||
│ │ ├── release.yml # Runs on tags
|
||||
│ │ ├── pr-requirements.yml # PR requirement checks
|
||||
│ │ ├── pr-check-command.yml # PR check commands
|
||||
│ │ ├── claude-issue-triage.yml # Automated issue triage
|
||||
│ │ └── auto-close-duplicates.yml # Close duplicate issues
|
||||
│ ├── ISSUE_TEMPLATE/ # Bug report & feature request templates
|
||||
│ ├── PULL_REQUEST_TEMPLATE.md # PR description template
|
||||
│ └── CODEOWNERS # Auto-assign reviewers
|
||||
@@ -166,7 +170,6 @@ hive/ # Repository root
|
||||
│ │ ├── testing/ # Testing utilities
|
||||
│ │ └── __init__.py
|
||||
│ ├── pyproject.toml # Package metadata and dependencies
|
||||
│ ├── requirements.txt # Python dependencies
|
||||
│ ├── README.md # Framework documentation
|
||||
│ ├── MCP_INTEGRATION_GUIDE.md # MCP server integration guide
|
||||
│ └── docs/ # Protocol documentation
|
||||
@@ -182,7 +185,6 @@ hive/ # Repository root
|
||||
│ │ ├── mcp_server.py # HTTP MCP server
|
||||
│ │ └── __init__.py
|
||||
│ ├── pyproject.toml # Package metadata
|
||||
│ ├── requirements.txt # Python dependencies
|
||||
│ └── README.md # Tools documentation
|
||||
│
|
||||
├── exports/ # AGENT PACKAGES (user-created, gitignored)
|
||||
@@ -191,14 +193,16 @@ hive/ # Repository root
|
||||
├── docs/ # Documentation
|
||||
│ ├── getting-started.md # Quick start guide
|
||||
│ ├── configuration.md # Configuration reference
|
||||
│ ├── architecture.md # System architecture
|
||||
│ └── articles/ # Technical articles
|
||||
│ ├── architecture/ # System architecture
|
||||
│ ├── articles/ # Technical articles
|
||||
│ ├── quizzes/ # Developer quizzes
|
||||
│ └── i18n/ # Translations
|
||||
│
|
||||
├── scripts/ # Build & utility scripts
|
||||
│ ├── setup-python.sh # Python environment setup
|
||||
│ └── setup.sh # Legacy setup script
|
||||
│
|
||||
├── quickstart.sh # Install Claude Code skills
|
||||
├── quickstart.sh # Interactive setup wizard
|
||||
├── ENVIRONMENT_SETUP.md # Complete Python setup guide
|
||||
├── README.md # Project overview
|
||||
├── DEVELOPER.md # This file
|
||||
@@ -375,7 +379,7 @@ def test_ticket_categorization():
|
||||
- **PEP 8** - Follow Python style guide
|
||||
- **Type hints** - Use for function signatures and class attributes
|
||||
- **Docstrings** - Document classes and public functions
|
||||
- **Black** - Code formatter (run with `black .`)
|
||||
- **Ruff** - Linter and formatter (run with `make check`)
|
||||
|
||||
```python
|
||||
# Good
|
||||
@@ -509,8 +513,8 @@ chore(deps): update React to 18.2.0
|
||||
|
||||
1. Create a feature branch from `main`
|
||||
2. Make your changes with clear commits
|
||||
3. Run tests locally: `PYTHONPATH=core:exports python -m pytest`
|
||||
4. Run linting: `black --check .`
|
||||
3. Run tests locally: `make test`
|
||||
4. Run linting: `make check`
|
||||
5. Push and create a PR
|
||||
6. Fill out the PR template
|
||||
7. Request review from CODEOWNERS
|
||||
@@ -528,16 +532,11 @@ chore(deps): update React to 18.2.0
|
||||
```bash
|
||||
# Add to core framework
|
||||
cd core
|
||||
pip install <package>
|
||||
# Then add to requirements.txt or pyproject.toml
|
||||
uv add <package>
|
||||
|
||||
# Add to tools package
|
||||
cd tools
|
||||
pip install <package>
|
||||
# Then add to requirements.txt or pyproject.toml
|
||||
|
||||
# Reinstall in editable mode
|
||||
pip install -e .
|
||||
uv add <package>
|
||||
```
|
||||
|
||||
### Creating a New Agent
|
||||
@@ -670,9 +669,8 @@ cat .env
|
||||
# Or check shell environment
|
||||
echo $ANTHROPIC_API_KEY
|
||||
|
||||
# Copy from .env.example if needed
|
||||
cp .env.example .env
|
||||
# Then edit .env with your API keys
|
||||
# Create .env if needed
|
||||
# Then add your API keys
|
||||
```
|
||||
|
||||
|
||||
|
||||
@@ -75,16 +75,6 @@ class SafeEvalVisitor(ast.NodeVisitor):
|
||||
def visit_Constant(self, node: ast.Constant) -> Any:
|
||||
return node.value
|
||||
|
||||
# --- Number/String/Bytes/NameConstant (Python < 3.8 compat if needed) ---
|
||||
def visit_Num(self, node: ast.Num) -> Any:
|
||||
return node.n
|
||||
|
||||
def visit_Str(self, node: ast.Str) -> Any:
|
||||
return node.s
|
||||
|
||||
def visit_NameConstant(self, node: ast.NameConstant) -> Any:
|
||||
return node.value
|
||||
|
||||
# --- Data Structures ---
|
||||
def visit_List(self, node: ast.List) -> list:
|
||||
return [self.visit(elt) for elt in node.elts]
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
# MCP Server
|
||||
fastmcp
|
||||
|
||||
# Tool dependencies
|
||||
diff-match-patch
|
||||
pypdf
|
||||
beautifulsoup4
|
||||
lxml
|
||||
playwright
|
||||
playwright-stealth
|
||||
requests
|
||||
|
||||
# Note: After installing, run `playwright install` to download browser binaries
|
||||
@@ -9,7 +9,7 @@ Philosophy: Google Strictness + Apple UX
|
||||
|
||||
Usage:
|
||||
from aden_tools.credentials import CredentialStoreAdapter
|
||||
from core.framework.credentials import CredentialStore
|
||||
from framework.credentials import CredentialStore
|
||||
|
||||
# With encrypted storage (production)
|
||||
store = CredentialStore.with_encrypted_storage() # defaults to ~/.hive/credentials
|
||||
|
||||
@@ -5,7 +5,7 @@ This provides backward compatibility, allowing existing tools to work unchanged
|
||||
while enabling new features (template resolution, multi-key credentials, etc.).
|
||||
|
||||
Usage:
|
||||
from core.framework.credentials import CredentialStore
|
||||
from framework.credentials import CredentialStore
|
||||
from aden_tools.credentials.store_adapter import CredentialStoreAdapter
|
||||
|
||||
# Create new credential store
|
||||
@@ -31,7 +31,7 @@ from typing import TYPE_CHECKING
|
||||
from .base import CredentialError, CredentialSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from core.framework.credentials import CredentialStore
|
||||
from framework.credentials import CredentialStore
|
||||
|
||||
|
||||
class CredentialStoreAdapter:
|
||||
@@ -368,7 +368,7 @@ class CredentialStoreAdapter:
|
||||
credentials = CredentialStoreAdapter.for_testing({"brave_search": "test-key"})
|
||||
assert credentials.get("brave_search") == "test-key"
|
||||
"""
|
||||
from core.framework.credentials import CredentialStore
|
||||
from framework.credentials import CredentialStore
|
||||
|
||||
# Convert to CredentialStore.for_testing format
|
||||
# Simple credentials get a single "api_key" key
|
||||
@@ -395,13 +395,14 @@ class CredentialStoreAdapter:
|
||||
Returns:
|
||||
CredentialStoreAdapter using env vars for storage
|
||||
"""
|
||||
from core.framework.credentials import CredentialStore
|
||||
from framework.credentials import CredentialStore
|
||||
|
||||
# Build env mapping from specs if not provided
|
||||
if env_mapping is None and specs is None:
|
||||
from . import CREDENTIAL_SPECS
|
||||
if env_mapping is None:
|
||||
if specs is None:
|
||||
from . import CREDENTIAL_SPECS
|
||||
|
||||
specs = CREDENTIAL_SPECS
|
||||
specs = CREDENTIAL_SPECS
|
||||
env_mapping = {name: spec.env_var for name, spec in specs.items()}
|
||||
|
||||
store = CredentialStore.with_env_storage(env_mapping)
|
||||
|
||||
@@ -415,13 +415,13 @@ class TestDealTools:
|
||||
|
||||
class TestHubSpotOAuth2Provider:
|
||||
def test_provider_id(self):
|
||||
from core.framework.credentials.oauth2.hubspot_provider import HubSpotOAuth2Provider
|
||||
from framework.credentials.oauth2.hubspot_provider import HubSpotOAuth2Provider
|
||||
|
||||
provider = HubSpotOAuth2Provider(client_id="cid", client_secret="csecret")
|
||||
assert provider.provider_id == "hubspot_oauth2"
|
||||
|
||||
def test_default_scopes(self):
|
||||
from core.framework.credentials.oauth2.hubspot_provider import (
|
||||
from framework.credentials.oauth2.hubspot_provider import (
|
||||
HUBSPOT_DEFAULT_SCOPES,
|
||||
HubSpotOAuth2Provider,
|
||||
)
|
||||
@@ -430,7 +430,7 @@ class TestHubSpotOAuth2Provider:
|
||||
assert provider.config.default_scopes == HUBSPOT_DEFAULT_SCOPES
|
||||
|
||||
def test_custom_scopes(self):
|
||||
from core.framework.credentials.oauth2.hubspot_provider import HubSpotOAuth2Provider
|
||||
from framework.credentials.oauth2.hubspot_provider import HubSpotOAuth2Provider
|
||||
|
||||
provider = HubSpotOAuth2Provider(
|
||||
client_id="cid",
|
||||
@@ -440,7 +440,7 @@ class TestHubSpotOAuth2Provider:
|
||||
assert provider.config.default_scopes == ["crm.objects.contacts.read"]
|
||||
|
||||
def test_endpoints(self):
|
||||
from core.framework.credentials.oauth2.hubspot_provider import (
|
||||
from framework.credentials.oauth2.hubspot_provider import (
|
||||
HUBSPOT_AUTHORIZATION_URL,
|
||||
HUBSPOT_TOKEN_URL,
|
||||
HubSpotOAuth2Provider,
|
||||
@@ -451,15 +451,15 @@ class TestHubSpotOAuth2Provider:
|
||||
assert provider.config.authorization_url == HUBSPOT_AUTHORIZATION_URL
|
||||
|
||||
def test_supported_types(self):
|
||||
from core.framework.credentials.models import CredentialType
|
||||
from core.framework.credentials.oauth2.hubspot_provider import HubSpotOAuth2Provider
|
||||
from framework.credentials.models import CredentialType
|
||||
from framework.credentials.oauth2.hubspot_provider import HubSpotOAuth2Provider
|
||||
|
||||
provider = HubSpotOAuth2Provider(client_id="cid", client_secret="csecret")
|
||||
assert CredentialType.OAUTH2 in provider.supported_types
|
||||
|
||||
def test_validate_no_access_token(self):
|
||||
from core.framework.credentials.models import CredentialObject
|
||||
from core.framework.credentials.oauth2.hubspot_provider import HubSpotOAuth2Provider
|
||||
from framework.credentials.models import CredentialObject
|
||||
from framework.credentials.oauth2.hubspot_provider import HubSpotOAuth2Provider
|
||||
|
||||
provider = HubSpotOAuth2Provider(client_id="cid", client_secret="csecret")
|
||||
cred = CredentialObject(id="test")
|
||||
|
||||
@@ -95,6 +95,7 @@ class TestPdfReadTool:
|
||||
def __init__(self, path: Path) -> None: # noqa: ARG002
|
||||
self.pages = [FakePage(f"Page {i + 1}") for i in range(50)]
|
||||
self.is_encrypted = False
|
||||
self.metadata = None
|
||||
|
||||
# Patch PdfReader used inside the tool so we don't need a real PDF
|
||||
from aden_tools.tools.pdf_read_tool import pdf_read_tool
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Tests for web_scrape tool (FastMCP)."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastmcp import FastMCP
|
||||
@@ -15,60 +15,135 @@ def web_scrape_fn(mcp: FastMCP):
|
||||
return mcp._tool_manager._tools["web_scrape"].fn
|
||||
|
||||
|
||||
def _make_playwright_mocks(html, status=200, final_url="https://example.com/page"):
|
||||
"""Build a full playwright mock chain and return (context_manager, response, page)."""
|
||||
mock_response = MagicMock(
|
||||
status=status,
|
||||
url=final_url,
|
||||
headers={"content-type": "text/html; charset=utf-8"},
|
||||
)
|
||||
|
||||
mock_page = AsyncMock()
|
||||
mock_page.goto.return_value = mock_response
|
||||
mock_page.content.return_value = html
|
||||
mock_page.wait_for_timeout.return_value = None
|
||||
|
||||
mock_context = AsyncMock()
|
||||
mock_context.new_page.return_value = mock_page
|
||||
|
||||
mock_browser = AsyncMock()
|
||||
mock_browser.new_context.return_value = mock_context
|
||||
|
||||
mock_pw = MagicMock()
|
||||
mock_pw.chromium.launch = AsyncMock(return_value=mock_browser)
|
||||
|
||||
# async context manager for async_playwright()
|
||||
mock_cm = MagicMock()
|
||||
mock_cm.__aenter__ = AsyncMock(return_value=mock_pw)
|
||||
mock_cm.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
return mock_cm, mock_response, mock_page
|
||||
|
||||
|
||||
_PW_PATH = "aden_tools.tools.web_scrape_tool.web_scrape_tool.async_playwright"
|
||||
_STEALTH_PATH = "aden_tools.tools.web_scrape_tool.web_scrape_tool.Stealth"
|
||||
|
||||
|
||||
class TestWebScrapeTool:
|
||||
"""Tests for web_scrape tool."""
|
||||
|
||||
def test_url_auto_prefixed_with_https(self, web_scrape_fn):
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_url_auto_prefixed_with_https(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""URLs without scheme get https:// prefix."""
|
||||
# This will fail to connect, but we can verify the behavior
|
||||
result = web_scrape_fn(url="example.com")
|
||||
# Should either succeed or have a network error (not a validation error)
|
||||
assert isinstance(result, dict)
|
||||
html = "<html><body>Hello</body></html>"
|
||||
mock_cm, _, _ = _make_playwright_mocks(html, final_url="https://example.com")
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
def test_max_length_clamped_low(self, web_scrape_fn):
|
||||
result = await web_scrape_fn(url="example.com")
|
||||
assert isinstance(result, dict)
|
||||
assert "error" not in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_max_length_clamped_low(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""max_length below 1000 is clamped to 1000."""
|
||||
# Test with a very low max_length - implementation clamps to 1000
|
||||
result = web_scrape_fn(url="https://example.com", max_length=500)
|
||||
# Should not error due to invalid max_length
|
||||
assert isinstance(result, dict)
|
||||
html = "<html><body>Hello</body></html>"
|
||||
mock_cm, _, _ = _make_playwright_mocks(html, final_url="https://example.com")
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
def test_max_length_clamped_high(self, web_scrape_fn):
|
||||
result = await web_scrape_fn(url="https://example.com", max_length=500)
|
||||
assert isinstance(result, dict)
|
||||
assert "error" not in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_max_length_clamped_high(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""max_length above 500000 is clamped to 500000."""
|
||||
# Test with a very high max_length - implementation clamps to 500000
|
||||
result = web_scrape_fn(url="https://example.com", max_length=600000)
|
||||
# Should not error due to invalid max_length
|
||||
assert isinstance(result, dict)
|
||||
html = "<html><body>Hello</body></html>"
|
||||
mock_cm, _, _ = _make_playwright_mocks(html, final_url="https://example.com")
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
def test_valid_max_length_accepted(self, web_scrape_fn):
|
||||
result = await web_scrape_fn(url="https://example.com", max_length=600000)
|
||||
assert isinstance(result, dict)
|
||||
assert "error" not in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_valid_max_length_accepted(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""Valid max_length values are accepted."""
|
||||
result = web_scrape_fn(url="https://example.com", max_length=10000)
|
||||
assert isinstance(result, dict)
|
||||
html = "<html><body>Hello</body></html>"
|
||||
mock_cm, _, _ = _make_playwright_mocks(html, final_url="https://example.com")
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
def test_include_links_option(self, web_scrape_fn):
|
||||
result = await web_scrape_fn(url="https://example.com", max_length=10000)
|
||||
assert isinstance(result, dict)
|
||||
assert "error" not in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_include_links_option(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""include_links parameter is accepted."""
|
||||
result = web_scrape_fn(url="https://example.com", include_links=True)
|
||||
assert isinstance(result, dict)
|
||||
html = '<html><body><a href="/link">Link</a></body></html>'
|
||||
mock_cm, _, _ = _make_playwright_mocks(html, final_url="https://example.com")
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
def test_selector_option(self, web_scrape_fn):
|
||||
"""selector parameter is accepted."""
|
||||
result = web_scrape_fn(url="https://example.com", selector=".content")
|
||||
result = await web_scrape_fn(url="https://example.com", include_links=True)
|
||||
assert isinstance(result, dict)
|
||||
assert "error" not in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_selector_option(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""selector parameter is accepted."""
|
||||
html = '<html><body><div class="content">Content here</div></body></html>'
|
||||
mock_cm, _, _ = _make_playwright_mocks(html, final_url="https://example.com")
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
result = await web_scrape_fn(url="https://example.com", selector=".content")
|
||||
assert isinstance(result, dict)
|
||||
assert "error" not in result
|
||||
|
||||
|
||||
class TestWebScrapeToolLinkConversion:
|
||||
"""Tests for link URL conversion (relative to absolute)."""
|
||||
|
||||
def _mock_response(self, html_content, final_url="https://example.com/page"):
|
||||
"""Create a mock httpx response object."""
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.text = html_content
|
||||
mock_response.url = final_url
|
||||
mock_response.headers = {"content-type": "text/html; charset=utf-8"}
|
||||
return mock_response
|
||||
|
||||
@patch("aden_tools.tools.web_scrape_tool.web_scrape_tool.httpx.get")
|
||||
def test_relative_links_converted_to_absolute(self, mock_get, web_scrape_fn):
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_relative_links_converted_to_absolute(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""Relative URLs like ../page are converted to absolute URLs."""
|
||||
html = """
|
||||
<html>
|
||||
@@ -78,9 +153,11 @@ class TestWebScrapeToolLinkConversion:
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
mock_get.return_value = self._mock_response(html, "https://example.com/blog/post")
|
||||
mock_cm, _, _ = _make_playwright_mocks(html, final_url="https://example.com/blog/post")
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
result = web_scrape_fn(url="https://example.com/blog/post", include_links=True)
|
||||
result = await web_scrape_fn(url="https://example.com/blog/post", include_links=True)
|
||||
|
||||
assert "error" not in result
|
||||
assert "links" in result
|
||||
@@ -95,8 +172,10 @@ class TestWebScrapeToolLinkConversion:
|
||||
expected = "https://example.com/blog/page.html"
|
||||
assert hrefs["Next Page"] == expected, f"Got {hrefs['Next Page']}"
|
||||
|
||||
@patch("aden_tools.tools.web_scrape_tool.web_scrape_tool.httpx.get")
|
||||
def test_root_relative_links_converted(self, mock_get, web_scrape_fn):
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_root_relative_links_converted(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""Root-relative URLs like /about are converted to absolute URLs."""
|
||||
html = """
|
||||
<html>
|
||||
@@ -106,9 +185,11 @@ class TestWebScrapeToolLinkConversion:
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
mock_get.return_value = self._mock_response(html, "https://example.com/blog/post")
|
||||
mock_cm, _, _ = _make_playwright_mocks(html, final_url="https://example.com/blog/post")
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
result = web_scrape_fn(url="https://example.com/blog/post", include_links=True)
|
||||
result = await web_scrape_fn(url="https://example.com/blog/post", include_links=True)
|
||||
|
||||
assert "error" not in result
|
||||
assert "links" in result
|
||||
@@ -119,8 +200,10 @@ class TestWebScrapeToolLinkConversion:
|
||||
assert hrefs["About"] == "https://example.com/about"
|
||||
assert hrefs["Contact"] == "https://example.com/contact"
|
||||
|
||||
@patch("aden_tools.tools.web_scrape_tool.web_scrape_tool.httpx.get")
|
||||
def test_absolute_links_unchanged(self, mock_get, web_scrape_fn):
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_absolute_links_unchanged(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""Absolute URLs remain unchanged."""
|
||||
html = """
|
||||
<html>
|
||||
@@ -130,9 +213,11 @@ class TestWebScrapeToolLinkConversion:
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
mock_get.return_value = self._mock_response(html)
|
||||
mock_cm, _, _ = _make_playwright_mocks(html)
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
result = web_scrape_fn(url="https://example.com", include_links=True)
|
||||
result = await web_scrape_fn(url="https://example.com", include_links=True)
|
||||
|
||||
assert "error" not in result
|
||||
assert "links" in result
|
||||
@@ -143,8 +228,10 @@ class TestWebScrapeToolLinkConversion:
|
||||
assert hrefs["Other Site"] == "https://other.com"
|
||||
assert hrefs["Internal"] == "https://example.com/page"
|
||||
|
||||
@patch("aden_tools.tools.web_scrape_tool.web_scrape_tool.httpx.get")
|
||||
def test_links_after_redirects(self, mock_get, web_scrape_fn):
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_links_after_redirects(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""Links are resolved relative to final URL after redirects."""
|
||||
html = """
|
||||
<html>
|
||||
@@ -155,12 +242,14 @@ class TestWebScrapeToolLinkConversion:
|
||||
</html>
|
||||
"""
|
||||
# Mock redirect: request to /old/url redirects to /new/location
|
||||
mock_get.return_value = self._mock_response(
|
||||
mock_cm, _, _ = _make_playwright_mocks(
|
||||
html,
|
||||
final_url="https://example.com/new/location", # Final URL after redirect
|
||||
)
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
result = web_scrape_fn(url="https://example.com/old/url", include_links=True)
|
||||
result = await web_scrape_fn(url="https://example.com/old/url", include_links=True)
|
||||
|
||||
assert "error" not in result
|
||||
assert "links" in result
|
||||
@@ -173,8 +262,10 @@ class TestWebScrapeToolLinkConversion:
|
||||
)
|
||||
assert hrefs["Next"] == "https://example.com/new/next"
|
||||
|
||||
@patch("aden_tools.tools.web_scrape_tool.web_scrape_tool.httpx.get")
|
||||
def test_fragment_links_preserved(self, mock_get, web_scrape_fn):
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_fragment_links_preserved(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""Fragment links (anchors) are preserved."""
|
||||
html = """
|
||||
<html>
|
||||
@@ -184,9 +275,11 @@ class TestWebScrapeToolLinkConversion:
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
mock_get.return_value = self._mock_response(html, "https://example.com/page")
|
||||
mock_cm, _, _ = _make_playwright_mocks(html, final_url="https://example.com/page")
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
result = web_scrape_fn(url="https://example.com/page", include_links=True)
|
||||
result = await web_scrape_fn(url="https://example.com/page", include_links=True)
|
||||
|
||||
assert "error" not in result
|
||||
assert "links" in result
|
||||
@@ -197,8 +290,10 @@ class TestWebScrapeToolLinkConversion:
|
||||
assert hrefs["Section 1"] == "https://example.com/page#section1"
|
||||
assert hrefs["Page Section 2"] == "https://example.com/page#section2"
|
||||
|
||||
@patch("aden_tools.tools.web_scrape_tool.web_scrape_tool.httpx.get")
|
||||
def test_query_parameters_preserved(self, mock_get, web_scrape_fn):
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_query_parameters_preserved(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""Query parameters in URLs are preserved."""
|
||||
html = """
|
||||
<html>
|
||||
@@ -208,9 +303,11 @@ class TestWebScrapeToolLinkConversion:
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
mock_get.return_value = self._mock_response(html, "https://example.com/blog/post")
|
||||
mock_cm, _, _ = _make_playwright_mocks(html, final_url="https://example.com/blog/post")
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
result = web_scrape_fn(url="https://example.com/blog/post", include_links=True)
|
||||
result = await web_scrape_fn(url="https://example.com/blog/post", include_links=True)
|
||||
|
||||
assert "error" not in result
|
||||
assert "links" in result
|
||||
@@ -222,8 +319,10 @@ class TestWebScrapeToolLinkConversion:
|
||||
assert "q=test" in hrefs["Search"]
|
||||
assert "sort=date" in hrefs["Search"]
|
||||
|
||||
@patch("aden_tools.tools.web_scrape_tool.web_scrape_tool.httpx.get")
|
||||
def test_empty_href_skipped(self, mock_get, web_scrape_fn):
|
||||
@pytest.mark.asyncio
|
||||
@patch(_STEALTH_PATH)
|
||||
@patch(_PW_PATH)
|
||||
async def test_empty_href_skipped(self, mock_pw, mock_stealth, web_scrape_fn):
|
||||
"""Links with empty or whitespace text are skipped."""
|
||||
html = """
|
||||
<html>
|
||||
@@ -234,9 +333,11 @@ class TestWebScrapeToolLinkConversion:
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
mock_get.return_value = self._mock_response(html)
|
||||
mock_cm, _, _ = _make_playwright_mocks(html)
|
||||
mock_pw.return_value = mock_cm
|
||||
mock_stealth.return_value.apply_stealth_async = AsyncMock()
|
||||
|
||||
result = web_scrape_fn(url="https://example.com", include_links=True)
|
||||
result = await web_scrape_fn(url="https://example.com", include_links=True)
|
||||
|
||||
assert "error" not in result
|
||||
assert "links" in result
|
||||
|
||||
Reference in New Issue
Block a user