diff --git a/.cursorrules b/.cursorrules deleted file mode 100644 index 953da6c..0000000 --- a/.cursorrules +++ /dev/null @@ -1,22 +0,0 @@ -# Cursor Rules for python-proxy-headers - -## Git Configuration -- Always commit as `cursor@proxymesh.com` with name `Cursor` -- Run these commands if not already configured: - ``` - git config user.email "cursor@proxymesh.com" - git config user.name "Cursor" - ``` -- NEVER push directly to main - always create a branch and PR -- NEVER bypass branch protection rules -- If a push is rejected due to branch protection, create a PR instead - -## Pull Requests -- Do NOT add "Made with Cursor" or similar footers to PR descriptions -- After creating a PR with `gh pr create`, immediately run `gh pr edit --body "..."` to ensure no footer was appended -- Keep PR descriptions concise and focused on the changes - -## Code Style -- Follow existing code patterns in the repository -- Use type hints for function signatures -- Include docstrings for public functions and classes diff --git a/AGENTS.md b/AGENTS.md index 234ab9d..5a56484 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,7 +1,5 @@ # Agent instructions -Read `.cursorrules` too (git identity, PR footer rule). - ## Checks (run before every push) - `scripts/lint.sh` must pass. CI runs the same command. Tools come from @@ -29,7 +27,7 @@ Read `.cursorrules` too (git identity, PR footer rule). ## Code standards (the owner rejects PRs that miss these) - Follow PEP 8, the Google Python Style Guide, and the Google Shell Style - Guide (see `.cursorrules`). Run the repo linters and fix everything. + Guide. Run the repo linters and fix everything. - Imports at the top of the module. No imports inside functions, methods or class bodies. A guarded try/except ImportError at module top is fine where versions truly differ. diff --git a/IMPLEMENTATION_PRIORITY.md b/IMPLEMENTATION_PRIORITY.md deleted file mode 100644 index f905375..0000000 --- a/IMPLEMENTATION_PRIORITY.md +++ /dev/null @@ -1,369 +0,0 @@ -# Implementation Priority List for python-proxy-headers Extensions - -Based on library analysis considering GitHub stars, activity level, technical feasibility, and user impact. - ---- - -## Priority Rankings - -### 🔴 Priority 1: HIGH - Implement First - -#### 1. pycurl Extension -**File:** `python_proxy_headers/pycurl_proxy.py` - -| Metric | Value | -|--------|-------| -| GitHub Stars | 1,146 | -| Last Active | 2026-01-30 | -| Feasibility | ✅ HIGH | -| Impact | HIGH - Direct libcurl access | - -**Why High Priority:** -- libcurl already supports `CURLOPT_PROXYHEADER` for sending custom headers -- Can capture CONNECT response headers via `CURLOPT_HEADERFUNCTION` -- Foundation for curl_cffi work - -**Implementation Plan:** -```python -# pycurl_proxy.py - Proposed API - -class ProxyCurl: - """PycURL wrapper with proxy header support.""" - - def __init__(self, proxy_headers=None): - self.proxy_headers = proxy_headers or {} - self._response_proxy_headers = {} - - def get(self, url, proxy=None) -> ProxyResponse: - """Make GET request with proxy header support.""" - pass - - @property - def received_proxy_headers(self) -> dict: - """Headers received from proxy during CONNECT.""" - return self._response_proxy_headers - -def request(method, url, proxy=None, proxy_headers=None) -> ProxyResponse: - """Convenience function for one-off requests.""" - pass -``` - -**Technical Approach:** -1. Use `pycurl.PROXYHEADER` option to send custom headers -2. Use `HEADERFUNCTION` callback to capture CONNECT response headers -3. Parse headers to separate proxy headers from origin headers -4. Expose via clean API matching existing python-proxy-headers style - ---- - -#### 2. curl_cffi Extension -**File:** `python_proxy_headers/curl_cffi_proxy.py` - -| Metric | Value | -|--------|-------| -| GitHub Stars | 4,873 | -| Last Active | 2026-01-30 | -| Feasibility | ⚠️ MEDIUM-HIGH | -| Impact | VERY HIGH - Popular anti-bot library | - -**Why High Priority:** -- Very popular for bypassing bot detection -- Uses libcurl which has proxy header capabilities -- Active development means potential upstream contributions - -**Implementation Plan:** -```python -# curl_cffi_proxy.py - Proposed API - -from curl_cffi import Session - -class ProxySession(Session): - """curl_cffi Session with proxy header support.""" - - def __init__(self, proxy_headers=None, **kwargs): - super().__init__(**kwargs) - self._proxy_headers = proxy_headers or {} - self._last_proxy_response_headers = {} - - def request(self, method, url, **kwargs) -> ProxyResponse: - """Make request capturing proxy headers.""" - pass - - @property - def proxy_response_headers(self) -> dict: - """Headers from last proxy CONNECT response.""" - return self._last_proxy_response_headers - -# Convenience functions -def get(url, proxy=None, proxy_headers=None, impersonate=None, **kwargs): - pass -``` - -**Technical Approach:** -1. Investigate if curl_cffi exposes low-level curl options -2. If yes: Use `CURLOPT_PROXYHEADER` directly -3. If no: Create PR to curl_cffi to expose these options -4. May need to work with curl_cffi maintainers - -**Upstream Contribution Opportunity:** -- File issue requesting `proxy_headers` parameter -- Contribute PR if welcomed - ---- - -#### 3. cloudscraper Extension -**File:** `python_proxy_headers/cloudscraper_proxy.py` - -| Metric | Value | -|--------|-------| -| GitHub Stars | 6,060 | -| Last Active | 2025-06-10 | -| Feasibility | ✅ HIGH | -| Impact | HIGH - Popular for Cloudflare bypass | - -**Why High Priority:** -- Built on requests - can use our existing adapter -- Popular for accessing protected sites -- Easy integration - -**Implementation Plan:** -```python -# cloudscraper_proxy.py - Proposed API - -import cloudscraper -from .requests_adapter import HTTPProxyHeaderAdapter - -class ProxyCloudScraper(cloudscraper.CloudScraper): - """CloudScraper with proxy header support.""" - - def __init__(self, proxy_headers=None, **kwargs): - super().__init__(**kwargs) - adapter = HTTPProxyHeaderAdapter(proxy_headers=proxy_headers) - self.mount('https://', adapter) - self.mount('http://', adapter) - -def create_scraper(proxy_headers=None, **kwargs): - """Create a CloudScraper with proxy header support.""" - return ProxyCloudScraper(proxy_headers=proxy_headers, **kwargs) -``` - -**Technical Approach:** -1. Subclass `cloudscraper.CloudScraper` -2. Mount our `HTTPProxyHeaderAdapter` -3. Preserve all cloudscraper functionality -4. Simple integration - likely <50 lines of code - ---- - -### 🟡 Priority 2: MEDIUM - Implement Second - -#### 4. autoscraper Extension -**File:** `python_proxy_headers/autoscraper_proxy.py` - -| Metric | Value | -|--------|-------| -| GitHub Stars | 7,082 | -| Last Active | 2025-06-09 | -| Feasibility | ✅ HIGH | -| Impact | MEDIUM - Niche use case | - -**Implementation Plan:** -```python -# autoscraper_proxy.py - -from autoscraper import AutoScraper -from .requests_adapter import ProxySession - -class ProxyAutoScraper(AutoScraper): - """AutoScraper with proxy header support.""" - - def __init__(self, proxy_headers=None): - super().__init__() - self._proxy_session = ProxySession(proxy_headers=proxy_headers) - - def build(self, url, wanted_list, proxy_headers=None, **kwargs): - """Build scraper with proxy header support.""" - # Use our ProxySession for requests - pass -``` - ---- - -#### 5. treq Extension -**File:** `python_proxy_headers/treq_proxy.py` - -| Metric | Value | -|--------|-------| -| GitHub Stars | 606 | -| Last Active | 2026-01-03 | -| Feasibility | ⚠️ MEDIUM | -| Impact | MEDIUM - Twisted ecosystem | - -**Implementation Plan:** -```python -# treq_proxy.py - -from twisted.web.client import Agent, ProxyAgent -from twisted.internet import reactor - -class ProxyHeaderAgent(ProxyAgent): - """Twisted Agent with proxy header support.""" - - def __init__(self, proxy_headers=None, **kwargs): - super().__init__(**kwargs) - self._proxy_headers = proxy_headers or {} - - # Override connection methods to inject headers -``` - -**Technical Approach:** -1. Subclass `ProxyAgent` -2. Override `_connect` method to add custom headers -3. Capture CONNECT response headers -4. More complex due to Twisted's async nature - ---- - -#### 6. crawlee-python Extension -**File:** `python_proxy_headers/crawlee_proxy.py` - -| Metric | Value | -|--------|-------| -| GitHub Stars | 7,968 | -| Last Active | 2026-01-30 | -| Feasibility | ⚠️ MEDIUM | -| Impact | MEDIUM - Only HTTP crawler portion | - -**Implementation Plan:** -```python -# crawlee_proxy.py - -from crawlee.crawlers import BeautifulSoupCrawler -from .httpx_proxy import HTTPProxyTransport - -class ProxyBeautifulSoupCrawler(BeautifulSoupCrawler): - """Crawler with proxy header support for HTTP requests.""" - - def __init__(self, proxy_headers=None, **kwargs): - # Configure httpx client with our transport - pass -``` - -**Note:** Only applies to `BeautifulSoupCrawler`, not `PlaywrightCrawler` - ---- - -#### 7. requestium Extension -**File:** `python_proxy_headers/requestium_proxy.py` - -| Metric | Value | -|--------|-------| -| GitHub Stars | 1,838 | -| Last Active | 2026-01-26 | -| Feasibility | ⚠️ MEDIUM | -| Impact | LOW - Requests portion only | - -**Implementation Plan:** -```python -# requestium_proxy.py - -from requestium import Session -from .requests_adapter import HTTPProxyHeaderAdapter - -class ProxySession(Session): - """Requestium Session with proxy header support.""" - - def __init__(self, proxy_headers=None, **kwargs): - super().__init__(**kwargs) - adapter = HTTPProxyHeaderAdapter(proxy_headers=proxy_headers) - self.mount('https://', adapter) - self.mount('http://', adapter) -``` - ---- - -#### 8. botasaurus Extension -**File:** `python_proxy_headers/botasaurus_proxy.py` - -| Metric | Value | -|--------|-------| -| GitHub Stars | 3,808 | -| Last Active | 2026-01-10 | -| Feasibility | ⚠️ MEDIUM | -| Impact | LOW - Request decorator only | - -**Implementation Plan:** -- Investigate botasaurus's request module internals -- May require monkey-patching or upstream PR - ---- - -### 🟢 Priority 3: LOW - Browser-Based (Not Recommended) - -These libraries use browser automation where proxy handling is delegated to the browser engine. Custom proxy header support is **not feasible** without browser extensions or significant browser-level modifications. - -| Library | Stars | Reason for Low Priority | -|---------|-------|------------------------| -| crawl4ai | 59,235 | Browser-based (Playwright) | -| Scrapegraph-ai | 22,434 | Browser-based (Playwright) | -| playwright-python | 14,209 | Browser handles proxy | -| SeleniumBase | 12,139 | Browser handles proxy | -| Selenium | N/A | Browser handles proxy | -| splash | 4,198 | Qt WebKit-based | - -**Recommendation:** Do not implement extensions for these libraries. Instead, document that proxy header support is not possible due to browser architecture limitations. - ---- - -## Implementation Roadmap - -### Phase 1: Foundation (Weeks 1-2) -1. ✅ pycurl extension -2. ✅ cloudscraper extension (quick win) - -### Phase 2: High-Impact (Weeks 3-4) -3. curl_cffi extension (may require upstream work) -4. autoscraper extension - -### Phase 3: Ecosystem (Weeks 5-6) -5. treq extension -6. crawlee-python extension -7. requestium extension - -### Phase 4: Optional -8. botasaurus extension (if feasible) - ---- - -## File Structure - -``` -python_proxy_headers/ -├── __init__.py -├── urllib3_proxy_manager.py # Existing -├── requests_adapter.py # Existing -├── httpx_proxy.py # Existing -├── aiohttp_proxy.py # Existing -├── pycurl_proxy.py # NEW - Priority 1 -├── curl_cffi_proxy.py # NEW - Priority 1 -├── cloudscraper_proxy.py # NEW - Priority 1 -├── autoscraper_proxy.py # NEW - Priority 2 -├── treq_proxy.py # NEW - Priority 2 -├── crawlee_proxy.py # NEW - Priority 2 -├── requestium_proxy.py # NEW - Priority 2 -└── botasaurus_proxy.py # NEW - Priority 2 -``` - ---- - -## Documentation Updates - -For each new extension, add: -1. RST doc file in `docs/` -2. Entry in `docs/index.rst` -3. Usage example in README.md -4. Example code in proxy-examples repo - ---- - -*Created: January 30, 2026* diff --git a/LIBRARY_RESEARCH.md b/LIBRARY_RESEARCH.md deleted file mode 100644 index d1af467..0000000 --- a/LIBRARY_RESEARCH.md +++ /dev/null @@ -1,316 +0,0 @@ -# Python Library Proxy Header Support Research - -This document analyzes Python web scraping/automation libraries for their proxy support and potential for python-proxy-headers extension modules. - -## Executive Summary - -After reviewing 14 Python libraries, **none of them natively support sending custom headers to proxies or receiving proxy response headers** during HTTPS CONNECT tunneling. This is because: - -1. Browser automation tools (Playwright, Selenium, etc.) delegate proxy handling to the browser, which doesn't expose proxy headers -2. Higher-level scraping frameworks use underlying HTTP libraries (requests, httpx, aiohttp) that already lack this support -3. Libraries wrapping curl inherit libcurl's limitations around proxy header handling - ---- - -## Library Analysis - -### 1. cloudscraper (6,060 stars) -**GitHub:** https://github.com/venomous/cloudscraper -**Last Pushed:** 2025-06-10 -**Description:** Python module to bypass Cloudflare's anti-bot page - -**Proxy Support:** -- Uses `requests` library under the hood -- Supports proxy rotation via `rotating_proxies` parameter -- Standard requests-style proxy dict: `proxies={"http": "...", "https": "..."}` - -**Custom Proxy Headers:** ❌ No -- Inherits requests' limitations -- Cannot send custom headers to proxy or receive proxy response headers - -**Extension Feasibility:** ✅ HIGH -- Since cloudscraper wraps requests.Session, users can use our existing `HTTPProxyHeaderAdapter` -- Could create a `CloudScraperProxySession` that combines cloudscraper's features with proxy header support - ---- - -### 2. curl_cffi (4,873 stars) -**GitHub:** https://github.com/lexiforest/curl_cffi -**Last Pushed:** 2026-01-30 (very active) -**Description:** Python binding for curl-impersonate via cffi, can impersonate browser TLS/JA3 fingerprints - -**Proxy Support:** -- Supports HTTP/SOCKS proxies via `proxies` parameter -- Uses libcurl under the hood - -**Custom Proxy Headers:** ❌ No -- libcurl's `CURLOPT_PROXYHEADER` could theoretically be exposed but isn't -- No mechanism to capture proxy CONNECT response headers - -**Extension Feasibility:** ⚠️ MEDIUM-HIGH -- Would require adding Python bindings for `CURLOPT_PROXYHEADER` -- Could potentially capture CONNECT response via `CURLOPT_HEADERFUNCTION` -- Significant value as this library is used for anti-bot bypass - ---- - -### 3. pycurl (1,146 stars) -**GitHub:** https://github.com/pycurl/pycurl -**Last Pushed:** 2026-01-30 (very active) -**Description:** Python interface to libcurl - -**Proxy Support:** -- Full libcurl proxy support via `CURLOPT_PROXY` -- Supports `CURLOPT_PROXYHEADER` for sending headers to proxy - -**Custom Proxy Headers:** ⚠️ Partial -- `CURLOPT_PROXYHEADER` allows sending custom headers to proxy -- Receiving proxy response headers during CONNECT is tricky but possible via `CURLOPT_HEADERFUNCTION` - -**Extension Feasibility:** ✅ HIGH -- Already has underlying support via libcurl options -- Need a wrapper module to simplify the API for sending/receiving proxy headers -- Could create `ProxyCurl` class with easy `proxy_headers` parameter - ---- - -### 4. autoscraper (7,082 stars) -**GitHub:** https://github.com/alirezamika/autoscraper -**Last Pushed:** 2025-06-09 -**Description:** Smart, automatic web scraper for Python - -**Proxy Support:** -- Uses requests under the hood -- Proxy passed via `request_args=dict(proxies=proxies)` - -**Custom Proxy Headers:** ❌ No -- Inherits requests' limitations - -**Extension Feasibility:** ✅ HIGH -- Since it uses requests, could integrate with our existing requests adapter -- Low priority as it's a higher-level abstraction - ---- - -### 5. treq (606 stars) -**GitHub:** https://github.com/twisted/treq -**Last Pushed:** 2026-01-03 -**Description:** Python requests-like API built on Twisted's HTTP client - -**Proxy Support:** -- Uses Twisted's `Agent` for HTTP operations -- Proxy support via `twisted.web.client.ProxyAgent` - -**Custom Proxy Headers:** ❌ No -- ProxyAgent doesn't expose custom proxy header handling -- CONNECT tunnel headers not accessible - -**Extension Feasibility:** ⚠️ MEDIUM -- Would require creating custom `ProxyAgent` subclass -- Twisted's Agent architecture is complex -- Lower priority due to smaller user base - ---- - -### 6. crawl4ai (59,235 stars) 🔥 -**GitHub:** https://github.com/unclecode/crawl4ai -**Last Pushed:** 2026-01-30 (very active) -**Description:** LLM-friendly web crawler & scraper - -**Proxy Support:** -- Uses Playwright under the hood for browser automation -- Proxy configuration via `BrowserConfig` - -**Custom Proxy Headers:** ❌ No -- Browser handles proxy connection internally -- No access to CONNECT tunnel headers - -**Extension Feasibility:** ❌ LOW -- Browser-based - proxy handling is delegated to Chromium/Firefox -- Would require browser extension or CDP protocol hacks -- Not practical for this project - ---- - -### 7. Scrapegraph-ai (22,434 stars) -**GitHub:** https://github.com/ScrapeGraphAI/Scrapegraph-ai -**Last Pushed:** 2026-01-30 (very active) -**Description:** Python scraper based on AI/LLM - -**Proxy Support:** -- Uses Playwright for browser automation -- Configurable via graph config - -**Custom Proxy Headers:** ❌ No -- Same limitations as crawl4ai - browser handles proxy - -**Extension Feasibility:** ❌ LOW -- Browser-based, same limitations as above - ---- - -### 8. Selenium (Selenium Python bindings) -**Docs:** https://selenium-python.readthedocs.io/ -**Description:** Browser automation library - -**Proxy Support:** -- Proxy configured via browser options/capabilities -- Different methods for Chrome, Firefox, etc. - -**Custom Proxy Headers:** ❌ No -- Browser handles proxy CONNECT internally -- No programmatic access to proxy headers - -**Extension Feasibility:** ❌ LOW -- Would require browser extension -- Not practical for HTTP-level header manipulation - ---- - -### 9. requestium (1,838 stars) -**GitHub:** https://github.com/tryolabs/requestium -**Last Pushed:** 2026-01-26 -**Description:** Integration layer between Requests and Selenium - -**Proxy Support:** -- Requests-side: standard proxy dict -- Selenium-side: browser proxy settings - -**Custom Proxy Headers:** ❌ No -- Requests portion inherits requests' limitations -- Selenium portion has browser limitations - -**Extension Feasibility:** ⚠️ MEDIUM -- Could integrate our requests adapter for the requests portion -- Selenium side would still lack support - ---- - -### 10. splash (4,198 stars) -**GitHub:** https://github.com/scrapinghub/splash -**Last Pushed:** 2024-08-02 (less active) -**Description:** Lightweight browser as a service with HTTP API - -**Proxy Support:** -- Proxy can be configured per request -- Uses Qt WebKit/WebEngine internally - -**Custom Proxy Headers:** ❌ No -- Browser-based rendering -- Proxy handled by Qt networking layer - -**Extension Feasibility:** ❌ LOW -- Would require Qt-level modifications -- Project appears less actively maintained - ---- - -### 11. playwright-python (14,209 stars) -**GitHub:** https://github.com/microsoft/playwright-python -**Last Pushed:** 2026-01-30 (very active) -**Description:** Python version of Playwright testing/automation library - -**Proxy Support:** -- Proxy configured via `browser.launch(proxy={...})` -- Supports username/password authentication - -**Custom Proxy Headers:** ❌ No -- Browser handles CONNECT tunnel internally -- No CDP protocol support for custom proxy headers - -**Extension Feasibility:** ❌ LOW -- Browser delegates to system/browser proxy handling -- Would require Chromium DevTools Protocol extensions that don't exist - ---- - -### 12. SeleniumBase (12,139 stars) -**GitHub:** https://github.com/seleniumbase/SeleniumBase -**Last Pushed:** 2026-01-27 (very active) -**Description:** Web automation framework with anti-bot detection bypass - -**Proxy Support:** -- `--proxy=IP:PORT` command line option -- Supports authenticated proxies -- SOCKS4/SOCKS5 support - -**Custom Proxy Headers:** ❌ No -- Uses Selenium under the hood - browser handles proxy - -**Extension Feasibility:** ❌ LOW -- Browser-based, same Selenium limitations - ---- - -### 13. botasaurus (3,808 stars) -**GitHub:** https://github.com/omkarcloud/botasaurus -**Last Pushed:** 2026-01-10 -**Description:** Framework to build undetectable scrapers - -**Proxy Support:** -- `@browser(proxy="...")` decorator -- `@request(proxy="...")` for HTTP requests -- Uses custom driver and requests under the hood - -**Custom Proxy Headers:** ❌ No -- Browser portion: browser handles proxy -- Request portion: uses requests-like interface but no proxy header support - -**Extension Feasibility:** ⚠️ MEDIUM -- The `@request` decorator could potentially be extended -- Would require understanding their custom request implementation - ---- - -### 14. crawlee-python (7,968 stars) -**GitHub:** https://github.com/apify/crawlee-python -**Last Pushed:** 2026-01-30 (very active) -**Description:** Web scraping/browser automation library by Apify - -**Proxy Support:** -- Integrated proxy rotation -- Supports both HTTP (httpx-based) and browser (Playwright) crawlers - -**Custom Proxy Headers:** ❌ No -- BeautifulSoupCrawler uses httpx - inherits its limitations -- PlaywrightCrawler - browser handles proxy - -**Extension Feasibility:** ⚠️ MEDIUM -- BeautifulSoupCrawler could use our httpx extension -- Would require creating integration middleware - ---- - -## Summary Table - -| Library | Stars | Last Active | Proxy Headers | Extension Priority | -|---------|-------|-------------|---------------|-------------------| -| crawl4ai | 59,235 | 2026-01-30 | ❌ | LOW (browser-based) | -| Scrapegraph-ai | 22,434 | 2026-01-30 | ❌ | LOW (browser-based) | -| playwright-python | 14,209 | 2026-01-30 | ❌ | LOW (browser-based) | -| SeleniumBase | 12,139 | 2026-01-27 | ❌ | LOW (browser-based) | -| crawlee-python | 7,968 | 2026-01-30 | ❌ | MEDIUM (httpx portion) | -| autoscraper | 7,082 | 2025-06-09 | ❌ | HIGH (uses requests) | -| cloudscraper | 6,060 | 2025-06-10 | ❌ | HIGH (uses requests) | -| curl_cffi | 4,873 | 2026-01-30 | ❌ | HIGH (libcurl potential) | -| splash | 4,198 | 2024-08-02 | ❌ | LOW (Qt-based) | -| botasaurus | 3,808 | 2026-01-10 | ❌ | MEDIUM | -| requestium | 1,838 | 2026-01-26 | ❌ | MEDIUM | -| pycurl | 1,146 | 2026-01-30 | ⚠️ Partial | HIGH (has libcurl support) | -| treq | 606 | 2026-01-03 | ❌ | MEDIUM | - ---- - -## Conclusion - -**Browser-based libraries (Playwright, Selenium, crawl4ai, etc.) cannot support custom proxy headers** because the browser handles proxy CONNECT tunneling internally without exposing headers to the automation layer. - -**HTTP client libraries have the best potential for extension modules:** -1. **pycurl** - Already has libcurl's `CURLOPT_PROXYHEADER`, just needs wrapper -2. **curl_cffi** - Could expose libcurl's proxy header options -3. **cloudscraper** - Uses requests, can leverage existing adapter -4. **autoscraper** - Uses requests, can leverage existing adapter - ---- - -*Research conducted: January 30, 2026* diff --git a/README.md b/README.md index 10dacc4..eac3811 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,15 @@ [![Documentation Status](https://readthedocs.org/projects/python-proxy-headers/badge/?version=latest)](https://python-proxy-headers.readthedocs.io/en/latest/?badge=latest) [![PyPI version](https://badge.fury.io/py/python-proxy-headers.svg)](https://badge.fury.io/py/python-proxy-headers) -Extensions for Python HTTP libraries to support **sending and receiving custom proxy headers** during HTTPS CONNECT tunneling. +Extensions for Python HTTP libraries that send custom **proxy headers** on HTTPS CONNECT and read **proxy response headers** back. Works with any proxy provider; works best with ProxyMesh. ## The Problem When making HTTPS requests through a proxy, the connection is established via a CONNECT tunnel. During this process: -1. **Sending headers to the proxy** - Most Python HTTP libraries don't provide an easy way to send custom headers (like `X-ProxyMesh-Country`) to the proxy server during the CONNECT handshake. +1. **Sending proxy headers** - Most Python HTTP libraries don't provide an easy way to send custom **proxy headers** (like `X-ProxyMesh-Country`) to the proxy server during the CONNECT handshake. -2. **Receiving headers from the proxy** - The proxy's response headers from the CONNECT request are typically discarded, making it impossible to read custom headers (like `X-ProxyMesh-IP`) that the proxy sends back. +2. **Reading proxy response headers** - The proxy's CONNECT response headers are typically discarded, making it impossible to read **proxy response headers** (like `X-ProxyMesh-IP`) that the proxy sends back. This library solves both problems for popular Python HTTP libraries. @@ -60,7 +60,7 @@ with ProxySession(proxy_headers={'X-ProxyMesh-Country': 'US'}) as session: session.proxies = {'https': 'http://user:pass@proxy.example.com:8080'} response = session.get('https://httpbin.org/ip') - # Proxy headers are merged into response.headers + # Proxy response headers are merged into response.headers print(response.headers.get('X-ProxyMesh-IP')) ``` @@ -90,7 +90,7 @@ async def main(): 'https://httpbin.org/ip', proxy='http://user:pass@proxy.example.com:8080' ) as response: - # Proxy headers merged into response.headers + # Proxy response headers merged into response.headers print(response.headers.get('X-ProxyMesh-IP')) asyncio.run(main()) @@ -112,7 +112,7 @@ capture = HeaderCapture(c) c.perform() -print(capture.proxy_headers) # Headers from proxy CONNECT response +print(capture.proxy_headers) # Proxy response headers from the CONNECT response c.close() ``` @@ -165,9 +165,9 @@ For detailed documentation, API reference, and more examples: - **[ruby-proxy-headers](https://github.com/proxymesh/ruby-proxy-headers)** - Send and read custom proxy headers on HTTPS CONNECT for Ruby HTTP clients - **[scrapy-proxy-headers](https://github.com/proxymesh/scrapy-proxy-headers)** - Proxy header support for Scrapy -## About +## Why ProxyMesh -Created by [ProxyMesh](https://proxymesh.com) to help our customers use custom headers to control proxy behavior. Works with any proxy that supports custom headers. +[ProxyMesh](https://proxymesh.com) created this library to help customers use custom headers to control proxy behavior, as documented at [docs.proxymesh.com](https://docs.proxymesh.com/). It works with any proxy that supports custom headers on CONNECT. ## License diff --git a/python_proxy_headers/__init__.py b/python_proxy_headers/__init__.py index e69de29..93c58ff 100644 --- a/python_proxy_headers/__init__.py +++ b/python_proxy_headers/__init__.py @@ -0,0 +1 @@ +"""Send custom proxy headers on HTTPS CONNECT and read proxy response headers back.""" diff --git a/python_proxy_headers/aiohttp_proxy.py b/python_proxy_headers/aiohttp_proxy.py index af94f04..5be7095 100644 --- a/python_proxy_headers/aiohttp_proxy.py +++ b/python_proxy_headers/aiohttp_proxy.py @@ -1,3 +1,5 @@ +"""aiohttp client integration that sends proxy headers on CONNECT and merges proxy response headers back.""" + from http import HTTPStatus from aiohttp.client_reqrep import ClientRequest, ClientResponse from aiohttp.connector import TCPConnector, Connection diff --git a/python_proxy_headers/httpx_proxy.py b/python_proxy_headers/httpx_proxy.py index 2b35005..7c6c709 100644 --- a/python_proxy_headers/httpx_proxy.py +++ b/python_proxy_headers/httpx_proxy.py @@ -1,3 +1,5 @@ +"""httpx client integration that sends proxy headers on CONNECT and merges proxy response headers back.""" + import httpx if not httpx.__version__.startswith("0."): diff --git a/python_proxy_headers/requests_adapter.py b/python_proxy_headers/requests_adapter.py index 870a606..e95c485 100644 --- a/python_proxy_headers/requests_adapter.py +++ b/python_proxy_headers/requests_adapter.py @@ -1,3 +1,5 @@ +"""requests Session integration that sends proxy headers on CONNECT and exposes proxy response headers.""" + from requests.adapters import HTTPAdapter from requests.sessions import Session from .header_utils import validate_headers diff --git a/python_proxy_headers/urllib3_proxy_manager.py b/python_proxy_headers/urllib3_proxy_manager.py index 6a5c7a2..8d1ea48 100644 --- a/python_proxy_headers/urllib3_proxy_manager.py +++ b/python_proxy_headers/urllib3_proxy_manager.py @@ -1,3 +1,5 @@ +"""urllib3 ProxyManager integration that sends proxy headers on CONNECT and merges proxy response headers back.""" + import http import sys from http.client import _read_headers