migrate
This commit is contained in:
Mike McFetridge
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import deque
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from requests import Response
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
|
||||
WEBSITE_URL = r"""<< website_url >>"""
|
||||
MAX_PAGES_DEFAULT = int(r"""<< max_pages >>""")
|
||||
REQUEST_TIMEOUT_SECONDS_DEFAULT = int(r"""<< request_timeout_seconds >>""")
|
||||
VERIFY_TLS_DEFAULT = r"""<< verify_tls >>"""
|
||||
FOLLOW_REDIRECTS_DEFAULT = r"""<< follow_redirects >>"""
|
||||
REQUEST_HEADERS_JSON_DEFAULT = r"""<< request_headers_json >>"""
|
||||
USER_AGENT_DEFAULT = r"""<< user_agent >>"""
|
||||
|
||||
|
||||
def str_to_bool(value: Any) -> bool:
|
||||
return str(value).strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
|
||||
|
||||
def parse_json_object(value: str, field_name: str) -> dict[str, Any]:
|
||||
if not value.strip():
|
||||
return {}
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise SystemExit(f"{field_name} must be valid JSON: {exc}") from exc
|
||||
if not isinstance(parsed, dict):
|
||||
raise SystemExit(f"{field_name} must be a JSON object.")
|
||||
return parsed
|
||||
|
||||
|
||||
def build_session(headers: dict[str, str], user_agent: str) -> requests.Session:
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": user_agent})
|
||||
session.headers.update(headers)
|
||||
return session
|
||||
|
||||
|
||||
def sanitize_url(url: str) -> str | None:
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme in {"mailto", "javascript", "tel", "data"}:
|
||||
return None
|
||||
cleaned = parsed._replace(fragment="").geturl()
|
||||
return cleaned or None
|
||||
|
||||
|
||||
def same_host(url: str, website_host: str) -> bool:
|
||||
host = (urlparse(url).hostname or "").lower()
|
||||
return bool(host) and host == website_host
|
||||
|
||||
|
||||
def is_html_response(response: Response) -> bool:
|
||||
content_type = response.headers.get("Content-Type", "").lower()
|
||||
return "text/html" in content_type or "application/xhtml+xml" in content_type
|
||||
|
||||
|
||||
def probe_url(
|
||||
session: requests.Session,
|
||||
url: str,
|
||||
timeout: int,
|
||||
verify_tls: bool,
|
||||
follow_redirects: bool,
|
||||
) -> tuple[Response | None, str | None]:
|
||||
try:
|
||||
response = session.get(
|
||||
url,
|
||||
timeout=timeout,
|
||||
verify=verify_tls,
|
||||
allow_redirects=follow_redirects,
|
||||
)
|
||||
return response, None
|
||||
except RequestException as exc:
|
||||
return None, str(exc)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
website_url = WEBSITE_URL.strip()
|
||||
if not website_url:
|
||||
raise SystemExit("website_url is required.")
|
||||
website_host = (urlparse(website_url).hostname or "").lower()
|
||||
if not website_host:
|
||||
raise SystemExit("website_url must include a valid hostname.")
|
||||
|
||||
verify_tls = str_to_bool(VERIFY_TLS_DEFAULT)
|
||||
follow_redirects = str_to_bool(FOLLOW_REDIRECTS_DEFAULT)
|
||||
|
||||
request_headers = parse_json_object(REQUEST_HEADERS_JSON_DEFAULT, "request_headers_json")
|
||||
session = build_session(request_headers, USER_AGENT_DEFAULT.strip())
|
||||
|
||||
queue: deque[tuple[str, str]] = deque([(website_url, website_url)])
|
||||
queued_urls = {website_url}
|
||||
visited_pages: set[str] = set()
|
||||
checked_resources: dict[str, dict[str, Any]] = {}
|
||||
broken: dict[str, dict[str, Any]] = {}
|
||||
|
||||
pages_crawled = 0
|
||||
|
||||
while queue and pages_crawled < MAX_PAGES_DEFAULT:
|
||||
page_url, source = queue.popleft()
|
||||
if page_url in visited_pages:
|
||||
continue
|
||||
visited_pages.add(page_url)
|
||||
|
||||
response, error = probe_url(
|
||||
session=session,
|
||||
url=page_url,
|
||||
timeout=REQUEST_TIMEOUT_SECONDS_DEFAULT,
|
||||
verify_tls=verify_tls,
|
||||
follow_redirects=follow_redirects,
|
||||
)
|
||||
pages_crawled += 1
|
||||
|
||||
if error:
|
||||
broken.setdefault(
|
||||
page_url,
|
||||
{"url": page_url, "kind": "page", "status_code": None, "error": error, "sources": []},
|
||||
)["sources"].append(source)
|
||||
continue
|
||||
|
||||
if response is None:
|
||||
continue
|
||||
|
||||
if response.status_code >= 400:
|
||||
broken.setdefault(
|
||||
page_url,
|
||||
{
|
||||
"url": page_url,
|
||||
"kind": "page",
|
||||
"status_code": response.status_code,
|
||||
"error": f"HTTP {response.status_code}",
|
||||
"sources": [],
|
||||
},
|
||||
)["sources"].append(source)
|
||||
continue
|
||||
|
||||
if not is_html_response(response):
|
||||
continue
|
||||
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
|
||||
discovered: list[tuple[str, str]] = []
|
||||
for tag_name, attribute, kind in [
|
||||
("a", "href", "link"),
|
||||
("img", "src", "asset"),
|
||||
("script", "src", "asset"),
|
||||
("link", "href", "asset"),
|
||||
]:
|
||||
for tag in soup.find_all(tag_name):
|
||||
raw_value = tag.get(attribute)
|
||||
if not raw_value:
|
||||
continue
|
||||
normalized = sanitize_url(urljoin(page_url, raw_value))
|
||||
if not normalized:
|
||||
continue
|
||||
discovered.append((normalized, kind))
|
||||
|
||||
for discovered_url, kind in discovered:
|
||||
if not same_host(discovered_url, website_host):
|
||||
continue
|
||||
|
||||
if kind == "link" and discovered_url not in visited_pages and discovered_url not in queued_urls:
|
||||
queue.append((discovered_url, page_url))
|
||||
queued_urls.add(discovered_url)
|
||||
|
||||
if discovered_url in checked_resources:
|
||||
checked_resources[discovered_url]["sources"].add(page_url)
|
||||
if discovered_url in broken:
|
||||
broken[discovered_url]["sources"].append(page_url)
|
||||
continue
|
||||
|
||||
resource_response, resource_error = probe_url(
|
||||
session=session,
|
||||
url=discovered_url,
|
||||
timeout=REQUEST_TIMEOUT_SECONDS_DEFAULT,
|
||||
verify_tls=verify_tls,
|
||||
follow_redirects=follow_redirects,
|
||||
)
|
||||
|
||||
checked_resources[discovered_url] = {
|
||||
"url": discovered_url,
|
||||
"kind": kind,
|
||||
"sources": {page_url},
|
||||
}
|
||||
|
||||
if resource_error:
|
||||
broken[discovered_url] = {
|
||||
"url": discovered_url,
|
||||
"kind": kind,
|
||||
"status_code": None,
|
||||
"error": resource_error,
|
||||
"sources": [page_url],
|
||||
}
|
||||
continue
|
||||
|
||||
if resource_response is not None and resource_response.status_code >= 400:
|
||||
broken[discovered_url] = {
|
||||
"url": discovered_url,
|
||||
"kind": kind,
|
||||
"status_code": resource_response.status_code,
|
||||
"error": f"HTTP {resource_response.status_code}",
|
||||
"sources": [page_url],
|
||||
}
|
||||
|
||||
for value in broken.values():
|
||||
value["sources"] = sorted(set(value["sources"]))
|
||||
|
||||
summary = {
|
||||
"pages_crawled": pages_crawled,
|
||||
"urls_checked": len(set(checked_resources) | visited_pages),
|
||||
"broken_urls": len(broken),
|
||||
}
|
||||
overall_status = "fail" if broken else "pass"
|
||||
sorted_broken = sorted(broken.values(), key=lambda item: item["url"])
|
||||
|
||||
print(f"Broken Link Crawler: {overall_status.upper()}")
|
||||
print(f"Website: {website_url}")
|
||||
print(f"Pages crawled: {pages_crawled}")
|
||||
print(f"URLs checked: {summary['urls_checked']}")
|
||||
print(f"Broken URLs: {summary['broken_urls']}")
|
||||
if sorted_broken:
|
||||
print("")
|
||||
print("Broken targets:")
|
||||
for item in sorted_broken:
|
||||
source = ", ".join(item["sources"][:3]) or "-"
|
||||
print(f"- [{item['kind']}] {item['url']} | source={source} | error={item['error']}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,2 @@
|
||||
beautifulsoup4==4.14.3
|
||||
requests==2.33.1
|
||||
@@ -0,0 +1,110 @@
|
||||
{
|
||||
"slug": "broken-link-crawler",
|
||||
"kind": "python",
|
||||
"metadata": {
|
||||
"name": "Broken Link Crawler",
|
||||
"description": "Standalone Python crawler for one website that walks same-host pages, checks discovered links and assets, and prints a stdout summary.",
|
||||
"tags": [
|
||||
"crawler",
|
||||
"links",
|
||||
"website",
|
||||
"audit",
|
||||
"python"
|
||||
],
|
||||
"icon": {
|
||||
"provider": "simple-icons",
|
||||
"id": "python",
|
||||
"color": "emerald"
|
||||
},
|
||||
"draft": false,
|
||||
"version": {
|
||||
"name": "1.0.0",
|
||||
"source_dep_name": "manual/broken-link-crawler"
|
||||
}
|
||||
},
|
||||
"variables": [
|
||||
{
|
||||
"title": "Targets",
|
||||
"name": "targets",
|
||||
"items": [
|
||||
{
|
||||
"name": "website_url",
|
||||
"type": "str",
|
||||
"title": "Website URL",
|
||||
"required": true,
|
||||
"default": "https://example.com",
|
||||
"description": "Single crawl entrypoint. The crawler stays on this exact host.",
|
||||
"config": {
|
||||
"placeholder": "https://example.com"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Crawler",
|
||||
"name": "crawler",
|
||||
"items": [
|
||||
{
|
||||
"name": "max_pages",
|
||||
"type": "int",
|
||||
"title": "Max Pages",
|
||||
"required": true,
|
||||
"default": 100,
|
||||
"description": "Maximum number of HTML pages to crawl.",
|
||||
"config": {
|
||||
"placeholder": "100"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "request_timeout_seconds",
|
||||
"type": "int",
|
||||
"title": "Timeout Seconds",
|
||||
"required": true,
|
||||
"default": 10,
|
||||
"description": "Timeout for each outbound HTTP request.",
|
||||
"config": {
|
||||
"placeholder": "10"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "verify_tls",
|
||||
"type": "bool",
|
||||
"title": "Verify TLS",
|
||||
"required": false,
|
||||
"default": true,
|
||||
"description": "Validate remote TLS certificates during requests."
|
||||
},
|
||||
{
|
||||
"name": "follow_redirects",
|
||||
"type": "bool",
|
||||
"title": "Follow Redirects",
|
||||
"required": false,
|
||||
"default": true,
|
||||
"description": "Follow redirects when checking URLs."
|
||||
},
|
||||
{
|
||||
"name": "request_headers_json",
|
||||
"type": "str",
|
||||
"title": "Request Headers JSON",
|
||||
"required": false,
|
||||
"default": "{}",
|
||||
"description": "JSON object of headers added to every request.",
|
||||
"config": {
|
||||
"placeholder": "{\"Authorization\": \"Bearer replace-me\"}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "user_agent",
|
||||
"type": "str",
|
||||
"title": "User Agent",
|
||||
"required": false,
|
||||
"default": "boilerplates-broken-link-crawler/1.0",
|
||||
"description": "User-Agent header value used by the crawler.",
|
||||
"config": {
|
||||
"placeholder": "boilerplates-broken-link-crawler/1.0"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
|
||||
MONITOR_PATH = r"""<< monitor_path >>"""
|
||||
WARNING_PERCENT = int(r"""<< warning_percent >>""")
|
||||
|
||||
DISCORD_WEBHOOK_URL = r"""<< discord_webhook_url >>"""
|
||||
DISCORD_USERNAME = r"""<< discord_username >>"""
|
||||
|
||||
|
||||
def fetch_disk_usage() -> dict[str, str | int]:
|
||||
path = MONITOR_PATH.strip() or "/"
|
||||
|
||||
if not 1 <= WARNING_PERCENT <= 100:
|
||||
raise SystemExit("warning_percent must be between 1 and 100.")
|
||||
|
||||
result = subprocess.run(
|
||||
["df", "-P", path],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = result.stderr.strip() or result.stdout.strip() or "unknown local error"
|
||||
raise RuntimeError(f"Local df command failed with exit code {result.returncode}: {detail}")
|
||||
|
||||
lines = [line.strip() for line in result.stdout.splitlines() if line.strip()]
|
||||
if len(lines) < 2:
|
||||
raise RuntimeError(f"Unexpected df output: {result.stdout!r}")
|
||||
|
||||
parts = lines[-1].split(maxsplit=5)
|
||||
if len(parts) != 6:
|
||||
raise RuntimeError(f"Unable to parse df output line: {lines[-1]!r}")
|
||||
|
||||
usage_raw = parts[4]
|
||||
if not usage_raw.endswith("%"):
|
||||
raise RuntimeError(f"Unable to parse disk usage percentage: {usage_raw!r}")
|
||||
|
||||
return {
|
||||
"filesystem": parts[0],
|
||||
"blocks_kb": parts[1],
|
||||
"used_kb": parts[2],
|
||||
"available_kb": parts[3],
|
||||
"usage_percent": int(usage_raw[:-1]),
|
||||
"mounted_on": parts[5],
|
||||
"path": path,
|
||||
"host": socket.gethostname(),
|
||||
}
|
||||
|
||||
|
||||
def send_discord_warning(result: dict[str, str | int]) -> bool:
|
||||
webhook_url = DISCORD_WEBHOOK_URL.strip()
|
||||
if not webhook_url:
|
||||
return False
|
||||
|
||||
content = "\n".join(
|
||||
[
|
||||
"Disk usage warning",
|
||||
f"Host: {result['host']}",
|
||||
f"Path: {result['path']}",
|
||||
f"Mounted on: {result['mounted_on']}",
|
||||
f"Filesystem: {result['filesystem']}",
|
||||
f"Usage: {result['usage_percent']}% (threshold {WARNING_PERCENT}%)",
|
||||
f"Available: {result['available_kb']} KB",
|
||||
]
|
||||
)
|
||||
payload = json.dumps(
|
||||
{
|
||||
"username": DISCORD_USERNAME.strip() or "Disk Monitor",
|
||||
"content": content[:1900],
|
||||
}
|
||||
).encode("utf-8")
|
||||
request = urllib.request.Request(
|
||||
webhook_url,
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=10) as response: # noqa: S310
|
||||
if response.status >= 400:
|
||||
raise RuntimeError(f"Discord webhook returned HTTP {response.status}")
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
result = fetch_disk_usage()
|
||||
usage_percent = int(result["usage_percent"])
|
||||
status = "warn" if usage_percent >= WARNING_PERCENT else "pass"
|
||||
alert_sent = False
|
||||
|
||||
if status == "warn":
|
||||
alert_sent = send_discord_warning(result)
|
||||
|
||||
print(f"Server Disk Usage Monitor: {status.upper()}")
|
||||
print(f"Host: {result['host']}")
|
||||
print(f"Path: {result['path']}")
|
||||
print(f"Mounted on: {result['mounted_on']}")
|
||||
print(f"Filesystem: {result['filesystem']}")
|
||||
print(f"Usage: {usage_percent}%")
|
||||
print(f"Threshold: {WARNING_PERCENT}%")
|
||||
print(f"Available: {result['available_kb']} KB")
|
||||
print(f"Discord alert sent: {'yes' if alert_sent else 'no'}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1 @@
|
||||
# No external dependencies required.
|
||||
@@ -0,0 +1,83 @@
|
||||
{
|
||||
"slug": "server-disk-usage-monitor",
|
||||
"kind": "python",
|
||||
"metadata": {
|
||||
"name": "Server Disk Usage Monitor",
|
||||
"description": "Standalone Python monitor that checks local disk usage for one path and sends a Discord warning when usage reaches a configurable threshold.",
|
||||
"tags": [
|
||||
"disk",
|
||||
"monitoring",
|
||||
"discord",
|
||||
"python"
|
||||
],
|
||||
"icon": {
|
||||
"provider": "simple-icons",
|
||||
"id": "python",
|
||||
"color": "emerald"
|
||||
},
|
||||
"draft": false,
|
||||
"version": {
|
||||
"name": "1.0.0",
|
||||
"source_dep_name": "manual/server-disk-usage-monitor"
|
||||
}
|
||||
},
|
||||
"variables": [
|
||||
{
|
||||
"title": "Monitor",
|
||||
"name": "monitor",
|
||||
"items": [
|
||||
{
|
||||
"name": "monitor_path",
|
||||
"type": "str",
|
||||
"title": "Monitor Path",
|
||||
"required": true,
|
||||
"default": "/",
|
||||
"description": "Local path passed to `df -P`.",
|
||||
"config": {
|
||||
"placeholder": "/"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Alerting",
|
||||
"name": "alerting",
|
||||
"items": [
|
||||
{
|
||||
"name": "warning_percent",
|
||||
"type": "int",
|
||||
"title": "Warning Threshold Percent",
|
||||
"required": true,
|
||||
"default": 80,
|
||||
"description": "Send a warning when disk usage reaches or exceeds this percentage.",
|
||||
"config": {
|
||||
"slider": true,
|
||||
"min": 50,
|
||||
"max": 100,
|
||||
"step": 1,
|
||||
"placeholder": "80"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "discord_webhook_url",
|
||||
"type": "secret",
|
||||
"title": "Discord Webhook URL",
|
||||
"required": false,
|
||||
"default": "",
|
||||
"description": "Optional Discord webhook endpoint for warning notifications."
|
||||
},
|
||||
{
|
||||
"name": "discord_username",
|
||||
"type": "str",
|
||||
"title": "Discord Username",
|
||||
"required": false,
|
||||
"default": "Disk Monitor",
|
||||
"description": "Sender name used for Discord webhook messages.",
|
||||
"config": {
|
||||
"placeholder": "Disk Monitor"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user