herocode/herolib_python
13
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
despiegk
2025-08-20 04:01:35 +02:00
parent 749c89aefc
commit 6b9f0cf291
32 changed files with 327 additions and 511 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
lib/downloader/__init__.py Normal file
View File

412
lib/downloader/downloader.py
View File

@@ -1,412 +0,0 @@
import json
import logging
import mimetypes # Added
import os
from datetime import datetime, timedelta
from urllib.parse import urljoin, urlparse
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.linkextractors import LinkExtractor
from scrapy.utils.project import get_project_settings
# Configure logging
logging.basicConfig(
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
)
logger = logging.getLogger(__name__)
STATE_FILE_NAME = ".download_state.json"
class GenericDownloaderSpider(scrapy.Spider):
name = "generic_downloader"
def __init__(
self,
start_url,
dest_dir,
allowed_domains,
ignore_paths=None,
depth_limit=0,
follow_links=True,
max_age_hours=0,
state_data=None,
*args,
**kwargs,
):
super(GenericDownloaderSpider, self).__init__(*args, **kwargs)
self.start_urls = [start_url]
self.dest_dir = dest_dir
self.allowed_domains = allowed_domains
self.ignore_paths = ignore_paths if ignore_paths else []
self.depth_limit = int(depth_limit)
self.follow_links = bool(follow_links)
self.max_age_hours = int(max_age_hours)
self.state_data = state_data if state_data else {}
self.link_extractor = LinkExtractor(allow_domains=self.allowed_domains)
os.makedirs(self.dest_dir, exist_ok=True)
logger.info(f"Downloader initialized for {start_url}")
logger.info(f"Destination directory: {self.dest_dir}")
logger.info(f"Allowed domains: {self.allowed_domains}")
logger.info(f"Ignore paths: {self.ignore_paths}")
logger.info(f"Depth limit: {self.depth_limit}")
logger.info(f"Follow links: {self.follow_links}")
logger.info(f"Max age (hours): {self.max_age_hours}")
def _should_ignore(self, url_path):
for pattern in self.ignore_paths:
if pattern in url_path: # Simple substring match for now, can be regex
return True
return False
def _get_file_path(self, response): # Changed signature to take response
url = response.url
parsed_url = urlparse(url)
original_path = parsed_url.path # e.g. /foo/bar.html or /foo/ or /
# Determine base_name and current_ext from original_path
if original_path.endswith("/"):
base_name = "index"
current_ext = ""
# path_for_dirs is the path part that forms the directory structure
path_for_dirs = original_path.lstrip("/")
else:
path_basename = os.path.basename(original_path)
if (
not path_basename and original_path == "/"
): # Root path e.g. http://example.com
base_name = "index"
current_ext = ""
else: # e.g. /file.txt or /file_no_ext or /.config
base_name, current_ext = os.path.splitext(path_basename)
if not base_name and current_ext: # Hidden file like /.bashrc
base_name = current_ext # Treat .bashrc as base_name
current_ext = "" # No further extension part
path_for_dirs = os.path.dirname(original_path.lstrip("/"))
# Try to get extension from Content-Type
content_type = (
response.headers.get("Content-Type", b"")
.decode("utf-8")
.split(";")[0]
.strip()
)
mime_ext = mimetypes.guess_extension(content_type) if content_type else None
final_ext = current_ext
if mime_ext and not current_ext: # No path extension, use MIME type's
final_ext = mime_ext
elif (
mime_ext
and current_ext.lower() in [".htm", ".html"]
and mime_ext
and mime_ext.lower() not in [".htm", ".html"]
):
# Path had .html/.htm, but MIME type suggests something more specific
final_ext = mime_ext
logger.debug(
f"URL {url}: Path ext {current_ext} overridden by Content-Type ext {mime_ext}."
)
elif not final_ext and (
content_type.startswith("text/")
or content_type
in ["application/javascript", "application/json", "application/xml"]
):
# Fallback for common text types if no extension determined yet and no path ext
if not base_name.endswith(
(".js", ".css", ".json", ".xml", ".txt")
): # Avoid double .html.html
final_ext = ".html"
filename = base_name + final_ext
# Create path components for the directory structure
components = []
if path_for_dirs:
components.extend(comp for comp in path_for_dirs.split("/") if comp)
components.append(filename)
# Sanitize components
sane_components = []
for comp_idx, comp_val in enumerate(components):
# Basic sanitization: replace invalid chars, limit length, avoid '..'
# Allow '.' for filenames but not as a full component name if it's not the only char
if comp_val == "..":
continue # Skip parent dir references in path construction
sane_comp = "".join(
c if c.isalnum() or c in ["-", "_", "."] else "_" for c in comp_val
)
sane_comp = sane_comp[:150] # Limit component length
if (
not sane_comp and comp_idx == len(components) - 1
): # last component (filename) became empty
sane_comp = "downloaded_file" + final_ext # fallback filename
elif not sane_comp:
sane_comp = "_" # placeholder for empty dir name
if sane_comp: # Ensure component is not empty after sanitization
sane_components.append(sane_comp)
if not sane_components: # If all components were sanitized away or skipped
sane_components = [filename if filename else "unknown_file" + final_ext]
file_path = os.path.join(self.dest_dir, *sane_components)
return file_path
def parse(self, response, depth=0):
url = response.url
logger.info(f"Processing URL (depth {depth}): {url}")
parsed_url = urlparse(url)
if self._should_ignore(parsed_url.path):
logger.info(f"Ignoring URL (matches ignore_paths): {url}")
return
file_path = self._get_file_path(response) # Pass response object
# Check download state and max_age
if url in self.state_data:
url_state = self.state_data[url]
last_download_time_str = url_state.get("timestamp")
# Consider previous status; only skip if it was a success or another skip
can_skip_based_on_history = url_state.get("status", "").startswith(
"success"
) or url_state.get("status", "").startswith("skipped")
if last_download_time_str and can_skip_based_on_history:
last_download_time = datetime.fromisoformat(last_download_time_str)
if self.max_age_hours > 0 and (
datetime.utcnow() - last_download_time
) < timedelta(hours=self.max_age_hours):
logger.info(
f"Skipping download for {url}, recently processed at {last_download_time_str} with status '{url_state.get('status')}'."
)
# Update state to reflect this skip check
self.state_data[url]["status"] = "skipped_max_age"
self.state_data[url]["skipped_timestamp"] = (
datetime.utcnow().isoformat()
)
# Still need to check for links if recursive
# Corrected depth condition:
# Follow if self.depth_limit is 0 (infinite) OR current depth is less than a positive limit.
if self.follow_links and (
self.depth_limit == 0 or depth < self.depth_limit
):
for link in self.link_extractor.extract_links(response):
parsed_link_url = urlparse(link.url)
if not self._should_ignore(parsed_link_url.path):
yield response.follow(link, callback=self.parse)
else:
logger.info(
f"Ignoring extracted link (matches ignore_paths): {link.url}"
)
return
logger.info(f"Processing and saving {url} to {file_path}")
os.makedirs(os.path.dirname(file_path), exist_ok=True)
try:
with open(file_path, "wb") as f:
f.write(response.body)
logger.info(f"Successfully saved {url} to {file_path}")
self.state_data[url] = {
"timestamp": datetime.utcnow().isoformat(),
"status": "success",
"path": file_path,
"size": len(response.body),
}
except Exception as e:
logger.error(f"Failed to save {url} to {file_path}: {e}")
self.state_data[url] = {
"timestamp": datetime.utcnow().isoformat(),
"status": "failed",
"error": str(e),
}
return # Do not proceed further if save failed
# Corrected depth condition for following links:
# Follow if self.depth_limit is 0 (infinite) OR current depth is less than a positive limit.
if self.follow_links and (self.depth_limit == 0 or depth < self.depth_limit):
logger.info(
f"Following links from {url} at custom depth {depth} (for next level {depth + 1})"
)
extracted_links = list(self.link_extractor.extract_links(response))
if not extracted_links:
logger.info(f" No links extracted from {url} by LinkExtractor.")
else:
logger.info(
f" LinkExtractor found {len(extracted_links)} links from {url}: {[l.url for l in extracted_links]}"
)
for link_idx, link in enumerate(extracted_links):
logger.debug(
f" Considering link {link_idx + 1}/{len(extracted_links)}: Text='{link.text}', URL='{link.url}'"
)
parsed_link_url = urlparse(link.url)
if self._should_ignore(parsed_link_url.path):
logger.info(
f" Ignoring extracted link (matches ignore_paths): {link.url}"
)
else:
logger.info(
f" Yielding request for: {link.url} (to be processed at custom depth {depth + 1})"
)
yield response.follow(link, callback=self.parse)
def closed(self, reason):
logger.info(f"Spider closed: {reason}. Finalizing and saving state.")
state_file_path = os.path.join(self.dest_dir, STATE_FILE_NAME)
try:
# Ensure the directory for the state file exists, though dest_dir should already.
os.makedirs(os.path.dirname(state_file_path), exist_ok=True)
with open(state_file_path, "w") as f:
json.dump(self.state_data, f, indent=4)
logger.info(
f"Spider successfully saved state ({len(self.state_data)} items) to {state_file_path}"
)
except Exception as e:
logger.error(
f"Spider failed to save state to {state_file_path}: {e}", exc_info=True
)
def download_site(
start_url,
dest_dir,
recursive=True,
ignore_paths=None,
depth_limit=0, # 0 means no limit if recursive is True
follow_links=True, # This is somewhat redundant if recursive is True, but good for clarity
max_age_hours=24, # Re-download if older than 24 hours
):
"""
Downloads a website or a single page.
:param start_url: The URL to start downloading from.
:param dest_dir: The directory to save downloaded files.
:param recursive: Whether to download recursively.
:param ignore_paths: A list of path substrings or regex patterns to ignore.
:param depth_limit: Maximum depth for recursive downloads (0 for no limit).
:param follow_links: Whether to follow links on pages (primarily for recursive).
:param max_age_hours: Max age of a file in hours. If a file was downloaded
more recently than this, it won't be re-downloaded.
0 means always re-download.
:return: A dictionary summarizing the download process.
"""
parsed_url = urlparse(start_url)
if not parsed_url.scheme or not parsed_url.netloc:
logger.error(
f"Invalid start_url: {start_url}. Must be a full URL (e.g., http://example.com)"
)
return None
allowed_domains = [parsed_url.hostname] # Changed from netloc to hostname
state_file_path = os.path.join(dest_dir, STATE_FILE_NAME)
state_data = {}
if os.path.exists(state_file_path):
try:
with open(state_file_path, "r") as f:
state_data = json.load(f)
logger.info(f"Loaded download state from {state_file_path}")
except json.JSONDecodeError:
logger.warning(
f"Could not decode JSON from state file {state_file_path}. Starting fresh."
)
except Exception as e:
logger.error(
f"Error loading state file {state_file_path}: {e}. Starting fresh."
)
settings = get_project_settings()
settings.set("ROBOTSTXT_OBEY", False) # Explicitly disable robots.txt
# settings.set('LOG_LEVEL', 'DEBUG') # Optionally enable for more Scrapy internal logs
effective_scrapy_depth = 0 # Default for non-recursive or depth_limit=0 with recursion (0 means infinite for Scrapy)
if recursive and int(depth_limit) > 0:
effective_scrapy_depth = int(depth_limit)
# If not recursive, effective_scrapy_depth remains 0.
# If recursive and depth_limit is 0, effective_scrapy_depth remains 0 (infinite).
settings.set("DEPTH_LIMIT", effective_scrapy_depth)
logger.info(f"Scrapy DEPTH_LIMIT set to: {effective_scrapy_depth}")
# Scrapy's DEPTH_PRIORITY and SCHEDULER_DISK_QUEUE might be useful for large crawls
# For now, keeping it simple.
process = CrawlerProcess(settings)
# The spider needs to be instantiated with all its custom args
# Scrapy's process.crawl can take kwargs which are passed to the spider's __init__
process.crawl(
GenericDownloaderSpider,
start_url=start_url,
dest_dir=dest_dir,
allowed_domains=allowed_domains,
ignore_paths=ignore_paths,
depth_limit=int(depth_limit)
if recursive
else 0, # Spider handles its own depth based on this
follow_links=follow_links and recursive,
max_age_hours=int(max_age_hours),
state_data=state_data,
)
logger.info(f"Starting download process for {start_url}...")
process.start() # This will block until the crawl is finished
# The spider's closed() method is now responsible for writing the final state.
# Load this definitive state to build the summary.
final_state_data_for_summary = {}
if os.path.exists(state_file_path):
try:
with open(state_file_path, "r") as f:
final_state_data_for_summary = json.load(f)
logger.info(
f"Loaded final state ({len(final_state_data_for_summary)} items) from {state_file_path} for summary construction."
)
except json.JSONDecodeError as e:
logger.error(
f"Error decoding JSON from final state file {state_file_path} for summary: {e}. Summary will be based on empty or incomplete state."
)
except Exception as e:
logger.error(
f"Error loading final state from {state_file_path} for summary: {e}. Summary will be based on empty or incomplete state."
)
else:
logger.warning(
f"State file {state_file_path} not found after spider closed. Summary will be based on empty state."
)
summary = {
"start_url": start_url,
"dest_dir": dest_dir,
"total_processed_urls": len(final_state_data_for_summary),
"successful_downloads": 0,
"failed_downloads": 0,
"skipped_max_age": 0,
"total_bytes_downloaded": 0,
"state_file_path": state_file_path,
"errors": [],
}
# Populate summary from the final_state_data_for_summary loaded from the file
for url_key, data_val in final_state_data_for_summary.items():
status = data_val.get("status")
if status == "success":
summary["successful_downloads"] += 1
summary["total_bytes_downloaded"] += data_val.get("size", 0)
elif status == "failed":
summary["failed_downloads"] += 1
if "error" in data_val:
summary["errors"].append(f"URL: {url_key}, Error: {data_val['error']}")
elif status == "skipped_max_age":
summary["skipped_max_age"] += 1
# Any errors during state file loading for summary should also be noted if critical
# For now, the logs capture it. If final_state_data_for_summary is empty due to load error, summary will reflect that.
logger.info(f"Download process finished. Summary: {json.dumps(summary, indent=2)}")
return summary

1
lib/downloader/scrape_dynamic Symbolic link
View File

@@ -0,0 +1 @@
../../../../tfgrid_research/tfdev/research/scrape_dynamic

1
lib/downloader/scrape_fast Symbolic link
View File

@@ -0,0 +1 @@
../../../../tfgrid_research/tfdev/research/scrape_fast

1
lib/downloader/scrape_scapegraph Symbolic link
View File

@@ -0,0 +1 @@
../../../../tfgrid_research/tfdev/research/scrape_scapegraph