WIZARDCODER 15B
Evolution-Instructed Code Generation

Downloading WizardCoder 15B model...\n\n📥 Model download: 15B parameters [████████████████████] 100%\n\n✅ WizardCoder 15B installed successfully!\n🚀 Model ready for code generation tasks!\n💻 Local AI development environment configured!

**Professional Web Scraping Implementation** **Python Web Scraper for Product Data Collection** ```python """ Professional Web Scraper for E-commerce Product Data Efficient data collection tool with proper error handling and rate limiting """ import requests from bs4 import BeautifulSoup import time import csv from dataclasses import dataclass from typing import List, Optional import logging # Configure logging for monitoring logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler('scraper.log'), logging.StreamHandler() ] ) @dataclass class ProductData: """Data structure for product information""" name: str price: str description: str url: str class ProductScraper: """ Professional web scraping class for e-commerce product data Includes proper error handling, rate limiting, and data validation """ def __init__(self, delay_range: tuple = (1, 3), max_retries: int = 3): self.delay_range = delay_range self.max_retries = max_retries self.session = requests.Session() self.scraped_products = [] # Professional headers for respectful web scraping self.session.headers.update({ 'User-Agent': 'Mozilla/5.0 (compatible; ProductScraper/1.0; Educational Purpose)', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate', 'Connection': 'keep-alive', }) def cast_politeness_spell(self): """A crucial incantation - respect the target realm's resources""" delay = random.uniform(*self.politeness_delay) logging.info(f"🕐 Casting politeness spell - waiting {delay:.2f} seconds...") time.sleep(delay) def divine_robots_txt(self, base_url: str) -> bool: """Consult the sacred robots.txt scroll before proceeding""" try: robots_url = urljoin(base_url, '/robots.txt') response = self.session.get(robots_url, timeout=10) if response.status_code == 200: logging.info(f"📜 Sacred robots.txt consulted for {base_url}") # Simple check - in production, use proper robots.txt parser return "Disallow: /" not in response.text return True except Exception as e: logging.warning(f"⚠️ Could not consult robots.txt: {e}") return True # Proceed with caution def cast_retrieval_spell(self, url: str) -> Optional[BeautifulSoup]: """The primary spell for retrieving and parsing web content""" for attempt in range(self.max_retries): try: logging.info(f"🌐 Casting retrieval spell on: {url} (Attempt {attempt + 1})") response = self.session.get(url, timeout=15) response.raise_for_status() # Detect encoding mystically if response.encoding is None: response.encoding = 'utf-8' soup = BeautifulSoup(response.text, 'html.parser') logging.info(f"✅ Spell successful! Retrieved {len(response.text)} characters") return soup except requests.RequestException as e: logging.error(f"❌ Spell failed on attempt {attempt + 1}: {e}") if attempt < self.max_retries - 1: self.cast_politeness_spell() else: logging.error(f"💀 All retrieval attempts failed for {url}") return None def extract_mystical_knowledge(self, soup: BeautifulSoup, url: str) -> ScrapedArtifact: """Extract the essential knowledge from the retrieved content""" # Title extraction spell title_spells = [ soup.find('title'), soup.find('h1'), soup.find('h2'), soup.find(attrs={'property': 'og:title'}), soup.find(attrs={'name': 'title'}) ] title = "Unknown Scroll" for spell_result in title_spells: if spell_result and spell_result.get_text(strip=True): title = spell_result.get_text(strip=True)[:200] # Limit length break # Content extraction spell content_containers = [ 'article', 'main', '.content', '#content', '.post', '.entry', '.article-body, 'section ] content = "" for container in content_containers: elements = soup.select(container) if elements: content = ' '.join([elem.get_text(strip=True) for elem in elements]) break # Fallback: extract all paragraph text if not content: paragraphs = soup.find_all('p') content = ' '.join([p.get_text(strip=True) for p in paragraphs]) # Metadata divination metadata = { 'description': '', 'keywords': '', 'author': '', 'published_date': '' } # Extract meta descriptions meta_desc = soup.find('meta', attrs={'name': 'description'}) or \ soup.find('meta', attrs={'property': 'og:description'}) if meta_desc and meta_desc.get('content'): metadata['description'] = meta_desc['content'] # Extract keywords meta_keywords = soup.find('meta', attrs={'name': 'keywords'}) if meta_keywords and meta_keywords.get('content'): metadata['keywords'] = meta_keywords['content'] # Extract author meta_author = soup.find('meta', attrs={'name': 'author'}) or \ soup.find('meta', attrs={'property': 'article:author'}) if meta_author and meta_author.get('content'): metadata['author'] = meta_author['content'] return ScrapedArtifact( title=title, url=url, content=content[:5000], # Limit content length metadata=metadata, timestamp=time.strftime('%Y-%m-%d %H:%M:%S') ) def perform_advanced_selenium_sorcery(self, url: str, wait_for_element: str = None) -> Optional[BeautifulSoup]: """Advanced spell for JavaScript-heavy realms using Selenium magic""" options = webdriver.ChromeOptions() options.add_argument('--headless') # Invisible spirit mode options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') options.add_argument('--user-agent=WizardCoder Archmage Educational Bot 1.0') try: logging.info(f"🔮 Invoking Selenium spirits for {url}") driver = webdriver.Chrome(options=options) driver.get(url) # Wait for specific element if specified if wait_for_element: wait = WebDriverWait(driver, 10) wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, wait_for_element))) # Additional wait for dynamic content time.sleep(3) # Extract the mystical HTML page_source = driver.page_source driver.quit() soup = BeautifulSoup(page_source, 'html.parser') logging.info("✨ Selenium sorcery successful!") return soup except Exception as e: logging.error(f"💀 Selenium spell failed: {e}") if 'driver' in locals(): driver.quit() return None def harvest_knowledge_from_realm(self, urls: List[str], use_selenium: bool = False) -> List[ScrapedArtifact]: """The master spell to harvest knowledge from multiple digital realms""" artifacts = [] for url in urls: try: # Check if we should proceed ethically base_url = f"{urlparse(url).scheme}://{urlparse(url).netloc}" if not self.divine_robots_txt(base_url): logging.warning(f"🚫 Robots.txt forbids access to {url}") continue # Cast appropriate retrieval spell if use_selenium: soup = self.perform_advanced_selenium_sorcery(url) else: soup = self.cast_retrieval_spell(url) if soup: artifact = self.extract_mystical_knowledge(soup, url) artifacts.append(artifact) self.harvested_artifacts.append(artifact) logging.info(f"📚 Knowledge harvested: {artifact.title[:50]}...") # Always cast politeness spell between requests if url != urls[-1]: # Not the last URL self.cast_politeness_spell() except Exception as e: logging.error(f"🔴 Failed to harvest from {url}: {e}") continue logging.info(f"🎉 Harvesting complete! Collected {len(artifacts)} mystical artifacts") return artifacts def transcribe_to_scroll(self, filename: str = 'harvested_knowledge.csv'): """Transcribe the harvested knowledge to a mystical scroll (CSV)""" if not self.harvested_artifacts: logging.warning("📜 No artifacts to transcribe!") return with open(filename, 'w', newline='', encoding='utf-8') as scroll: fieldnames = ['title', 'url', 'content', 'description', 'keywords', 'author', 'timestamp'] scribe = csv.DictWriter(scroll, fieldnames=fieldnames) scribe.writeheader() for artifact in self.harvested_artifacts: scribe.writerow({ 'title': artifact.title, 'url': artifact.url, 'content': artifact.content, 'description': artifact.metadata.get('description', ''), 'keywords': artifact.metadata.get('keywords', ''), 'author': artifact.metadata.get('author', ''), 'timestamp': artifact.timestamp }) logging.info(f"📜 Knowledge transcribed to mystical scroll: {filename}") # Demonstration of the Archmage's Web Scraping Powers if __name__ == "__main__": # Initialize the Web Scraping Wizard wizard = WebScrapingWizard(politeness_delay=(2, 4)) # Example URLs for educational demonstration educational_urls = [ 'https://httpbin.org/html', # Safe testing endpoint 'https://quotes.toscrape.com/', # Designed for scraping practice 'https://books.toscrape.com/', # Another practice site ] print("🧙‍♂️ Behold! The Archmage's Web Scraping Demonstration!") print("=" * 60) # Cast the knowledge harvesting spell artifacts = wizard.harvest_knowledge_from_realm(educational_urls) # Display the mystical results for i, artifact in enumerate(artifacts, 1): print(f"\n📚 Artifact {i}:") print(f" Title: {artifact.title}") print(f" URL: {artifact.url}") print(f" Content Preview: {artifact.content[:200]}...") print(f" Harvested: {artifact.timestamp}") # Transcribe to mystical scroll wizard.transcribe_to_scroll() print("\n✨ The Archmage's web scraping spell is complete!") print("Remember, young apprentice: With great power comes great responsibility.") print("Always respect robots.txt, rate limits, and website terms of service!") ``` **The Archmage's Wisdom:** *Adjusts mystical robes and peers through ancient spectacles* Remember, dear apprentice, this enchantment embodies the highest principles of ethical web scraping: 🔮 **Mystical Ethics**: Always consult robots.txt and respect website wishes ⏰ **Politeness Charms**: Built-in delays to avoid overwhelming servers 🛡️ **Protective Wards**: Error handling and retry mechanisms 📜 **Knowledge Preservation**: Structured data storage and logging ✨ **Selenium Sorcery**: Advanced spells for JavaScript-heavy realms Use this power wisely, for the great arcane law states: "With great scraping power comes great responsibility to be courteous and ethical!" *Wizard's staff glows with satisfied magical energy*