eelewator.py

# -*- coding: utf-8 -*-
"""eelewator

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1VjsFowfXZ7HJb84GVpp8qDMtSPaeDgo6
"""

!pip install regex beautifulsoup4 lxml tqdm xlsxwriter

#%%
from __future__ import unicode_literals
import requests
from bs4 import BeautifulSoup
import pandas as pd
import regex as re
import time
from datetime import datetime
import json
from tqdm.notebook import tqdm
from concurrent.futures import ThreadPoolExecutor, as_completed
import xlsxwriter
import os

from google.colab import drive
drive.mount('/content/drive')

BASE_URL = "https://e-elewator.org"
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
}
MAX_WORKERS = 5
SAVE_PATH = '/content/drive/MyDrive/data/'
os.makedirs(SAVE_PATH, exist_ok=True)

# === TEST MODE ===
TEST_MODE = False

# Strony do wykluczenia (nie są artykułami)
EXCLUDE_PATTERNS = [
    '/o-e-elewatorze', '/aktualny-numer', '/archiwum', '/spotkania',
    '/media', '/autorzy-e-elewatora', '/bibliografia', '/wydawca',
    '/kontakt', '/polityka-prywatnosci', '/kurier-szczecinski',
    '/23-11-2023-spotkanie', '/nr-', '/spotkania-'
]

#%%
def get_sitemap_links():
    """Pobiera linki artykułów z sitemapy"""
    url = f"{BASE_URL}/page-sitemap.xml"
    r = requests.get(url, headers=HEADERS, timeout=30)
    r.encoding = 'utf-8'
    soup = BeautifulSoup(r.text, 'lxml-xml')  # parser XML
    links = []
    for loc in soup.find_all('loc'):
        link = loc.text.strip()
        # Filtruj: tylko artykuły (wzorzec e-e-{numer}-{autor})
        if re.search(r'/e-e-\d+-', link):
            links.append(link)
    return links


def dictionary_of_article(article_link):
    """Scrapuje pojedynczy artykuł z e-elewator.org"""
    try:
        r = requests.get(article_link, headers=HEADERS, timeout=30)
        r.encoding = 'utf-8'
        if r.status_code != 200:
            print(f"  [{r.status_code}] {article_link}")
            return None
        soup = BeautifulSoup(r.text, 'lxml')

        # Tytuł - w <h1><strong>...</strong></h1> wewnątrz Elementor widget
        title_of_article = ""
        try:
            for h1 in soup.find_all('h1'):
                strong = h1.find('strong')
                if strong:
                    title_of_article = strong.text.strip()
                    break
                elif h1.text.strip():
                    title_of_article = h1.text.strip()
                    break
        except:
            pass

        # Numer wydania z URL (e-e-3 -> 3)
        numer = ""
        try:
            match = re.search(r'/e-e-(\d+)-', article_link)
            if match:
                numer = match.group(1)
        except:
            pass

        # Tekst artykułu - zbieramy z elementor-widget-container
        # Główna treść jest w kontenerze między nagłówkiem a biogramem
        article = ""
        try:
            # Zbieramy wszystkie paragrafy z widgetów tekstowych
            text_widgets = soup.select('div.elementor-widget-text-editor div.elementor-widget-container')
            all_paragraphs = []
            for widget in text_widgets:
                text = widget.get_text(separator='\n', strip=True)
                # Pomijamy krótkie nawigacyjne elementy i footer
                if len(text) > 50 and 'copyright' not in text.lower() and 'aktualności' not in text[:30]:
                    all_paragraphs.append(text)

            if all_paragraphs:
                # Najdłuższy blok to prawdopodobnie treść artykułu
                # Ale lepiej weźmy wszystkie poza ostatnim (biogram)
                if len(all_paragraphs) >= 2:
                    article = '\n\n'.join(all_paragraphs[:-1])  # bez biogramu
                else:
                    article = all_paragraphs[0]

            article = re.sub(r'\n{3,}', '\n\n', article).strip()
        except:
            pass

        # Biogram autora - ostatni widget tekstowy (zazwyczaj)
        biogram = ""
        try:
            if all_paragraphs and len(all_paragraphs) >= 2:
                biogram = all_paragraphs[-1]
        except:
            pass

        # Autor - z biogramu (pierwsze pogrubione imię i nazwisko) lub z URL
        author = ""
        try:
            if biogram:
                # Szukamy pogrubionego tekstu na początku biogramu
                match = re.match(r'^([A-ZŁŚŻŹĆŃ][a-ząęółśżźćń]+\s+[A-ZŁŚŻŹĆŃ][a-ząęółśżźćń\-]+)', biogram)
                if match:
                    author = match.group(1)
            if not author:
                # Fallback: z URL
                match = re.search(r'/e-e-\d+-(.+?)/?$', article_link)
                if match:
                    author = match.group(1).replace('-', ' ').title()
        except:
            pass

        # Data z sitemapy - nie ma w artykule, pomijamy lub bierzemy z lastmod
        date_of_publication = ""

        # Zdjęcia
        photos_links = None
        try:
            imgs = soup.select('div.elementor-widget-image img')
            photo_urls = []
            for img in imgs:
                src = img.get('data-src') or img.get('src', '')
                if src and 'e-e-' in src and 'logo' not in src and 'polecamy' not in src and 'MKiDN' not in src and 'eleWatorg' not in src:
                    photo_urls.append(src)
            photos_links = ' | '.join(photo_urls) if photo_urls else None
        except:
            pass

        return {
            "Link": article_link,
            "Data publikacji": date_of_publication,
            "Numer": numer,
            "Tytuł artykułu": title_of_article.replace('\xa0', ' '),
            "Tekst artykułu": article.replace('\xa0', ' '),
            "Autor": author,
            "Biogram": biogram.replace('\xa0', ' '),
            "Tagi": "",
            "Linki zewnętrzne": None,
            "Zdjęcia/Grafika": True if photos_links else False,
            "Linki do zdjęć": photos_links
        }
    except Exception as e:
        print(f"  WYJĄTEK: {article_link} - {e}")
        return None


#%% KROK 1: Test połączenia + pobranie linków z sitemapy
print("=== Pobieram linki z sitemapy ===")
articles_links = get_sitemap_links()
print(f"Znaleziono {len(articles_links)} artykułów w sitemapie")

if TEST_MODE:
    articles_links = articles_links[:10]

print(f"\nLinki do pobrania: {len(articles_links)}")
for link in articles_links[:5]:
    print(f"  {link}")

#%% KROK 2: Scrapuj artykuły
print(f"\n=== Scrapuję {len(articles_links)} artykułów ===")
all_results = []
errors = []

if TEST_MODE:
    for i, link in enumerate(articles_links):
        print(f"[{i+1}/{len(articles_links)}] {link}")
        result = dictionary_of_article(link)
        if result:
            all_results.append(result)
            print(f"  ✅ '{result['Tytuł artykułu'][:60]}' | nr {result['Numer']} | {result['Autor']}")
        else:
            errors.append(link)
            print(f"  ❌ błąd")
        time.sleep(0.5)
else:
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        futures = {executor.submit(dictionary_of_article, link): link for link in articles_links}
        for future in tqdm(as_completed(futures), total=len(futures), desc="Artykuły"):
            result = future.result()
            if result is not None:
                all_results.append(result)
            else:
                errors.append(futures[future])

print(f"\n=== WYNIK: {len(all_results)}/{len(articles_links)} ===")

#%% KROK 3: Podgląd
if all_results:
    df = pd.DataFrame(all_results)
    df = df.sort_values('Numer', ascending=False)
    print(df[["Numer", "Tytuł artykułu", "Autor"]].to_string())
    # Pokaż fragment tekstu pierwszego artykułu
    print(f"\n--- Fragment tekstu pierwszego artykułu ---")
    print(all_results[0]['Tekst artykułu'][:500])
else:
    print("Brak wyników!")

#%% KROK 4: Zapis (odkomentuj po teście + TEST_MODE = False)
today = datetime.today().date()

with open(f'{SAVE_PATH}e-elewator_{today}.json', 'w', encoding='utf-8') as f:
    json.dump(all_results, f, ensure_ascii=False, default=str)

with pd.ExcelWriter(
    f"{SAVE_PATH}e-elewator_{today}.xlsx",
    engine='xlsxwriter',
    engine_kwargs={'options': {'strings_to_urls': False}}
) as writer:
    df.to_excel(writer, 'Posts', index=False)

if errors:
    with open(f'{SAVE_PATH}e-elewator_errors_{today}.json', 'w', encoding='utf-8') as f:
        json.dump(errors, f, ensure_ascii=False, default=str)

print(f"Zapisano do {SAVE_PATH}e-elewator_{today}.json i .xlsx")
print(f"Artykułów: {len(all_results)}, Błędów: {len(errors)}")