app.py

"""
LinkedIn Job Scraper - Streamlit App
Search LinkedIn job postings using the Piloterr API
"""

import streamlit as st
import requests
import pandas as pd
from datetime import datetime
import json
from pathlib import Path

# Config file to store API key
CONFIG_FILE = Path(__file__).parent / ".config.json"


def load_api_key():
    """Load API key from config file"""
    try:
        if CONFIG_FILE.exists():
            with open(CONFIG_FILE, 'r') as f:
                config = json.load(f)
                return config.get('api_key', '')
    except Exception:
        pass
    return ''


def save_api_key(api_key):
    """Save API key to config file"""
    try:
        config = {}
        if CONFIG_FILE.exists():
            with open(CONFIG_FILE, 'r') as f:
                config = json.load(f)
        config['api_key'] = api_key
        with open(CONFIG_FILE, 'w') as f:
            json.dump(config, f)
        return True
    except Exception:
        return False


# Page config
st.set_page_config(
    page_title="LinkedIn Job Scraper",
    page_icon="💼",
    layout="wide",
    initial_sidebar_state="expanded"
)

# Header
st.title("LinkedIn Job Scraper")
st.caption("Search and export LinkedIn job postings via the Piloterr API")

# Sidebar for filters
with st.sidebar:
    st.header("⚙️ Settings")
    
    # Load saved API key
    saved_api_key = load_api_key()
    
    # API Key
    api_key = st.text_input(
        "🔑 Piloterr API Key",
        value=saved_api_key,
        type="password",
        placeholder="pk_xxxxxxxx...",
        help="Get your key at piloterr.com"
    )
    
    # API key management buttons
    col1, col2 = st.columns(2)
    with col1:
        save_clicked = st.button("💾 Save", use_container_width=True, key="save_api")
    with col2:
        clear_clicked = st.button("🗑️ Clear", use_container_width=True, key="clear_api")
    
    # Full width messages
    if save_clicked:
        if api_key:
            if save_api_key(api_key):
                st.success("✅ Saved!")
            else:
                st.error("❌ Error saving")
        else:
            st.warning("⚠️ Please enter an API key")
    
    if clear_clicked:
        if save_api_key(""):
            st.rerun()
    
    st.link_button("🔑 Get an API Key", "https://piloterr.com/login", use_container_width=True)
    
    st.divider()
    st.header("🔍 Filters")
    
    # Keyword
    keyword = st.text_input(
        "Keyword",
        placeholder="Data Engineer, Marketing...",
        help="Search term"
    )
    
    # Company ID
    company_id = st.text_input(
        "Company ID",
        placeholder="33246798",
        help="LinkedIn ID (e.g.: 33246798 = TikTok)"
    )
    
    # Posted date
    when_options = {
        "Any time": "",
        "Past 24 hours": "day",
        "Past week": "week",
        "Past month": "month"
    }
    when_label = st.selectbox(
        "📅 Date Posted",
        options=list(when_options.keys()),
        help="Publication date"
    )
    when = when_options[when_label]
    
    # Experience Level
    experience_options = {
        "All levels": "",
        "Internship": "internship",
        "Entry level": "entry_level",
        "Associate": "associate",
        "Mid-Senior": "mid_senior",
        "Director": "director"
    }
    experience_label = st.selectbox(
        "📊 Experience",
        options=list(experience_options.keys())
    )
    experience = experience_options[experience_label]
    
    # Job Type
    job_type_options = {
        "All types": "",
        "Full-time": "full_time",
        "Part-time": "part_time",
        "Contract": "contract",
        "Temporary": "temporary",
        "Internship": "internship"
    }
    job_type_label = st.selectbox(
        "💼 Job Type",
        options=list(job_type_options.keys())
    )
    job_type = job_type_options[job_type_label]
    
    # Flexibility
    flexibility_options = {
        "All": "",
        "Remote": "remote",
        "Hybrid": "flexible",
        "On-site": "on_site"
    }
    flexibility_label = st.selectbox(
        "🏠 Work Mode",
        options=list(flexibility_options.keys())
    )
    flexibility = flexibility_options[flexibility_label]
    
    # Geo ID
    geo_id = st.text_input(
        "📍 Location ID",
        placeholder="105015875",
        help="LinkedIn Geo ID (e.g.: 105015875 = France)"
    )
    
    # Limit
    limit = st.selectbox(
        "📊 Results Limit",
        options=[10, 20, 50, 100, 200],
        index=0,
        help="Maximum number of results to fetch"
    )
    
    st.divider()
    
    # Search button
    search_button = st.button("🔍 Search", use_container_width=True, type="primary")
    
    # API info
    st.info("📖 [Piloterr Documentation](https://docs.piloterr.com/v2/api-reference/linkedin/job/search)")


def search_linkedin_jobs_page(api_key, params, page=1):
    """Search jobs for a given page"""
    url = "https://piloterr.com/api/v2/linkedin/job/search"
    
    headers = {
        "Content-Type": "application/json",
        "x-api-key": api_key
    }
    
    # Filter empty params and add page
    filtered_params = {k: v for k, v in params.items() if v}
    filtered_params["page"] = page
    
    try:
        response = requests.get(url, headers=headers, params=filtered_params)
        response.raise_for_status()
        return response.json(), None
    except requests.exceptions.HTTPError as e:
        if response.status_code == 401:
            return None, "Invalid or expired API key"
        elif response.status_code == 429:
            return None, "Rate limit reached. Try again later."
        else:
            return None, f"HTTP Error {response.status_code}: {str(e)}"
    except requests.exceptions.RequestException as e:
        return None, f"Connection error: {str(e)}"
    except json.JSONDecodeError:
        return None, "Invalid API response"


def search_linkedin_jobs(api_key, params, limit=10, progress_bar=None):
    """Fetch jobs by looping through pages until limit is reached"""
    all_jobs = []
    page = 1
    max_pages = (limit // 10) + 1  # Estimate: ~10 results per page
    
    while len(all_jobs) < limit and page <= max_pages:
        if progress_bar:
            progress = min(len(all_jobs) / limit, 0.99)
            progress_bar.progress(progress, text=f"Page {page} • {len(all_jobs)}/{limit} results...")
        
        data, error = search_linkedin_jobs_page(api_key, params, page)
        
        if error:
            if len(all_jobs) > 0:
                break
            return None, error
        
        jobs = data if isinstance(data, list) else data.get('jobs', [])
        
        if not jobs or len(jobs) == 0:
            break
        
        all_jobs.extend(jobs)
        page += 1
    
    if progress_bar:
        progress_bar.progress(1.0, text="Done!")
    
    return all_jobs[:limit], None


def format_date(date_str):
    """Format ISO date to readable format"""
    try:
        dt = datetime.fromisoformat(date_str.replace('Z', '+00:00'))
        now = datetime.now(dt.tzinfo) if dt.tzinfo else datetime.now()
        delta = now - dt
        
        if delta.days == 0:
            return "Today"
        elif delta.days == 1:
            return "Yesterday"
        elif delta.days < 7:
            return f"{delta.days} days ago"
        elif delta.days < 30:
            weeks = delta.days // 7
            return f"{weeks} week{'s' if weeks > 1 else ''} ago"
        else:
            months = delta.days // 30
            return f"{months} month{'s' if months > 1 else ''} ago"
    except:
        return date_str


# Main area
if search_button:
    if not api_key:
        st.warning("⚠️ Please enter your API key in the sidebar")
    else:
        # Prepare params
        params = {
            "keyword": keyword,
            "company_id": company_id,
            "when": when,
            "experience_level": experience,
            "job_type": job_type,
            "flexibility": flexibility,
            "geo_id": geo_id
        }
        
        # Progress bar
        progress_bar = st.progress(0, text="Searching...")
        
        jobs, error = search_linkedin_jobs(api_key, params, limit=limit, progress_bar=progress_bar)
        
        if error:
            st.error(f"❌ {error}")
        elif jobs:
            if len(jobs) == 0:
                st.warning("🔍 No results found. Try adjusting your filters.")
            else:
                # Statistics
                pages_fetched = (len(jobs) // 10) + (1 if len(jobs) % 10 > 0 else 0)
                
                col1, col2, col3 = st.columns(3)
                col1.metric("📋 Jobs Found", len(jobs))
                col2.metric("🏢 Companies", len(set(j.get('company_name', '') for j in jobs)))
                col3.metric("📄 Pages Fetched", pages_fetched)
                
                st.divider()
                
                # Tabs for different views
                tab1, tab2 = st.tabs(["📊 Table", "📥 Export"])
                
                with tab1:
                    # Data table
                    df = pd.DataFrame(jobs)
                    
                    # Rename columns
                    column_mapping = {
                        'id': 'ID',
                        'title': 'Title',
                        'company_name': 'Company',
                        'location': 'Location',
                        'list_date': 'Date',
                        'url': 'Link'
                    }
                    
                    display_cols = [col for col in column_mapping.keys() if col in df.columns]
                    df_display = df[display_cols].rename(columns=column_mapping)
                    
                    st.dataframe(
                        df_display,
                        use_container_width=True,
                        hide_index=True,
                        column_config={
                            "Link": st.column_config.LinkColumn("Link")
                        }
                    )
                
                with tab2:
                    # Data export
                    st.subheader("📥 Export Results")
                    
                    col1, col2 = st.columns(2)
                    
                    with col1:
                        df = pd.DataFrame(jobs)
                        csv = df.to_csv(index=False)
                        st.download_button(
                            label="📄 Download CSV",
                            data=csv,
                            file_name=f"linkedin_jobs_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",
                            mime="text/csv",
                            use_container_width=True
                        )
                    
                    with col2:
                        json_str = json.dumps(jobs, indent=2, ensure_ascii=False)
                        st.download_button(
                            label="📋 Download JSON",
                            data=json_str,
                            file_name=f"linkedin_jobs_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json",
                            mime="application/json",
                            use_container_width=True
                        )
                    
                    st.success("✅ Ready to export to Airtable, NocoDB, or your CRM!")

else:
    # Welcome message
    st.info("👈 Configure your filters in the sidebar and click **Search** to get started.")
    
    with st.expander("ℹ️ How to use this app?"):
        st.markdown("""
        1. **Enter your Piloterr API key** in the sidebar
        2. **Configure your filters**: keyword, company, date posted, etc.
        3. **Click Search** to start scraping
        4. **Export results** as CSV or JSON
        
        **Useful parameters:**
        - `Company ID`: Find a company's LinkedIn ID in its URL (e.g.: `linkedin.com/company/33246798`)
        - `Location ID`: LinkedIn Geo ID (103644278 = United States, 105015875 = France)
        """)