Java Web Scraping: Jsoup, Playwright and Virtual Threads

<!-- Add to your pom.xml -->

<dependencies>
    <!-- Jsoup: The standard for HTML parsing -->
    <dependency>
        <groupId>org.jsoup</groupId>
        <artifactId>jsoup</artifactId>
        <version>1.17.2</version>
    </dependency>

    <!-- Playwright: The modern Selenium replacement -->
    <dependency>
        <groupId>com.microsoft.playwright</groupId>
        <artifactId>playwright</artifactId>
        <version>1.41.0</version>
    </dependency>

    <!-- Thordata SDK: For proxies & Web Unlocker -->
    <dependency>
        <groupId>com.thordata</groupId>
        <artifactId>thordata-java-sdk</artifactId>
        <version>1.1.0</version>
    </dependency>
</dependencies>

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

public class StaticScraper {
    public static void main(String[] args) throws Exception {
        String url = "https://books.toscrape.com/";

        // Configure connection to look like Chrome
        Document doc = Jsoup.connect(url)
                .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36...")
                .header("Accept-Language", "en-US,en;q=0.9")
                .ignoreContentType(true) // Important for some APIs
                .timeout(30000) 
                .get();

        // Extract data using CSS selectors
        for (Element book : doc.select("article.product_pod h3 a")) {
            System.out.println("Found Book: " + book.attr("title"));
        }
    }
}

import com.microsoft.playwright.*;

public class DynamicScraper {
    public static void main(String[] args) {
        try (Playwright playwright = Playwright.create()) {
            // Launch browser (Headless by default)
            Browser browser = playwright.chromium().launch(
                new BrowserType.LaunchOptions().setHeadless(true)
            );

            // Create a context (like an incognito window)
            BrowserContext context = browser.newContext();
            Page page = context.newPage();
            
            page.navigate("https://quotes.toscrape.com/js/");
            
            // Wait for dynamic content to render automatically
            Locator quotes = page.locator(".quote .text");
            System.out.println("First Quote: " + quotes.first().innerText());
        }
    }
}

import com.thordata.sdk.*;

public class ThordataExample {
    public static void main(String[] args) throws Exception {
        // 1. Initialize Configuration
        ThordataConfig config = new ThordataConfig(
            System.getenv("THORDATA_SCRAPER_TOKEN"), 
            null, null
        );
        ThordataClient client = new ThordataClient(config);

        // 2. Configure Universal Scrape (Web Unlocker)
        UniversalOptions opt = new UniversalOptions();
        opt.url = "https://example.com/spa";
        opt.jsRender = true;          // Execute JS remotely
        opt.waitFor = ".content";     // Wait for selector
        opt.outputFormat = "html";    

        // 3. Get Result without local browser overhead
        Object result = client.universalScrape(opt);
        System.out.println(result);
    }
}

import java.util.concurrent.Executors;
import java.util.stream.IntStream;

public class HighPerfScraper {
    public static void main(String[] args) {
        // New in Java 21: Virtual Thread Executor
        try (var executor = Executors.newVirtualThreadPerTaskExecutor()) {

            // Launch 1000 concurrent scrapers instantly
            IntStream.range(0, 1000).forEach(i -> {
                executor.submit(() -> {
                    System.out.println("Scraping task " + i + " on " + Thread.currentThread());
                    scrapeUrl(i); // Your Jsoup or SDK logic here
                });
            });
            
        } // Auto-waits for all tasks to finish
    }
}