Web Scraping in Java in 2024: The Complete Guide

java -version

openjdk version "21" 2023-09-19 LTS
OpenJDK Runtime Environment Temurin-21+35 (build 21+35-LTS)
OpenJDK 64-Bit Server VM Temurin-21+35 (build 21+35-LTS, mixed mode, sharing)

gradle -v

 ------------------------------------------------------------
Gradle 8.4
------------------------------------------------------------

Build time:   2023-10-04 20:52:13 UTC
Revision:     e9251e572c9bd1d01e503a0dfdf43aedaeecdc3f

Kotlin:       1.9.10
Groovy:       3.0.17
Ant:          Apache Ant(TM) version 1.10.13 compiled on January 4 2023
JVM:          21 (Eclipse Adoptium 21+35-LTS)
OS:           Linux 6.2.0-34-generic amd64

mvn -v

Apache Maven 3.9.5 (57804ffe001d7215b5e7bcb531cf83df38f93546)

implementation "org.jsoup:jsoup:1.16.1"

<dependency> 
	<groupId>org.jsoup</groupId> 
	<artifactId>jsoup</artifactId> 
	<version>1.16.1</version> 
</dependency>

import org.jsoup.*; 
import org.jsoup.nodes.*; 
import org.jsoup.select.*;

// initializing the HTML Document page variable 
Document doc; 
 
try { 
	// fetching the target website 
	doc = Jsoup.connect("https://www.scrapingcourse.com/ecommerce/").get(); 
} catch (IOException e) { 
	throw new RuntimeException(e); 
}

doc = Jsoup 
	.connect("https://www.scrapingcourse.com/ecommerce/") 
	.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36") 
	.header("Accept-Language", "*") 
	.get();

package com.zenrows.data; 
 
public class Product { 
	private String url; 
	private String image; 
	private String name; 
	private String price; 
 
	// getters and setters omitted for brevity... 
 
	@Override 
	public String toString() { 
		return "{ \"url\":\"" + url + "\", " 
				+ " \"image\": \"" + image + "\", " 
				+ "\"name\":\"" + name + "\", " 
				+ "\"price\": \"" + price + "\" }"; 
	} 
}

Elements productElements = doc.select("li.product");

// initializing the list of Java object to store 
// the scraped data 
List<Product> products = new ArrayList<>(); 
 
// retrieving the list of product HTML elements 
Elements productElements = doc.select("li.product"); 
 
// iterating over the list of HTML products 
for (Element productElement : productElements) { 
	Product product = new Product(); 
 
	// extracting the data of interest from the product HTML element 
	// and storing it in product
	product.setUrl(productElement.selectFirst("a").attr("href")); 
	product.setImage(productElement.selectFirst("img").attr("src")); 
	product.setName(productElement.selectFirst("h2").text()); 
	product.setPrice(productElement.selectFirst("span").text()); 
 
	// adding product to the list of the scraped products 
	products.add(product); 
}

products.toString();
System.out.println(products);

[ 
	{ 
		"url": "https://www.scrapingcourse.com/ecommerce/product/abominable-hoodie/", 
		"image": "https://www.scrapingcourse.com/ecommerce/wp-content/uploads/2024/03/mh09-blue_main-324x324.jpg", 
		"name": "Abominable Hoodie", 
		"price": "$69.00" 
	}, 
 
	// ... 
 
	{ 
		"url": "https://www.scrapingcourse.com/ecommerce/product/artemis-running-short/", 
		"image": "https://www.scrapingcourse.com/ecommerce/wp-content/uploads/2024/03/wsh04-black_main-324x324.jpg", 
		"name": "Artemis Running Short", 
		"price": "$49.00" 
	} 
]

Elements paginationElements = doc.select("a.page-numbers");

package com.zenrows; 
 
import com.zenrows.data.Product; 
import org.jsoup.*; 
import org.jsoup.nodes.*; 
import org.jsoup.select.*; 
import java.io.IOException; 
import java.util.*; 
 
public class Scraper { 
	public static void scrapeProductPage( 
			List<Product> products, 
			Set<String> pagesDiscovered, 
			List<String> pagesToScrape 
	) { 
		// the current web page is about to be scraped and 
		// should no longer be part of the scraping queue 
		String url = pagesToScrape.remove(0); 
 
		pagesDiscovered.add(url); 
 
		// ... scraping logic omitted for brevity

		Elements paginationElements = doc.select("a.page-numbers");
 
		// iterating over the pagination HTML elements 
		for (Element pageElement : paginationElements) { 
			// the new link discovered 
			String pageUrl = pageElement.attr("href"); 
 
			// if the web page discovered is new and should be scraped 
			if (!pagesDiscovered.contains(pageUrl) && !pagesToScrape.contains(pageUrl)) { 
				pagesToScrape.add(pageUrl); 
			} 
 
			// adding the link just discovered 
			// to the set of pages discovered so far 
			pagesDiscovered.add(pageUrl); 
		} 
	} 
 
	public static void main(String[] args) { 
		// initializing the list of Java object to store 
		// the scraped data 
		List<Product> products = new ArrayList<>(); 
 
		// initializing the set of web page urls 
		// discovered while crawling the target website 
		Set<String> pagesDiscovered = new HashSet<>(); 
 
		// initializing the queue of urls to scrape 
		List<String> pagesToScrape = new ArrayList<>(); 
		// initializing the scraping queue with the 
		// first pagination page 
		pagesToScrape.add("https://www.scrapingcourse.com/ecommerce/page/1/"); 
 
		// the number of iteration executed 
		int i = 0; 
		// to limit the number to scrape to 5 
		int limit = 12; 
 
		while (!pagesToScrape.isEmpty() && i < limit) { 
			scrapeProductPage(products, pagesDiscovered, pagesToScrape); 
			// incrementing the iteration number 
			i++; 
		} 
 
		System.out.println(products.size()); 
 
		// writing the scraped data to a db or export it to a file... 
	} 
}

package com.zenrows;

import com.zenrows.data.Product;
import org.jsoup.*;
import org.jsoup.nodes.*;
import org.jsoup.select.*;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

public class Scraper {
    public static void scrapeProductPage(
            List<Product> products,
            Set<String> pagesDiscovered,
            List<String> pagesToScrape
    ) {
        if (!pagesToScrape.isEmpty()) {
            // the current web page is about to be scraped and
            // should no longer be part of the scraping queue
            String url = pagesToScrape.remove(0);

            pagesDiscovered.add(url);

            // initializing the HTML Document page variable
            Document doc;

            try {
                // fetching the target website
                doc = Jsoup
                        .connect(url)
                        .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
                        .get();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }

            // retrieving the list of product HTML elements
            // in the target page
            Elements productElements = doc.select("li.product");

            // iterating over the list of HTML products
            for (Element productElement : productElements) {
                Product Product = new Product();

                // extracting the data of interest from the product HTML element
                // and storing it in Product
                Product.setUrl(productElement.selectFirst("a").attr("href"));
                Product.setImage(productElement.selectFirst("img").attr("src"));
                Product.setName(productElement.selectFirst("h2").text());
                Product.setPrice(productElement.selectFirst("span").text());

                // adding Product to the list of the scraped products
                products.add(Product);
            }

            // retrieving the list of pagination HTML element
            Elements paginationElements = doc.select("a.page-numbers");

            // iterating over the pagination HTML elements
            for (Element pageElement : paginationElements) {
                // the new link discovered
                String pageUrl = pageElement.attr("href");

                // if the web page discovered is new and should be scraped
                if (!pagesDiscovered.contains(pageUrl) && !pagesToScrape.contains(pageUrl)) {
                    pagesToScrape.add(pageUrl);
                }

                // adding the link just discovered
                // to the set of pages discovered so far
                pagesDiscovered.add(pageUrl);
            }

            // logging the end of the scraping operation
            System.out.println(url + " -> page scraped");
        }
    }

    public static void main(String[] args) { 
		// initializing the list of Java object to store 
		// the scraped data 
		List<Product> products = new ArrayList<>(); 
 
		// initializing the set of web page urls 
		// discovered while crawling the target website 
		Set<String> pagesDiscovered = new HashSet<>(); 
 
		// initializing the queue of urls to scrape 
		List<String> pagesToScrape = new ArrayList<>(); 
		// initializing the scraping queue with the 
		// first pagination page 
		pagesToScrape.add("https://www.scrapingcourse.com/ecommerce/page/1/"); 
 
		// the number of iteration executed 
		int i = 0; 
		// to limit the number to scrape to 5 
		int limit = 12; 
 
		while (!pagesToScrape.isEmpty() && i < limit) { 
			scrapeProductPage(products, pagesDiscovered, pagesToScrape); 
			// incrementing the iteration number 
			i++; 
		} 
 
		System.out.println(products.size());
        // writing the scraped data to a db or export it to a file...
    }
}

package com.zenrows; 
 
import com.zenrows.data.Product; 
import org.jsoup.*; 
import org.jsoup.nodes.*; 
import org.jsoup.select.*; 
import java.io.IOException; 
import java.util.*; 
import java.util.concurrent.ExecutorService; 
import java.util.concurrent.Executors; 
import java.util.concurrent.TimeUnit; 
 
public class Scraper { 
	public static void scrapeProductPage( 
			List<Product> products, 
			Set<String> pagesDiscovered, 
			List<String> pagesToScrape 
	) { 
		//... omitted for brevity
	} 
 
	public static void main(String[] args) throws InterruptedException { 
		// initializing the list of Java object to store 
		// the scraped data 
		List<Product> products = Collections.synchronizedList(new ArrayList<>()); 
 
		// initializing the set of web page urls 
		// discovered while crawling the target website 
		Set<String> pagesDiscovered = Collections.synchronizedSet(new HashSet<>()); 
 
		// initializing the queue of urls to scrape 
		List<String> pagesToScrape = Collections.synchronizedList(new ArrayList<>()); 
		// initializing the scraping queue with the 
		// first pagination page 
		pagesToScrape.add("https://www.scrapingcourse.com/ecommerce/page/1/"); 
 
		// initializing the ExecutorService to run the 
		// web scraping process in parallel on 4 pages at a time 
		ExecutorService executorService = Executors.newFixedThreadPool(4); 
 
		// launching the web scraping process to discover some 
		// urls and take advantage of the parallelization process 
		scrapeProductPage(products, pagesDiscovered, pagesToScrape); 
 
		// the number of iteration executed 
		int i = 1; 
		// to limit the number to scrape to 5 
		int limit = 10; 
 
		while (!pagesToScrape.isEmpty() && i < limit) { 
			// registering the web scraping task 
			executorService.execute(() -> scrapeProductPage(products, pagesDiscovered, pagesToScrape)); 
 
			// adding a 200ms delay to avoid overloading the server 
			TimeUnit.MILLISECONDS.sleep(200); 
 
			// incrementing the iteration number 
			i++; 
		} 
 
		// waiting up to 300 seconds for all pending tasks to end 
		executorService.shutdown(); 
		executorService.awaitTermination(300, TimeUnit.SECONDS); 
 
		System.out.println(products.size()); 
	} 
}

package com.zenrows;

import com.zenrows.data.Product;
import org.jsoup.*;
import org.jsoup.nodes.*;
import org.jsoup.select.*;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

public class Scraper {
    public static void scrapeProductPage(
            List<Product> products,
            Set<String> pagesDiscovered,
            List<String> pagesToScrape
    ) {
        if (!pagesToScrape.isEmpty()) {
            // the current web page is about to be scraped and
            // should no longer be part of the scraping queue
            String url = pagesToScrape.remove(0);

            pagesDiscovered.add(url);

            // initializing the HTML Document page variable
            Document doc;

            try {
                // fetching the target website
                doc = Jsoup
                        .connect(url)
                        .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
                        .get();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }

            // retrieving the list of product HTML elements
            // in the target page
            Elements productElements = doc.select("li.product");

            // iterating over the list of HTML products
            for (Element productElement : productElements) {
                Product Product = new Product();

                // extracting the data of interest from the product HTML element
                // and storing it in Product
                Product.setUrl(productElement.selectFirst("a").attr("href"));
                Product.setImage(productElement.selectFirst("img").attr("src"));
                Product.setName(productElement.selectFirst("h2").text());
                Product.setPrice(productElement.selectFirst("span").text());

                // adding Product to the list of the scraped products
                products.add(Product);
            }

            // retrieving the list of pagination HTML element
            Elements paginationElements = doc.select("a.page-numbers");

            // iterating over the pagination HTML elements
            for (Element pageElement : paginationElements) {
                // the new link discovered
                String pageUrl = pageElement.attr("href");

                // if the web page discovered is new and should be scraped
                if (!pagesDiscovered.contains(pageUrl) && !pagesToScrape.contains(pageUrl)) {
                    pagesToScrape.add(pageUrl);
                }

                // adding the link just discovered
                // to the set of pages discovered so far
                pagesDiscovered.add(pageUrl);
            }

            // logging the end of the scraping operation
            System.out.println(url + " -> page scraped");
        }
    }

    public static void main(String[] args) throws InterruptedException {
        // initializing the list of Java object to store
        // the scraped data
        List<Product> products = Collections.synchronizedList(new ArrayList<>());

        // initializing the set of web page urls
        // discovered while crawling the target website
        Set<String> pagesDiscovered = Collections.synchronizedSet(new HashSet<>());

        // initializing the queue of urls to scrape
        List<String> pagesToScrape = Collections.synchronizedList(new ArrayList<>());
        // initializing the scraping queue with the
        // first pagination page
        pagesToScrape.add("https://www.scrapingcourse.com/ecommerce/page/1/");

        // initializing the ExecutorService to run the
        // web scraping process in parallel on 4 pages at a time
        ExecutorService executorService = Executors.newFixedThreadPool(4) ;

        // launching the web scraping process to discover some
        // urls and take advantage of the parallelization process
        scrapeProductPage(products, pagesDiscovered, pagesToScrape);

        // the number of iteration executed
        int i = 1;
        // to limit the number to scrape to 5
        int limit = 12;

        while (!pagesToScrape.isEmpty() && i < limit) {
            // registering the web scraping task
            executorService.execute(() -> scrapeProductPage(products, pagesDiscovered, pagesToScrape));

            // adding a 200ms delay for avoid overloading the server
            TimeUnit.MILLISECONDS.sleep(200);

            // incrementing the iteration number
            i++;
        }

        // waiting up to 300 seconds to all pending tasks to end
        executorService.shutdown();
        executorService.awaitTermination(300, TimeUnit.SECONDS);

        System.out.println(products.size());

        // writing the scraped data to a db or export it to a file...
    }
}

implementation "org.seleniumhq.selenium:selenium-java:4.14.1"

<dependency> 
	<groupId>org.seleniumhq.selenium</groupId> 
	<artifactId>selenium-java</artifactId> 
	<version>4.14.1</version> 
</dependency>

import org.openqa.selenium.*; 
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.chrome.ChromeDriver; 
import org.openqa.selenium.support.ui.WebDriverWait; 
import java.util.*; 
import com.zenrows.data.Product; 

 
public class Scraper { 
	public static void main(String[] args) {
		// defining the options to run Chrome in headless mode 
		ChromeOptions options = new ChromeOptions(); 
		options.addArguments("--headless"); 
 
		// initializing a Selenium WebDriver ChromeDriver instance 
		// to run Chrome in headless mode 
		WebDriver driver = new ChromeDriver(options); 
 
		// connecting to the target web page 
		driver.get("https://www.scrapingcourse.com/ecommerce/"); 
 
		// initializing the list of Java object to store 
		// the scraped data 
		List<Product> products = new ArrayList<>(); 
		 
		// retrieving the list of product HTML elements 
		List<WebElement>productElements = driver.findElements(By.cssSelector("li.product")); 
		 
		// iterating over the list of HTML products 
		for (WebElement productElement : productElements) { 
			Product product = new Product(); 
		 
			// extracting the data of interest from the product HTML element 
			// and storing it in product 
			product.setUrl(productElement.findElement(By.tagName("a")).getAttribute("href"));
            product.setImage(productElement.findElement(By.tagName("img")).getAttribute("src"));
            product.setName(productElement.findElement(By.tagName("h2")).getText());
            product.setPrice(productElement.findElement(By.tagName("span")).getText());

			// adding product to the list of the scraped products 
			products.add(product); 
		} 

		// ... 
		driver.quit();
	} 
}

WebElement paginationElement = driver.findElement(By.cssSelector("a.page-numbers")); 
// navigating to a new web page 
paginationElement.click(); 
 
// wait for the page to load... 
 
System.out.println(driver.getTitle()); // "Ecommerce Test Site to Learn Web Scraping – ScrapingCourse.com"