diff --git a/kibu.db b/kibu.db deleted file mode 100644 index e69de29..0000000 diff --git a/kibu.sqlite b/kibu.sqlite index 85b561c..fa6081d 100644 Binary files a/kibu.sqlite and b/kibu.sqlite differ diff --git a/pom.xml b/pom.xml index d3d7e42..293ed65 100644 --- a/pom.xml +++ b/pom.xml @@ -69,6 +69,25 @@ 1.18.38 provided + + + io.github.bonigarcia + webdrivermanager + 5.8.0 + + + + org.seleniumhq.selenium + selenium-java + 4.15.0 + + + + com.google.guava + guava + 31.1-jre + + diff --git a/src/main/java/de/roko/archiv/kibubackend/KibuBackendApplication.java b/src/main/java/de/roko/archiv/kibubackend/KibuBackendApplication.java index 7cd33ba..2949b7e 100644 --- a/src/main/java/de/roko/archiv/kibubackend/KibuBackendApplication.java +++ b/src/main/java/de/roko/archiv/kibubackend/KibuBackendApplication.java @@ -3,7 +3,7 @@ package de.roko.archiv.kibubackend; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -@SpringBootApplication +@SpringBootApplication(scanBasePackages = "de.roko.archiv.kibubackend") public class KibuBackendApplication { public static void main(String[] args) { diff --git a/src/main/java/de/roko/archiv/kibubackend/archion/ArchionLoginHelper.java b/src/main/java/de/roko/archiv/kibubackend/archion/ArchionLoginHelper.java new file mode 100644 index 0000000..6789b98 --- /dev/null +++ b/src/main/java/de/roko/archiv/kibubackend/archion/ArchionLoginHelper.java @@ -0,0 +1,87 @@ +package de.roko.archiv.kibubackend.archion; + +import org.openqa.selenium.By; +import org.openqa.selenium.TimeoutException; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.support.ui.ExpectedConditions; +import org.openqa.selenium.support.ui.WebDriverWait; +import org.springframework.beans.factory.annotation.Value; + +import java.time.Duration; + +public class ArchionLoginHelper { + + + private final WebDriver driver; + private final WebDriverWait wait; + private boolean loggedIn; + private final String username; + private final String password; + private final String archionUrl; + + public ArchionLoginHelper(WebDriver driver, String username, String password, String archionUrl) { + this.driver = driver; + this.wait = new WebDriverWait(driver, Duration.ofSeconds(10)); + this.archionUrl = archionUrl; + this.username = username; + this.password = password; + } + + public void login() { + + // Seite aufrufen + driver.get(archionUrl); + + + wait.until(ExpectedConditions.presenceOfElementLocated(By.cssSelector("a[title='Anmelden']"))); + // "Anmelden"-Link per Attribut (title="Anmelden") finden + WebElement loginLink = wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector("a[title='Anmelden']"))); + loginLink.click(); + + // Eingabefelder finden und ausfüllen + WebElement emailField = wait.until(ExpectedConditions.visibilityOfElementLocated(By.name("user"))); + WebElement passwordField = wait.until(ExpectedConditions.visibilityOfElementLocated(By.name("pass"))); + + emailField.sendKeys(username); + passwordField.sendKeys(password); + + // Login-Button klicken + WebElement submitButton = wait.until( + ExpectedConditions.elementToBeClickable(By.name("submit")) + ); + // Klick auf den Button "Anmelden" + submitButton.click(); + + System.out.println("Login abgeschlossen. Aktuelle URL: " + driver.getCurrentUrl()); + + } + + public boolean isLoggedIn() { + try { + // Eventuell Dropdown öffnen, wenn nötig: + WebElement kontoDropdownToggle = wait.until(ExpectedConditions.elementToBeClickable( + By.cssSelector("a.nav-link.dropdown-toggle[href='#'], a.nav-link.dropdown-toggle.show"))); + kontoDropdownToggle.click(); + + // Warte auf Eintrag "Konto-Übersicht" + WebElement kontoUebersicht = wait.until(ExpectedConditions.visibilityOfElementLocated( + By.cssSelector("a[href*='/konto-uebersicht']"))); + + System.out.println("✅ Menüeintrag 'Konto-Übersicht' gefunden."); + loggedIn = kontoUebersicht.isDisplayed(); + return loggedIn; + } catch (TimeoutException e) { + System.out.println("❌ Menüeintrag 'Konto-Übersicht' nicht sichtbar."); + loggedIn = false; + return false; + } + } + + public void openAlleArchive() { + driver.get("https://www.archion.de/de/alle-archive"); + wait.until(ExpectedConditions.urlContains("/de/alle-archive")); + System.out.println("✅ Seite 'Alle Archive' geöffnet."); + } + +} diff --git a/src/main/java/de/roko/archiv/kibubackend/config/WebConfig.java b/src/main/java/de/roko/archiv/kibubackend/config/WebConfig.java new file mode 100644 index 0000000..5955f27 --- /dev/null +++ b/src/main/java/de/roko/archiv/kibubackend/config/WebConfig.java @@ -0,0 +1,22 @@ +package de.roko.archiv.kibubackend.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.CorsRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +@Configuration +public class WebConfig { + + @Bean + public WebMvcConfigurer corsConfigurer() { + return new WebMvcConfigurer() { + @Override + public void addCorsMappings(CorsRegistry registry) { + registry.addMapping("/**") + .allowedOrigins("http://localhost:4200") + .allowedMethods("*"); + } + }; + } +} diff --git a/src/main/java/de/roko/archiv/kibubackend/controller/BundeslandController.java b/src/main/java/de/roko/archiv/kibubackend/controller/BundeslandController.java new file mode 100644 index 0000000..0190426 --- /dev/null +++ b/src/main/java/de/roko/archiv/kibubackend/controller/BundeslandController.java @@ -0,0 +1,67 @@ +package de.roko.archiv.kibubackend.controller; + + +import de.roko.archiv.kibubackend.model.Archiv; +import de.roko.archiv.kibubackend.model.Bundesland; +import de.roko.archiv.kibubackend.repository.BundeslandRepository; +import de.roko.archiv.kibubackend.service.BundeslandService; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +import java.util.List; + +@RestController +@RequestMapping("api/bundesland") +@CrossOrigin(origins = "*") +public class BundeslandController { + + private final BundeslandRepository bundeslandRepository; + private final BundeslandService bundeslandService; + + public BundeslandController(BundeslandRepository bundeslandRepository, BundeslandService bundeslandService) { + this.bundeslandRepository = bundeslandRepository; + this.bundeslandService = bundeslandService; + } + + @PostMapping + public Bundesland create(@RequestBody Bundesland bundesland) { + return bundeslandRepository.save(bundesland); + } + + @GetMapping + public List getAll() { + return bundeslandRepository.findAll(); + } + + @GetMapping("/{id}") + public ResponseEntity getById(@PathVariable Long id) { + return bundeslandService.findById(id) + .map(ResponseEntity::ok) + .orElse(ResponseEntity.notFound().build()); + } + + @PutMapping("/{id}") + public ResponseEntity update(@PathVariable Long id, @RequestBody Bundesland updatedBundesland) { + updatedBundesland.setId(id); + return ResponseEntity.ok(bundeslandService.save(updatedBundesland)); + } + + @DeleteMapping("/{id}") + public ResponseEntity delete(@PathVariable Long id) { + if(bundeslandRepository.existsById(id)) { + bundeslandRepository.deleteById(id); + return ResponseEntity.noContent().build(); + }else { + return ResponseEntity.notFound().build(); + } + } + + // GET /api/bundesland/{id}/archive + @GetMapping("/{id}/archive") + public ResponseEntity> getArchiveForBundesland(@PathVariable Long id) { + return bundeslandRepository.findById(id) + .map(bundesland -> ResponseEntity.ok(bundesland.getArchive())) + .orElse(ResponseEntity.notFound().build()); + } + +} diff --git a/src/main/java/de/roko/archiv/kibubackend/controller/archion/BundeslandScraperController.java b/src/main/java/de/roko/archiv/kibubackend/controller/archion/BundeslandScraperController.java new file mode 100644 index 0000000..3f269c9 --- /dev/null +++ b/src/main/java/de/roko/archiv/kibubackend/controller/archion/BundeslandScraperController.java @@ -0,0 +1,25 @@ +package de.roko.archiv.kibubackend.controller.archion; + +import de.roko.archiv.kibubackend.model.Bundesland; +import de.roko.archiv.kibubackend.service.archion.BundeslandScraperService; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@RequestMapping("/api/bundesland") +@RequiredArgsConstructor +public class BundeslandScraperController { + + private final BundeslandScraperService bundeslandScraperService; + + @PostMapping("/scrap") + public ResponseEntity> scrapeBundeslaender() { + List result = bundeslandScraperService.scrapeBundeslaender(); + return ResponseEntity.ok(result); + } +} diff --git a/src/main/java/de/roko/archiv/kibubackend/model/Archiv.java b/src/main/java/de/roko/archiv/kibubackend/model/Archiv.java index 413ad6a..d3dd180 100644 --- a/src/main/java/de/roko/archiv/kibubackend/model/Archiv.java +++ b/src/main/java/de/roko/archiv/kibubackend/model/Archiv.java @@ -1,5 +1,6 @@ package de.roko.archiv.kibubackend.model; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import jakarta.persistence.*; import lombok.*; @@ -24,4 +25,9 @@ public class Archiv { @OneToMany(mappedBy = "archiv", cascade = CascadeType.ALL, orphanRemoval = true) private List kreise; + @ManyToOne + @JoinColumn(name = "bundesland_id", nullable = false) + @JsonIgnoreProperties("archive") + private Bundesland bundesland; + } diff --git a/src/main/java/de/roko/archiv/kibubackend/model/Bundesland.java b/src/main/java/de/roko/archiv/kibubackend/model/Bundesland.java new file mode 100644 index 0000000..f2f06a4 --- /dev/null +++ b/src/main/java/de/roko/archiv/kibubackend/model/Bundesland.java @@ -0,0 +1,28 @@ +package de.roko.archiv.kibubackend.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import jakarta.persistence.*; +import lombok.*; + +import java.util.List; + +@Builder +@Getter +@Setter +@Entity +@NoArgsConstructor +@AllArgsConstructor +public class Bundesland { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + private String name; + private String link; + + @OneToMany(mappedBy = "bundesland", cascade = CascadeType.ALL) + @JsonIgnoreProperties("bundesland") + private List archive; + +} diff --git a/src/main/java/de/roko/archiv/kibubackend/repository/ArchivRepository.java b/src/main/java/de/roko/archiv/kibubackend/repository/ArchivRepository.java index dbb8efe..42e4c31 100644 --- a/src/main/java/de/roko/archiv/kibubackend/repository/ArchivRepository.java +++ b/src/main/java/de/roko/archiv/kibubackend/repository/ArchivRepository.java @@ -2,6 +2,8 @@ package de.roko.archiv.kibubackend.repository; import org.springframework.data.jpa.repository.JpaRepository; import de.roko.archiv.kibubackend.model.Archiv; +import org.springframework.stereotype.Repository; +@Repository public interface ArchivRepository extends JpaRepository { } diff --git a/src/main/java/de/roko/archiv/kibubackend/repository/BundeslandRepository.java b/src/main/java/de/roko/archiv/kibubackend/repository/BundeslandRepository.java new file mode 100644 index 0000000..d3c8987 --- /dev/null +++ b/src/main/java/de/roko/archiv/kibubackend/repository/BundeslandRepository.java @@ -0,0 +1,9 @@ +package de.roko.archiv.kibubackend.repository; + +import de.roko.archiv.kibubackend.model.Bundesland; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface BundeslandRepository extends JpaRepository { +} diff --git a/src/main/java/de/roko/archiv/kibubackend/service/BundeslandService.java b/src/main/java/de/roko/archiv/kibubackend/service/BundeslandService.java new file mode 100644 index 0000000..3f23c93 --- /dev/null +++ b/src/main/java/de/roko/archiv/kibubackend/service/BundeslandService.java @@ -0,0 +1,30 @@ +package de.roko.archiv.kibubackend.service; + +import de.roko.archiv.kibubackend.model.Bundesland; +import de.roko.archiv.kibubackend.repository.BundeslandRepository; +import org.springframework.stereotype.Service; + +import java.util.Optional; + +@Service +public class BundeslandService { + + private final BundeslandRepository bundeslandRepository; + + public BundeslandService(BundeslandRepository bundeslandRepository) { + this.bundeslandRepository = bundeslandRepository; + } + + public Bundesland save(Bundesland bundesland) { + return bundeslandRepository.save(bundesland); + } + + + public Optional findById(Long id) { + return bundeslandRepository.findById(id); + } + + public void deleteById(Long id) { + bundeslandRepository.deleteById(id);} + +} diff --git a/src/main/java/de/roko/archiv/kibubackend/service/archion/BundeslandScraperService.java b/src/main/java/de/roko/archiv/kibubackend/service/archion/BundeslandScraperService.java new file mode 100644 index 0000000..dcb2509 --- /dev/null +++ b/src/main/java/de/roko/archiv/kibubackend/service/archion/BundeslandScraperService.java @@ -0,0 +1,109 @@ +package de.roko.archiv.kibubackend.service.archion; + +import de.roko.archiv.kibubackend.archion.ArchionLoginHelper; +import de.roko.archiv.kibubackend.model.Archiv; +import de.roko.archiv.kibubackend.model.Bundesland; +import de.roko.archiv.kibubackend.repository.ArchivRepository; +import de.roko.archiv.kibubackend.repository.BundeslandRepository; +import io.github.bonigarcia.wdm.WebDriverManager; +import jakarta.transaction.Transactional; +import lombok.RequiredArgsConstructor; +import org.openqa.selenium.*; +import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeOptions; +import org.openqa.selenium.support.ui.ExpectedConditions; +import org.openqa.selenium.support.ui.WebDriverWait; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + + +@Service +@RequiredArgsConstructor +public class BundeslandScraperService { + + private final BundeslandRepository bundeslandRepository; + private final ArchivRepository archivRepository; + + @Value("${archion.user}") + String user; + + @Value("${archion.pass}") + String pass; + + @Value("${archion.url}") + String url; + @Transactional + public List scrapeBundeslaender() { + WebDriverManager.chromedriver().setup(); + ChromeOptions options = new ChromeOptions(); + options.addArguments("--remote-allow-origins=*"); + options.addArguments("--window-size=1920,1080"); + // options.addArguments("--headless=new"); // für Headless-Modus + + WebDriver driver = new ChromeDriver(options); + + + try { + ArchionLoginHelper loginHelper = new ArchionLoginHelper(driver, user, pass, url); + loginHelper.login(); + if (!loginHelper.isLoggedIn()) { + throw new IllegalStateException("Login fehlgeschlagen."); + } + loginHelper.openAlleArchive(); + + // Bundesländer und Archive scrapen + List scraped = scrapeFromPage(driver); + + // in DB speichern (neu, vorheriges Löschen optional) + for (Bundesland b : scraped) { + Bundesland saved = bundeslandRepository.save(b); + for (Archiv a : b.getArchive()) { + a.setBundesland(saved); + archivRepository.save(a); + } + } + + return scraped; + + } finally { + driver.quit(); + } + } + + private List scrapeFromPage(WebDriver driver) { + List bundeslaender = new ArrayList<>(); + + WebElement container = new WebDriverWait(driver, Duration.ofSeconds(10)) + .until(ExpectedConditions.visibilityOfElementLocated(By.id("federal-nav"))); + + List divs = container.findElements(By.cssSelector("div.mb-4")); + + for (WebElement div : divs) { + WebElement blLink = div.findElement(By.cssSelector("a.h6.text-muted")); + String blName = blLink.getText().trim(); + String blHref = "https://www.archion.de" + blLink.getAttribute("href"); + + Bundesland bl = new Bundesland(); + bl.setName(blName); + bl.setLink(blHref); + + List archive = new ArrayList<>(); + List archLinks = div.findElements(By.cssSelector("ul > li > a")); + for (WebElement a : archLinks) { + Archiv ar = new Archiv(); + ar.setName(a.getText().trim()); + ar.setLink("https://www.archion.de" + a.getAttribute("href")); + archive.add(ar); + } + + bl.setArchive(archive); + bundeslaender.add(bl); + } + + return bundeslaender; + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 0855c63..36fd1d2 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -3,3 +3,7 @@ spring.datasource.url=jdbc:sqlite:file:./kibu.sqlite spring.datasource.driver-class-name=org.sqlite.JDBC spring.jpa.database-platform=org.hibernate.community.dialect.SQLiteDialect spring.jpa.hibernate.ddl-auto=update +archion.user=${ARCHION_USER} +archion.pass=${ARCHION_PASS} +archion.url=${ARCHION_URL} + diff --git a/src/test/java/de/roko/archiv/kibubackend/archion/ArchionLoginHelperTest.java b/src/test/java/de/roko/archiv/kibubackend/archion/ArchionLoginHelperTest.java new file mode 100644 index 0000000..6ed394f --- /dev/null +++ b/src/test/java/de/roko/archiv/kibubackend/archion/ArchionLoginHelperTest.java @@ -0,0 +1,47 @@ +package de.roko.archiv.kibubackend.archion; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeOptions; +import org.springframework.beans.factory.annotation.Value; + +import static org.junit.jupiter.api.Assertions.*; + +class ArchionLoginHelperTest { + + WebDriver driver; + ChromeOptions options; + + @BeforeEach + void setUp() { + options = new ChromeOptions(); + // Sichtbar = KEIN Headless-Modus + // options.addArguments("--headless=new"); // AUSKOMMENTIEREN, wenn sichtbar + options.addArguments("--window-size=1920,1080"); + options.addArguments("--remote-allow-origins=*"); // optional, siehe Hinweis + + } + + + @Test + void login() { + + String user = "robatkoch"; + String password = "PaLiNa2016$$"; + String archioUrl = "https://www.archion.de/de/"; + + driver = new ChromeDriver(options); + ArchionLoginHelper archionLoginHelper = new ArchionLoginHelper(driver,user,password,archioUrl); + archionLoginHelper.login(); + System.out.println(archionLoginHelper.isLoggedIn()); + } + + @AfterEach + void tearDown() { + driver.quit(); + } + +} \ No newline at end of file diff --git a/src/test/java/de/roko/archiv/kibubackend/service/archion/BundeslandScraperServiceTest.java b/src/test/java/de/roko/archiv/kibubackend/service/archion/BundeslandScraperServiceTest.java new file mode 100644 index 0000000..20e57fd --- /dev/null +++ b/src/test/java/de/roko/archiv/kibubackend/service/archion/BundeslandScraperServiceTest.java @@ -0,0 +1,14 @@ +package de.roko.archiv.kibubackend.service.archion; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class BundeslandScraperServiceTest { + + @Test + void scrapeBundeslaender() { + BundeslandScraperService bundeslandScraperService = new BundeslandScraperService(); + bundeslandScraperService.scrapeBundeslaender(); + } +} \ No newline at end of file