diff --git a/pom.xml b/pom.xml index 2edb223..ca9943d 100644 --- a/pom.xml +++ b/pom.xml @@ -53,6 +53,36 @@ maven-surefire-plugin 3.5.2 + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + package + + shade + + + + + com.ets.scraper.EtsScraper + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + diff --git a/src/main/java/com/ets/scraper/EtsScraper.java b/src/main/java/com/ets/scraper/EtsScraper.java index 7d1fa7a..7a8ce1c 100644 --- a/src/main/java/com/ets/scraper/EtsScraper.java +++ b/src/main/java/com/ets/scraper/EtsScraper.java @@ -34,48 +34,97 @@ public class EtsScraper { private static final String PROXY_HOST = "http://127.0.0.1:8081"; public static void main(String[] args) throws Exception { + // Parse CLI arguments + String proxyHost = null; + String proxyUser = null; + String proxyPass = null; + String dateStr = null; + + for (int i = 0; i < args.length; i++) { + switch (args[i]) { + case "-h": + printHelp(); + return; + case "-s": + proxyHost = args[++i]; + break; + case "-u": + proxyUser = args[++i]; + break; + case "-p": + proxyPass = args[++i]; + break; + case "-d": + dateStr = args[++i]; + break; + default: + System.err.println("[-] Unknown option: " + args[i]); + printHelp(); + return; + } + } + + if (proxyHost == null) { + proxyHost = "http://127.0.0.1:8081"; + } + if (proxyUser == null) { + proxyUser = "admin"; + } + if (proxyPass == null) { + proxyPass = "123456"; + } + + java.time.LocalDate targetDate; + if (dateStr != null) { + targetDate = java.time.LocalDate.parse(dateStr); + } else { + targetDate = java.time.LocalDate.now().minusDays(1); + } + String dateStrFormatted = targetDate.format(java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd")); + String dateStrFileName = targetDate.format(DateTimeFormatter.ofPattern("yyyyMMdd")); + try { createDirectories(SCREENSHOT_DIR); - } catch (Exception e) { + } catch (Exception e) { System.err.println("Failed to create directories: " + e.getMessage()); - } + } try (Playwright playwright = Playwright.create()) { Browser browser = playwright.chromium().launch( new BrowserType.LaunchOptions().setHeadless(false) - ); + ); BrowserContext context = browser.newContext( new Browser.NewContextOptions().setIgnoreHTTPSErrors(true) - ); + ); Page page = context.newPage(); try { - // Navigate to frame.html first to establish session/cookies + // Navigate to frame.html first to establish session/cookies System.out.println("[*] Establishing session via " + FRAME_URL); page.navigate(FRAME_URL, new Page.NavigateOptions() - .setTimeout(30000) - .setWaitUntil(WaitUntilState.DOMCONTENTLOADED)); + .setTimeout(30000) + .setWaitUntil(WaitUntilState.DOMCONTENTLOADED)); sleep(3000); - // Navigate directly to the login page + // Navigate directly to the login page System.out.println("[*] Navigating to login page: " + LOGIN_URL); page.navigate(LOGIN_URL, new Page.NavigateOptions() - .setTimeout(30000) - .setWaitUntil(WaitUntilState.NETWORKIDLE)); + .setTimeout(30000) + .setWaitUntil(WaitUntilState.NETWORKIDLE)); sleep(2000); - // Close notification dialog FIRST (before filling credentials) + // Close notification dialog FIRST (before filling credentials) closeNotificationDialog(page); screenshot(page, "after_close_dialog"); - // Download captcha image + // Download captcha image downloadCaptcha(page); - // Close dialog again after page reload + // Close dialog again after page reload closeNotificationDialog(page); - // Recognize captcha and perform login + // Recognize captcha and perform login boolean loggedin = doLoginWithCaptcha(page); if (loggedin) { @@ -87,7 +136,7 @@ public class EtsScraper { System.out.println("[+] Page title: " + page.title()); System.out.println("[+] Page URL: " + page.url()); - // 点击三联单菜单 + // 点击三联单菜单 System.out.println("[*] Clicking 三联单 menu..."); page.locator("#module_2094F683-C542-4904-B33E-0D227C4DE199").first().click(); sleep(3000); @@ -95,101 +144,116 @@ public class EtsScraper { screenshot(page, "after_sanliandan"); System.out.println("[+] 三联单 page title: " + page.title()); - // 设置日期筛选：选择昨天的日期 - java.time.LocalDate yesterday = java.time.LocalDate.now().minusDays(1); - String yesterdayStr = yesterday.format(java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd")); - System.out.println("[*] Setting date filter to yesterday: " + yesterdayStr); + // 设置日期筛选 + System.out.println("[*] Setting date filter to: " + dateStrFormatted); - // 检查元素是否存在 + // 检查元素是否存在 boolean startDateExists = page.locator("#Search_ThreeBillList_startWdate").count() > 0; boolean endDateExists = page.locator("#Search_ThreeBillList_endWdate").count() > 0; boolean queryBtnExists = page.locator("#Search_ThreeBillList_Button").count() > 0; System.out.println("[*] Elements found - startDate: " + startDateExists + ", endDate: " + endDateExists + ", queryBtn: " + queryBtnExists); - // 直接设置日期值（WdatePicker 类型输入框） + // 直接设置日期值（WdatePicker 类型输入框） if (startDateExists) { - System.out.println("[*] Setting start date to: " + yesterdayStr); - page.locator("#Search_ThreeBillList_startWdate").first().fill(yesterdayStr); + System.out.println("[*] Setting start date to: " + dateStrFormatted); + page.locator("#Search_ThreeBillList_startWdate").first().fill(dateStrFormatted); sleep(500); - } else { + } else { System.out.println("[!] Start date element not found"); - } + } - // 设置结束日期（也用昨天的日期） + // 设置结束日期 if (endDateExists) { - System.out.println("[*] Setting end date to: " + yesterdayStr); - page.locator("#Search_ThreeBillList_endWdate").first().fill(yesterdayStr); + System.out.println("[*] Setting end date to: " + dateStrFormatted); + page.locator("#Search_ThreeBillList_endWdate").first().fill(dateStrFormatted); sleep(500); - } else { + } else { System.out.println("[!] End date element not found"); - } + } - // 点击查询按钮，等待列表加载 + // 点击查询按钮，等待列表加载 if (queryBtnExists) { System.out.println("[*] Clicking query button..."); page.locator("#Search_ThreeBillList_Button").first().click(); - // 等待列表内容出现 + // 等待列表内容出现 try { page.waitForSelector("tbody tr", new Page.WaitForSelectorOptions() - .setTimeout(30000)); + .setTimeout(30000)); System.out.println("[+] Query completed, list loaded"); - } catch (Exception e) { + } catch (Exception e) { System.out.println("[!] Wait for list timeout, but query was submitted"); - } - } else { + } + } else { System.out.println("[!] Query button not found"); - } + } screenshot(page, "after_query"); - // 点击导出按钮（先弹出导出选项对话框，再点击对话框内的导出） + // 点击导出按钮 if (page.locator("#Export_ThreeBillList_Button").count() > 0) { System.out.println("[*] Clicking export button..."); - // 设置下载目录 + // 设置下载目录 Path downloadPath = Path.of("downloads").toAbsolutePath().normalize(); java.nio.file.Files.createDirectories(downloadPath); - // 点击主导出按钮打开对话框，再用 JS click 触发对话框内导出按钮 + // 点击主导出按钮打开对话框，再用 JS click 触发对话框内导出按钮 Download dl = page.waitForDownload( new Page.WaitForDownloadOptions().setTimeout(300000), - () -> { + () -> { page.locator("#Export_ThreeBillList_Button").first().click(); sleep(2000); System.out.println("[*] Triggering dialog export via JS..."); page.evaluate("document.querySelectorAll('button').forEach(b => { if (b.textContent.trim() === '导出') b.click(); })"); - }); + }); System.out.println("[*] Waiting for download to complete..."); - String dateStr = yesterday.format(DateTimeFormatter.ofPattern("yyyyMMdd")); - Path savedFile = downloadPath.resolve("三联单列表_" + dateStr + ".xls"); + Path savedFile = downloadPath.resolve("三联单列表_" + dateStrFileName + ".xls"); dl.saveAs(savedFile); System.out.println("[+] Download saved to: " + savedFile); if (java.nio.file.Files.size(savedFile) == 0) { System.out.println("[-] Downloaded file is empty"); - } else { + } else { System.out.println("[+] Download size: " + java.nio.file.Files.size(savedFile) + " bytes"); - } - // Auto-import to ets-proxy - autoImportBill(savedFile); - } + // Auto-import to ets-proxy + autoImportBill(savedFile, proxyHost, proxyUser, proxyPass); + } + } screenshot(page, "after_export"); System.out.println("[+] Query and export completed!"); String content = page.textContent("body"); if (content != null) { String preview = content.length() > 500 - ? content.substring(0, 500) + ".." - : content; + ? content.substring(0, 500) + ".." + : content; System.out.println("[+] Page content preview:\n" + preview); - } - } else { + } + } else { System.out.println("[-] Login failed. Check screenshots/ for debugging."); screenshot(page, "login_failed"); - } - } finally { + } + } finally { browser.close(); - } - } - } + } + } + } + + private static void printHelp() { + System.out.println(""" + ETS 三联单爬虫 - 导出并导入三联单 Excel 数据 + + 用法: java -jar ets-playwright.jar [选项] + + 选项: + -s ets-proxy 服务器地址 (默认: http://127.0.0.1:8081) + -u ets-proxy 用户名 (默认: admin) + -p ets-proxy 密码 (默认: 123456) + -d 查询日期，格式 yyyy-MM-dd (默认: 昨天) + -h 显示此帮助信息 + + 示例: + java -jar ets-playwright.jar -s https://api.ets.niko.red -u admin -p 123456 -d 2026-05-04 + """); + } public static boolean doLoginWithCaptcha(Page page) throws Exception { // Recognize captcha first @@ -396,13 +460,13 @@ public class EtsScraper { return s; } - public static void autoImportBill(Path filePath) { - String token = proxyLogin(PROXY_HOST, USERNAME, PASSWORD); + public static void autoImportBill(Path filePath, String proxyHost, String username, String password) { + String token = proxyLogin(proxyHost, username, password); if (token == null) { System.out.println("[-] Proxy login failed, skipping import"); return; } - proxyImport(filePath, token); + proxyImport(filePath, proxyHost, token); } public static String proxyLogin(String proxyHost, String username, String password) { @@ -440,12 +504,12 @@ public class EtsScraper { } } - public static void proxyImport(Path filePath, String token) { + public static void proxyImport(Path filePath, String proxyHost, String token) { try { java.io.File file = filePath.toFile(); String boundary = "----FormBoundary" + System.currentTimeMillis(); String boundaryLine = "--" + boundary; - java.net.URI uri = java.net.URI.create(PROXY_HOST + "/api/bill/import"); + java.net.URI uri = java.net.URI.create(proxyHost + "/api/bill/import"); java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); java.io.OutputStream os = out; diff --git a/src/test/java/com/ets/scraper/ImportTest.java b/src/test/java/com/ets/scraper/ImportTest.java index b91e821..acbb650 100644 --- a/src/test/java/com/ets/scraper/ImportTest.java +++ b/src/test/java/com/ets/scraper/ImportTest.java @@ -31,6 +31,6 @@ class ImportTest { String token = EtsScraper.proxyLogin(PROXY_HOST, USERNAME, PASSWORD); assertNotNull(token); - EtsScraper.proxyImport(testFile.toPath(), token); + EtsScraper.proxyImport(testFile.toPath(), PROXY_HOST, token); } }