niko/ets-playwright
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add CLI argument parsing and fat jar build support

Browse Source 
- Add -s/-u/-p/-d/-h command line options
- Add maven-shade-plugin for fat jar packaging
- Fix proxyImport signature to accept proxyHost parameter

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Niko 2026-05-05 15:54:27 +08:00
parent 9ef9f8bce4
commit 01505285b0
3 changed files with 158 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
pom.xml
View File

@ -53,6 +53,36 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>3.5.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.6.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.ets.scraper.EtsScraper</mainClass>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

190
src/main/java/com/ets/scraper/EtsScraper.java
View File

@ -34,48 +34,97 @@ public class EtsScraper {
private static final String PROXY_HOST = "http://127.0.0.1:8081";
public static void main(String[] args) throws Exception {
// Parse CLI arguments
String proxyHost = null;
String proxyUser = null;
String proxyPass = null;
String dateStr = null;
for (int i = 0; i < args.length; i++) {
switch (args[i]) {
case "-h":
printHelp();
return;
case "-s":
proxyHost = args[++i];
break;
case "-u":
proxyUser = args[++i];
break;
case "-p":
proxyPass = args[++i];
break;
case "-d":
dateStr = args[++i];
break;
default:
System.err.println("[-] Unknown option: " + args[i]);
printHelp();
return;
}
}
if (proxyHost == null) {
proxyHost = "http://127.0.0.1:8081";
}
if (proxyUser == null) {
proxyUser = "admin";
}
if (proxyPass == null) {
proxyPass = "123456";
}
java.time.LocalDate targetDate;
if (dateStr != null) {
targetDate = java.time.LocalDate.parse(dateStr);
} else {
targetDate = java.time.LocalDate.now().minusDays(1);
}
String dateStrFormatted = targetDate.format(java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd"));
String dateStrFileName = targetDate.format(DateTimeFormatter.ofPattern("yyyyMMdd"));
try {
createDirectories(SCREENSHOT_DIR);
} catch (Exception e) {
} catch (Exception e) {
System.err.println("Failed to create directories: " + e.getMessage());
}
}
try (Playwright playwright = Playwright.create()) {
Browser browser = playwright.chromium().launch(
new BrowserType.LaunchOptions().setHeadless(false)
);
);
BrowserContext context = browser.newContext(
new Browser.NewContextOptions().setIgnoreHTTPSErrors(true)
);
);
Page page = context.newPage();
try {
// Navigate to frame.html first to establish session/cookies
// Navigate to frame.html first to establish session/cookies
System.out.println("[*] Establishing session via " + FRAME_URL);
page.navigate(FRAME_URL, new Page.NavigateOptions()
.setTimeout(30000)
.setWaitUntil(WaitUntilState.DOMCONTENTLOADED));
.setTimeout(30000)
.setWaitUntil(WaitUntilState.DOMCONTENTLOADED));
sleep(3000);
// Navigate directly to the login page
// Navigate directly to the login page
System.out.println("[*] Navigating to login page: " + LOGIN_URL);
page.navigate(LOGIN_URL, new Page.NavigateOptions()
.setTimeout(30000)
.setWaitUntil(WaitUntilState.NETWORKIDLE));
.setTimeout(30000)
.setWaitUntil(WaitUntilState.NETWORKIDLE));
sleep(2000);
// Close notification dialog FIRST (before filling credentials)
// Close notification dialog FIRST (before filling credentials)
closeNotificationDialog(page);
screenshot(page, "after_close_dialog");
// Download captcha image
// Download captcha image
downloadCaptcha(page);
// Close dialog again after page reload
// Close dialog again after page reload
closeNotificationDialog(page);
// Recognize captcha and perform login
// Recognize captcha and perform login
boolean loggedin = doLoginWithCaptcha(page);
if (loggedin) {
@ -87,7 +136,7 @@ public class EtsScraper {
System.out.println("[+] Page title: " + page.title());
System.out.println("[+] Page URL: " + page.url());
// 点击三联单菜单
// 点击三联单菜单
System.out.println("[*] Clicking 三联单 menu...");
page.locator("#module_2094F683-C542-4904-B33E-0D227C4DE199").first().click();
sleep(3000);
@ -95,101 +144,116 @@ public class EtsScraper {
screenshot(page, "after_sanliandan");
System.out.println("[+] 三联单 page title: " + page.title());
// 设置日期筛选：选择昨天的日期
java.time.LocalDate yesterday = java.time.LocalDate.now().minusDays(1);
String yesterdayStr = yesterday.format(java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd"));
System.out.println("[*] Setting date filter to yesterday: " + yesterdayStr);
// 设置日期筛选
System.out.println("[*] Setting date filter to: " + dateStrFormatted);
// 检查元素是否存在
// 检查元素是否存在
boolean startDateExists = page.locator("#Search_ThreeBillList_startWdate").count() > 0;
boolean endDateExists = page.locator("#Search_ThreeBillList_endWdate").count() > 0;
boolean queryBtnExists = page.locator("#Search_ThreeBillList_Button").count() > 0;
System.out.println("[*] Elements found - startDate: " + startDateExists + ", endDate: " + endDateExists + ", queryBtn: " + queryBtnExists);
// 直接设置日期值（WdatePicker 类型输入框）
// 直接设置日期值（WdatePicker 类型输入框）
if (startDateExists) {
System.out.println("[*] Setting start date to: " + yesterdayStr);
page.locator("#Search_ThreeBillList_startWdate").first().fill(yesterdayStr);
System.out.println("[*] Setting start date to: " + dateStrFormatted);
page.locator("#Search_ThreeBillList_startWdate").first().fill(dateStrFormatted);
sleep(500);
} else {
} else {
System.out.println("[!] Start date element not found");
}
}
// 设置结束日期（也用昨天的日期）
// 设置结束日期
if (endDateExists) {
System.out.println("[*] Setting end date to: " + yesterdayStr);
page.locator("#Search_ThreeBillList_endWdate").first().fill(yesterdayStr);
System.out.println("[*] Setting end date to: " + dateStrFormatted);
page.locator("#Search_ThreeBillList_endWdate").first().fill(dateStrFormatted);
sleep(500);
} else {
} else {
System.out.println("[!] End date element not found");
}
}
// 点击查询按钮，等待列表加载
// 点击查询按钮，等待列表加载
if (queryBtnExists) {
System.out.println("[*] Clicking query button...");
page.locator("#Search_ThreeBillList_Button").first().click();
// 等待列表内容出现
// 等待列表内容出现
try {
page.waitForSelector("tbody tr", new Page.WaitForSelectorOptions()
.setTimeout(30000));
.setTimeout(30000));
System.out.println("[+] Query completed, list loaded");
} catch (Exception e) {
} catch (Exception e) {
System.out.println("[!] Wait for list timeout, but query was submitted");
}
} else {
}
} else {
System.out.println("[!] Query button not found");
}
}
screenshot(page, "after_query");
// 点击导出按钮（先弹出导出选项对话框，再点击对话框内的导出）
// 点击导出按钮
if (page.locator("#Export_ThreeBillList_Button").count() > 0) {
System.out.println("[*] Clicking export button...");
// 设置下载目录
// 设置下载目录
Path downloadPath = Path.of("downloads").toAbsolutePath().normalize();
java.nio.file.Files.createDirectories(downloadPath);
// 点击主导出按钮打开对话框，再用 JS click 触发对话框内导出按钮
// 点击主导出按钮打开对话框，再用 JS click 触发对话框内导出按钮
Download dl = page.waitForDownload(
new Page.WaitForDownloadOptions().setTimeout(300000),
() -> {
() -> {
page.locator("#Export_ThreeBillList_Button").first().click();
sleep(2000);
System.out.println("[*] Triggering dialog export via JS...");
page.evaluate("document.querySelectorAll('button').forEach(b => { if (b.textContent.trim() === '导出') b.click(); })");
});
});
System.out.println("[*] Waiting for download to complete...");
String dateStr = yesterday.format(DateTimeFormatter.ofPattern("yyyyMMdd"));
Path savedFile = downloadPath.resolve("三联单列表_" + dateStr + ".xls");
Path savedFile = downloadPath.resolve("三联单列表_" + dateStrFileName + ".xls");
dl.saveAs(savedFile);
System.out.println("[+] Download saved to: " + savedFile);
if (java.nio.file.Files.size(savedFile) == 0) {
System.out.println("[-] Downloaded file is empty");
} else {
} else {
System.out.println("[+] Download size: " + java.nio.file.Files.size(savedFile) + " bytes");
}
// Auto-import to ets-proxy
autoImportBill(savedFile);
}
// Auto-import to ets-proxy
autoImportBill(savedFile, proxyHost, proxyUser, proxyPass);
}
}
screenshot(page, "after_export");
System.out.println("[+] Query and export completed!");
String content = page.textContent("body");
if (content != null) {
String preview = content.length() > 500
? content.substring(0, 500) + ".."
: content;
? content.substring(0, 500) + ".."
: content;
System.out.println("[+] Page content preview:\n" + preview);
}
} else {
}
} else {
System.out.println("[-] Login failed. Check screenshots/ for debugging.");
screenshot(page, "login_failed");
}
} finally {
}
} finally {
browser.close();
}
}
}
}
}
}
private static void printHelp() {
System.out.println("""
ETS 三联单爬虫 - 导出并导入三联单 Excel 数据
用法: java -jar ets-playwright.jar [选项]
选项:
-s <url> ets-proxy 服务器地址 (默认: http://127.0.0.1:8081)
-u <user> ets-proxy 用户名 (默认: admin)
-p <pass> ets-proxy 密码 (默认: 123456)
-d <date> 查询日期，格式 yyyy-MM-dd (默认: 昨天)
-h 显示此帮助信息
示例:
java -jar ets-playwright.jar -s https://api.ets.niko.red -u admin -p 123456 -d 2026-05-04
""");
}
public static boolean doLoginWithCaptcha(Page page) throws Exception {
// Recognize captcha first
@ -396,13 +460,13 @@ public class EtsScraper {
return s;
}
public static void autoImportBill(Path filePath) {
String token = proxyLogin(PROXY_HOST, USERNAME, PASSWORD);
public static void autoImportBill(Path filePath, String proxyHost, String username, String password) {
String token = proxyLogin(proxyHost, username, password);
if (token == null) {
System.out.println("[-] Proxy login failed, skipping import");
return;
}
proxyImport(filePath, token);
proxyImport(filePath, proxyHost, token);
}
public static String proxyLogin(String proxyHost, String username, String password) {
@ -440,12 +504,12 @@ public class EtsScraper {
}
}
public static void proxyImport(Path filePath, String token) {
public static void proxyImport(Path filePath, String proxyHost, String token) {
try {
java.io.File file = filePath.toFile();
String boundary = "----FormBoundary" + System.currentTimeMillis();
String boundaryLine = "--" + boundary;
java.net.URI uri = java.net.URI.create(PROXY_HOST + "/api/bill/import");
java.net.URI uri = java.net.URI.create(proxyHost + "/api/bill/import");
java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream();
java.io.OutputStream os = out;

2
src/test/java/com/ets/scraper/ImportTest.java
View File

@ -31,6 +31,6 @@ class ImportTest {
String token = EtsScraper.proxyLogin(PROXY_HOST, USERNAME, PASSWORD);
assertNotNull(token);
EtsScraper.proxyImport(testFile.toPath(), token);
EtsScraper.proxyImport(testFile.toPath(), PROXY_HOST, token);
}
}