From 1b7ebb43256f6d547d29e91f91e1b32cdc2f9dd8 Mon Sep 17 00:00:00 2001 From: Niko <1377382065@qq.com> Date: Tue, 5 May 2026 22:05:32 +0800 Subject: [PATCH] fix download excel --- .DS_Store | Bin 8196 -> 6148 bytes src/main/java/com/ets/scraper/EtsScraper.java | 52 ++++------ .../java/com/ets/scraper/EtsScraperTest.java | 90 ------------------ src/test/java/com/ets/scraper/ImportTest.java | 36 ------- 4 files changed, 20 insertions(+), 158 deletions(-) delete mode 100644 src/test/java/com/ets/scraper/EtsScraperTest.java delete mode 100644 src/test/java/com/ets/scraper/ImportTest.java diff --git a/.DS_Store b/.DS_Store index 83fc51f87afcb007cf259c74b8d4ce0819fd2423..f44927b5b0eedd7d347be9ec2fa682dd173eee5f 100644 GIT binary patch delta 138 zcmZp1XfcprU|?W$DortDU=RQ@Ie-{MGjdEU6q~50D9Q@rfW;~pav5?wbMljuawZlo zXJnnMCH!||!#~!=>>M0|%s^#8AixbITtV757Jg@*%r9f901^fozyzgPKs1m8vIt~1 K!{&INIm`e#7!=t6 literal 8196 zcmeHMPfrs;6n_HJ!Ky1QpQP9S7fZ$jPZG~ltf{J6x(*sddqK+6u z($R16a70^SS)!x^k#ry`Gf^iLBC;dT65&8>iMrJaXa&wHz|Zb6Jh3w<0X={!^7W5Lu1pW<`$#HXf@VTTdb6doM9Qx zT4c=6RBR`+3-%w8rk-J3NdT-HF=|7Sh{(fwEU(@<7xx1lr{Far?Ud(!A`hS{nIM zS=x8gtSl~ex%Eoh)poO^v#TTC(cRtMbFH_p^ZJdn*_Cm!+xdV`dW!ihNcuZ&+H4PT z>9biz=L2^am%JRgdr``JiLTBX!h)N7cG9)DU&NUBNrxEpQQ*3CH6M7nxy{Ycjh0W@ zD?D2^V7@=kLND?dU0D;^BeyV1HyLK+GngxKSGeQ`A>tX%@`@7#4$ZsC{dM2Txc(eX zvM6GOWe-=uXD(8vicZ#}`NYGYQqWk!z9lUa5*3Y&c*Rmk5}!O<*o%}=7fJKdldvDF z5CaBa5>l`RFQ5!>;T^n(&+rv~z%Mu=ZKQ|vlL0bF?vin0lS#5bR>=l&$aBeWsC?Zw z)p!a>h(?{i{oPbReMa-><;)f+;@wezA=ty#pY^Nmj1L%5eP=Ad49<2Mo*)h(BUjBJ z{WXkM2q6mrxcH%HZ(+8Gzd<{lhrFp_HDY7{+3KhUzA2ubO*4+>7LI&Mv1^%i6 FzX4C+9qs@C diff --git a/src/main/java/com/ets/scraper/EtsScraper.java b/src/main/java/com/ets/scraper/EtsScraper.java index 47c7902..023a9fe 100644 --- a/src/main/java/com/ets/scraper/EtsScraper.java +++ b/src/main/java/com/ets/scraper/EtsScraper.java @@ -99,8 +99,10 @@ public class EtsScraper { try (Playwright playwright = Playwright.create()) { Browser browser = playwright.chromium().launch( - new BrowserType.LaunchOptions().setHeadless(true) + new BrowserType.LaunchOptions() + .setHeadless(false) ); + BrowserContext context = browser.newContext( new Browser.NewContextOptions() .setIgnoreHTTPSErrors(true) @@ -127,7 +129,7 @@ public class EtsScraper { // Close notification dialog FIRST (before filling credentials) closeNotificationDialog(page); - screenshot(page, "after_close_dialog"); +// screenshot(page, "after_close_dialog"); // Download captcha image downloadCaptcha(page); @@ -142,7 +144,7 @@ public class EtsScraper { System.out.println("[+] Login successful!"); sleep(2000); - screenshot(page, "after_login"); +// screenshot(page, "after_login"); System.out.println("[+] Page title: " + page.title()); System.out.println("[+] Page URL: " + page.url()); @@ -152,7 +154,7 @@ public class EtsScraper { page.locator("#module_2094F683-C542-4904-B33E-0D227C4DE199").first().click(); sleep(3000); - screenshot(page, "after_sanliandan"); +// screenshot(page, "after_sanliandan"); System.out.println("[+] 三联单 page title: " + page.title()); // 设置日期筛选 @@ -185,21 +187,16 @@ public class EtsScraper { // 点击查询按钮，等待列表加载 if (queryBtnExists) { System.out.println("[*] Clicking query button..."); - page.locator("#Search_ThreeBillList_Button").first().click(); - // 等待列表内容出现 - try { - page.waitForSelector("tbody tr", new Page.WaitForSelectorOptions() - .setTimeout(30000)); - System.out.println("[+] Query completed, list loaded"); - } catch (Exception e) { - System.out.println("[!] Wait for list timeout, but query was submitted"); - } + page.waitForResponse("https://101.227.180.215/SHCityEnvCW/Services/CWSServ.asmx/ThreeBillQueryBiTripList", () -> { + page.locator("#Search_ThreeBillList_Button").first().click(); + }); + page.waitForTimeout(3 * 1000); } else { System.out.println("[!] Query button not found"); } - screenshot(page, "after_query"); +// screenshot(page, "after_query"); // 点击导出按钮 if (page.locator("#Export_ThreeBillList_Button").count() > 0) { @@ -210,34 +207,25 @@ public class EtsScraper { () -> { page.locator("#Export_ThreeBillList_Button").first().click(); sleep(2000); - System.out.println("[*] Triggering dialog export via JS..."); - page.evaluate("document.querySelectorAll('button').forEach(b => { if (b.textContent.trim() === '导出') b.click(); })"); }); + System.out.println("[*] Waiting for download to complete..."); - long[] totalBytes = {0}; - try (java.io.InputStream stream = dl.createReadStream(); - java.io.OutputStream out = Files.newOutputStream(savedFile)) { - byte[] buf = new byte[8192]; - int n; - while ((n = stream.read(buf)) > 0) { - out.write(buf, 0, n); - totalBytes[0] += n; - } - } - System.out.println("[+] Download saved to: " + savedFile + " (" + totalBytes[0] + " bytes)"); - if (totalBytes[0] == 0) { + dl.saveAs(savedFile); + + long totalBytes = savedFile.toFile().length(); + System.out.println("[+] Download saved to: " + savedFile + " (" + totalBytes + " bytes)"); + if (totalBytes == 0) { System.out.println("[-] Downloaded file is empty"); } else { - System.out.println("[+] Download size: " + totalBytes[0] + " bytes"); - // Auto-import to ets-proxy + System.out.println("[+] Download size: " + totalBytes + " bytes"); autoImportBill(savedFile, proxyHost, proxyUser, proxyPass); } } - screenshot(page, "after_export"); +// screenshot(page, "after_export"); System.out.println("[+] Query and export completed!"); } else { System.out.println("[-] Login failed. Check screenshots/ for debugging."); - screenshot(page, "login_failed"); +// screenshot(page, "login_failed"); } } finally { browser.close(); diff --git a/src/test/java/com/ets/scraper/EtsScraperTest.java b/src/test/java/com/ets/scraper/EtsScraperTest.java deleted file mode 100644 index 7336e55..0000000 --- a/src/test/java/com/ets/scraper/EtsScraperTest.java +++ /dev/null @@ -1,90 +0,0 @@ -package com.ets.scraper; - -import org.junit.jupiter.api.*; -import org.junit.jupiter.api.io.TempDir; - -import javax.imageio.ImageIO; -import java.awt.image.BufferedImage; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.*; - -class EtsScraperTest { - - @TempDir - Path tempDir; - - @Test - void testGifToPngConversion() throws Exception { - Path gifPath = tempDir.resolve("captcha.png"); - // Copy the actual captcha (GIF stored as .png) to temp - Files.copy( - Path.of("screenshots/captcha.png").toAbsolutePath(), - gifPath - ); - - byte[] imageBytes = Files.readAllBytes(gifPath); - ByteArrayInputStream bais = new ByteArrayInputStream(imageBytes); - BufferedImage gifImage = ImageIO.read(bais); - - assertNotNull(gifImage, "GIF should be readable by ImageIO"); - assertTrue(gifImage.getWidth() > 0, "Image should have positive width"); - - ByteArrayOutputStream pngOut = new ByteArrayOutputStream(); - ImageIO.write(gifImage, "png", pngOut); - byte[] pngBytes = pngOut.toByteArray(); - - assertTrue(pngBytes.length > 0, "PNG output should not be empty"); - - // Verify converted PNG is valid - ByteArrayInputStream bais2 = new ByteArrayInputStream(pngBytes); - BufferedImage pngImage = ImageIO.read(bais2); - assertNotNull(pngImage, "Converted PNG should be readable"); - } - - @Test - void testGifToPngProducesValidPng() throws Exception { - byte[] gifBytes = Files.readAllBytes(Path.of("screenshots/captcha.png").toAbsolutePath()); - ByteArrayInputStream bais = new ByteArrayInputStream(gifBytes); - BufferedImage image = ImageIO.read(bais); - - ByteArrayOutputStream pngOut = new ByteArrayOutputStream(); - ImageIO.write(image, "png", pngOut); - - // PNG header: 89 50 4E 47 0D 0A 1A 0A - byte[] pngHeader = pngOut.toByteArray(); - // bytes are signed in Java, mask with & 0xFF - assertEquals(0x89 & 0xFF, pngHeader[0] & 0xFF, "PNG magic number"); - assertEquals(0x50 & 0xFF, pngHeader[1] & 0xFF, "P"); - assertEquals(0x4E & 0xFF, pngHeader[2] & 0xFF, "N"); - assertEquals(0x47 & 0xFF, pngHeader[3] & 0xFF, "G"); - } - - @Test - void testBase64Encoding() throws Exception { - byte[] imageBytes = Files.readAllBytes(Path.of("screenshots/captcha.png").toAbsolutePath()); - String base64 = java.util.Base64.getEncoder().encodeToString(imageBytes); - - assertNotNull(base64); - assertTrue(base64.length() > 0, "Base64 should not be empty"); - assertFalse(base64.contains("\n"), "Base64 should be single line"); - - // Verify roundtrip - byte[] decoded = java.util.Base64.getDecoder().decode(base64); - assertArrayEquals(imageBytes, decoded, "Base64 roundtrip should match original"); - } - - @Test - void testCaptchaRecognition() throws Exception { - Path captchaPath = Path.of("screenshots/captcha.png"); - String captchaText = EtsScraper.recognizeCaptcha(captchaPath); - - System.out.println("[+] Recognized captcha: " + captchaText); - assertNotNull(captchaText, "Captcha recognition should return a result"); - assertFalse(captchaText.isEmpty(), "Captcha text should not be empty"); - System.out.println("[+] Captcha length: " + captchaText.length() + " chars"); - } -} diff --git a/src/test/java/com/ets/scraper/ImportTest.java b/src/test/java/com/ets/scraper/ImportTest.java deleted file mode 100644 index acbb650..0000000 --- a/src/test/java/com/ets/scraper/ImportTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.ets.scraper; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.File; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.*; - -class ImportTest { - - private static final String PROXY_HOST = "https://api.ets.niko.red"; - private static final String USERNAME = "admin"; - private static final String PASSWORD = "123456"; - - @TempDir - Path tempDir; - - @Test - void testImportExcel() throws Exception { - File testFile = tempDir.resolve("三联单列表_20260504.xls").toFile(); - Path source = Path.of("downloads/三联单列表_20260504.xls"); - if (source.toFile().exists()) { - java.nio.file.Files.copy(source, testFile.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING); - } else { - System.out.println("[!] Test file not found at " + source + ", using temp file as fallback"); - } - assertTrue(testFile.exists(), "Test file must exist"); - - String token = EtsScraper.proxyLogin(PROXY_HOST, USERNAME, PASSWORD); - assertNotNull(token); - - EtsScraper.proxyImport(testFile.toPath(), PROXY_HOST, token); - } -}