From 47bfd3976c23ba35a69e72c8b8c910fa1209a98c Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Fri, 18 Jul 2025 23:57:45 -0400 Subject: [PATCH 01/22] Fix log4j2 "Unrecognized format specifier" --- build.gradle.kts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index 2859514e2..b7883429f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -12,6 +12,7 @@ plugins { id("jacoco") id("java") id("maven-publish") + id("com.gradleup.shadow") version "9.0.0-rc1" } repositories { @@ -85,6 +86,10 @@ tasks.withType { }) } +tasks.shadowJar { + transform() +} + publishing { publications { create("maven") { From 83f6efc839be2dd023c6745a2dc2101a4dc83348 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Sat, 19 Jul 2025 01:54:15 -0400 Subject: [PATCH 02/22] Limit log pane lines --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index f41e695e8..313410a11 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -15,8 +15,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; -import java.util.Date; +import java.util.*; import java.util.List; import java.util.stream.Stream; @@ -69,6 +68,8 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JButton optionLog; private static JPanel logPanel; private static JTextPane logText; + private static final Queue logLineLengths = new LinkedList<>(); + private static final int MAX_LOG_PANE_LINES = 1000; // History private static JButton optionHistory; @@ -1257,7 +1258,11 @@ private void appendLog(final String text, final Color color) { StyledDocument sd = logText.getStyledDocument(); try { synchronized (this) { + if (logLineLengths.size() > MAX_LOG_PANE_LINES) { + sd.remove(0, logLineLengths.remove()); + } sd.insertString(sd.getLength(), text + "\n", sas); + logLineLengths.add(text.length() + 1); } } catch (BadLocationException e) { LOGGER.warn(e.getMessage()); From 770ffced5239f302d6636b6b24da7ea8ff719960 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Sun, 20 Jul 2025 02:59:41 -0400 Subject: [PATCH 03/22] Add graceful stop --- .../ripme/ripper/AbstractRipper.java | 22 ++++- .../ripme/ripper/DownloadFileThread.java | 18 ++--- .../ripme/ripper/DownloadVideoThread.java | 5 +- .../com/rarchives/ripme/ui/MainWindow.java | 81 ++++++++++++------- 4 files changed, 82 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 8ccff6481..28f36f1f7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -56,7 +56,7 @@ public abstract class AbstractRipper DownloadThreadPool threadPool; RipStatusHandler observer = null; - private boolean completed = true; + private final AtomicBoolean completed = new AtomicBoolean(false); public abstract void rip() throws IOException, URISyntaxException; @@ -71,6 +71,7 @@ public boolean hasASAPRipping() { // Everytime addUrlToDownload skips a already downloaded url this increases by 1 public int alreadyDownloadedUrls = 0; private final AtomicBoolean shouldStop = new AtomicBoolean(false); + private final AtomicBoolean shouldPanic = new AtomicBoolean(false); private static boolean thisIsATest = false; public void stop() { @@ -78,10 +79,24 @@ public void stop() { shouldStop.set(true); } + public void panic() { + logger.trace("panic()"); + shouldStop.set(true); + shouldPanic.set(true); + } + + public boolean isPanicked() { + return shouldPanic.get(); + } + public boolean isStopped() { return shouldStop.get(); } + public boolean isCompleted() { + return completed.get(); + } + protected void stopCheck() throws IOException { if (shouldStop.get()) { throw new IOException("Ripping interrupted"); @@ -480,7 +495,7 @@ public static String getFileName(URL url, String prefix, String fileName, String */ protected void waitForThreads() { logger.debug("Waiting for threads to finish"); - completed = false; + completed.set(false); threadPool.waitForThreads(); checkIfComplete(); } @@ -532,8 +547,7 @@ void checkIfComplete() { return; } - if (!completed) { - completed = true; + if (!completed.getAndSet(true)) { logger.info(" Rip completed!"); RipStatusComplete rsc = new RipStatusComplete(workingDir.toPath(), getCount()); diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index e9c6f2427..417c4370b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -63,6 +63,13 @@ public void setCookies(Map cookies) { */ @Override public void run() { + + if (observer.isStopped()) { + // TODO add handler for graceful stop + observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); + return; + } + // First thing we make sure the file name doesn't have any illegal chars in it saveAs = new File( saveAs.getParentFile().getAbsolutePath() + File.separator + Utils.sanitizeSaveAs(saveAs.getName())); @@ -72,12 +79,7 @@ public void run() { if (saveAs.exists() && observer.tryResumeDownload()) { fileSize = saveAs.length(); } - try { - observer.stopCheck(); - } catch (IOException e) { - observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); - return; - } + if (saveAs.exists() && !observer.tryResumeDownload() && !getFileExtFromMIME || Utils.fuzzyExists(Paths.get(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME && !observer.tryResumeDownload()) { @@ -251,9 +253,7 @@ public void run() { logger.debug("Not downloading whole file because it is over 10mb and this is a test"); } else { while ((bytesRead = bis.read(data)) != -1) { - try { - observer.stopCheck(); - } catch (IOException e) { + if (observer.isPanicked()) { observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); return; } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index 9430adce3..9f164ecef 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -48,6 +48,7 @@ public void run() { try { observer.stopCheck(); } catch (IOException e) { + // TODO create status for gracefully-stopped download observer.downloadErrored(url, "Download interrupted"); return; } @@ -107,9 +108,7 @@ public void run() { bis = new BufferedInputStream(huc.getInputStream()); fos = Files.newOutputStream(saveAs); while ( (bytesRead = bis.read(data)) != -1) { - try { - observer.stopCheck(); - } catch (IOException e) { + if (observer.isPanicked()) { observer.downloadErrored(url, "Download interrupted"); return; } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 313410a11..a242dbbf9 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -17,6 +17,7 @@ import java.nio.file.Paths; import java.util.*; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Stream; import javax.imageio.ImageIO; @@ -48,13 +49,11 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static final Logger LOGGER = LogManager.getLogger(MainWindow.class); - /* not static! */ - private boolean isRipping = false; // Flag to indicate if we're ripping something - private static JFrame mainFrame; private static JTextField ripTextfield; private static JButton ripButton, stopButton; + private static JButton panicButton; private static JLabel statusLabel; private static JButton openButton; @@ -130,6 +129,9 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static AbstractRipper ripper; + private static final AtomicBoolean gracefulStop = new AtomicBoolean(false); // Allow active transfers to finish, then stop ripping. + private static final AtomicBoolean panicStop = new AtomicBoolean(false); // Immediately stop active transfers, then stop ripping. + private void updateQueue(DefaultListModel model) { if (model == null) model = queueListModel; @@ -328,6 +330,8 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib ripButton = new JButton("Rip", ripIcon); stopButton = new JButton("Stop"); stopButton.setEnabled(false); + panicButton = new JButton("Panic!"); + panicButton.setEnabled(false); try { Image stopIcon = ImageIO.read(getClass().getClassLoader().getResource("stop.png")); stopButton.setIcon(new ImageIcon(stopIcon)); @@ -350,6 +354,8 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib ripPanel.add(ripButton, gbc); gbc.gridx = 3; ripPanel.add(stopButton, gbc); + gbc.gridx = 4; + ripPanel.add(panicButton, gbc); gbc.weightx = 1; statusLabel = new JLabel(Utils.getLocalizedString("inactive")); @@ -812,13 +818,32 @@ private void update() { stopButton.addActionListener(event -> { if (ripper != null) { ripper.stop(); - isRipping = false; + gracefulStop.set(true); + queueListModel.add(0, ripper.getURL().toString()); + stopButton.setEnabled(false); + statusProgress.setValue(0); + statusProgress.setVisible(false); + pack(); + statusProgress.setValue(0); + //status(Utils.getLocalizedString("download.interrupted")); + status("Rip gracefully stopping"); + appendLog("Download interrupted", Color.RED); + } + }); + + panicButton.addActionListener(event -> { + if (ripper != null) { + ripper.stop(); + ripper.panic(); + panicStop.set(true); + queueListModel.add(0, ripper.getURL().toString()); stopButton.setEnabled(false); + panicButton.setEnabled(false); statusProgress.setValue(0); statusProgress.setVisible(false); pack(); statusProgress.setValue(0); - status(Utils.getLocalizedString("download.interrupted")); + status("Rip interrupted"); // TODO localize appendLog("Download interrupted", Color.RED); } }); @@ -1071,10 +1096,7 @@ public void mouseClicked(MouseEvent e) { @Override public void intervalAdded(ListDataEvent arg0) { updateQueue(); - - if (!isRipping) { - ripNextAlbum(); - } + ripNextAlbum(); } @Override @@ -1334,14 +1356,20 @@ private void saveHistory() { } private void ripNextAlbum() { - isRipping = true; - // Save current state of queue to configuration. Utils.setConfigList("queue", queueListModel.elements()); + boolean wasGracefulStop = gracefulStop.getAndSet(false); + boolean wasPanicStop = gracefulStop.getAndSet(false); + if (wasGracefulStop || wasPanicStop) { + // Stop requested + ripFinishCleanup(); + return; + } + if (queueListModel.isEmpty()) { // End of queue - isRipping = false; + ripFinishCleanup(); return; } @@ -1363,6 +1391,13 @@ private void ripNextAlbum() { } } + private void ripFinishCleanup() { + stopButton.setEnabled(false); + panicButton.setEnabled(false); + statusProgress.setValue(0); + statusProgress.setVisible(false); + } + private Thread ripAlbum(String urlString) { if (!logPanel.isVisible()) { optionLog.doClick(); @@ -1383,6 +1418,7 @@ private Thread ripAlbum(String urlString) { return null; } stopButton.setEnabled(true); + panicButton.setEnabled(true); statusProgress.setValue(100); openButton.setVisible(false); statusLabel.setVisible(true); @@ -1492,9 +1528,7 @@ public void actionPerformed(ActionEvent event) { mainWindow.statusWithColor("This URL is already in queue: " + url, Color.ORANGE); ripTextfield.setText(""); } - else if(!mainWindow.isRipping){ - mainWindow.ripNextAlbum(); - } + mainWindow.ripNextAlbum(); } } @@ -1513,10 +1547,6 @@ public void run() { } private synchronized void handleEvent(StatusEvent evt) { - if (ripper.isStopped()) { - return; - } - RipStatusMessage msg = evt.msg; int completedPercent = evt.ripper.getCompletionPercentage(); @@ -1562,9 +1592,7 @@ private synchronized void handleEvent(StatusEvent evt) { if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } - stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); + ripFinishCleanup(); openButton.setVisible(false); pack(); statusWithColor("Error: " + msg.getObject(), Color.RED); @@ -1595,9 +1623,8 @@ private synchronized void handleEvent(StatusEvent evt) { Utils.playSound("camera.wav"); } saveHistory(); - stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); + Utils.saveConfig(); + ripFinishCleanup(); openButton.setVisible(true); Path f = rsc.dir; String prettyFile = Utils.shortenPath(f); @@ -1667,9 +1694,7 @@ private synchronized void handleEvent(StatusEvent evt) { if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } - stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); + ripFinishCleanup(); openButton.setVisible(false); pack(); statusWithColor("Error: " + msg.getObject(), Color.RED); From 4c96fd8bd315e7db0f4043d6c22c6392b33382da Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 9 Sep 2025 03:37:01 -0400 Subject: [PATCH 04/22] Fix method name --- .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 2 +- .../java/com/rarchives/ripme/ripper/AbstractJSONRipper.java | 2 +- src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java | 4 +++- src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java | 2 +- src/main/java/com/rarchives/ripme/ripper/VideoRipper.java | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 0740f62c4..aed3959a8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -475,7 +475,7 @@ protected void checkIfComplete() { return; } if (itemsPending.isEmpty()) { - super.checkIfComplete(); + notifyComplete(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index a49084c63..94cd12bf1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -277,7 +277,7 @@ protected void checkIfComplete() { return; } if (itemsPending.isEmpty()) { - super.checkIfComplete(); + notifyComplete(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 28f36f1f7..54e586b32 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -538,10 +538,12 @@ int getCount() { return 1; } + protected abstract void checkIfComplete(); + /** * Notifies observers and updates state if all files have been ripped. */ - void checkIfComplete() { + protected void notifyComplete() { if (observer == null) { logger.debug("observer is null"); return; diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index bda3bf6fb..870143ac7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -187,7 +187,7 @@ protected void checkIfComplete() { return; } if (itemsPending.isEmpty()) { - super.checkIfComplete(); + notifyComplete(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 785f3d92b..67515e789 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -207,7 +207,7 @@ protected void checkIfComplete() { } if (bytesCompleted >= bytesTotal) { - super.checkIfComplete(); + notifyComplete(); } } From b66f090a291e7531652a524868a1ab5dbba1a87f Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 10 Aug 2026 05:51:10 +0200 Subject: [PATCH 05/22] Add transfer rate monitor Includes throttling the progress update to every 200ms (instead of every read chunk, which could fire thousands of times/sec) and a fix for the throttled byte tracking undercounting the byte-based progress bar / dropping the final <200ms chunk of every download. Co-Authored-By: Claude Sonnet 5 --- .../ripme/ripper/DownloadFileThread.java | 21 ++++- .../ripme/ripper/DownloadVideoThread.java | 1 + .../com/rarchives/ripme/ui/MainWindow.java | 57 +++++++++++++- .../rarchives/ripme/ui/RipStatusMessage.java | 1 + .../rarchives/ripme/utils/TransferRate.java | 77 +++++++++++++++++++ 5 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/utils/TransferRate.java diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 417c4370b..c1579582f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -252,14 +252,33 @@ public void run() { if (shouldSkipFileDownload) { logger.debug("Not downloading whole file because it is over 10mb and this is a test"); } else { + long lastProgressUpdate = 0; + long bytesSinceLastProgressUpdate = 0; while ((bytesRead = bis.read(data)) != -1) { if (observer.isPanicked()) { observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); return; } fos.write(data, 0, bytesRead); + bytesSinceLastProgressUpdate += bytesRead; + long now = System.currentTimeMillis(); + if (now > lastProgressUpdate + 200) { + lastProgressUpdate = now; + observer.sendUpdate(STATUS.CHUNK_BYTES, bytesSinceLastProgressUpdate); + if (observer.useByteProgessBar()) { + bytesDownloaded += bytesSinceLastProgressUpdate; + observer.setBytesCompleted(bytesDownloaded); + observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); + } + bytesSinceLastProgressUpdate = 0; + } + } + if (bytesSinceLastProgressUpdate > 0) { + // Flush the remaining bytes read since the last throttled update, otherwise the + // tail of every download is silently dropped from the progress/transfer-rate totals. + observer.sendUpdate(STATUS.CHUNK_BYTES, bytesSinceLastProgressUpdate); if (observer.useByteProgessBar()) { - bytesDownloaded += bytesRead; + bytesDownloaded += bytesSinceLastProgressUpdate; observer.setBytesCompleted(bytesDownloaded); observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index 9f164ecef..b825598f1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -113,6 +113,7 @@ public void run() { return; } fos.write(data, 0, bytesRead); + observer.sendUpdate(STATUS.CHUNK_BYTES, bytesRead); bytesDownloaded += bytesRead; observer.setBytesCompleted(bytesDownloaded); observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index a242dbbf9..652430881 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -18,6 +18,10 @@ import java.util.*; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import javax.imageio.ImageIO; @@ -40,6 +44,7 @@ import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.uiUtils.ContextActionProtections; import com.rarchives.ripme.utils.RipUtils; +import com.rarchives.ripme.utils.TransferRate; import com.rarchives.ripme.utils.Utils; /** @@ -56,6 +61,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JButton panicButton; private static JLabel statusLabel; + private static final JLabel transferRateLabel = new JLabel(); private static JButton openButton; private static JProgressBar statusProgress; @@ -132,6 +138,30 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static final AtomicBoolean gracefulStop = new AtomicBoolean(false); // Allow active transfers to finish, then stop ripping. private static final AtomicBoolean panicStop = new AtomicBoolean(false); // Immediately stop active transfers, then stop ripping. + public static final int TRANSFER_RATE_REFRESH_RATE = 200; + private static final TransferRate transferRate = new TransferRate(); + + private static final ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + private Future rateRefresherFuture = null; + private final Runnable rateRefresher = () -> { + if (isHalted()) { + if (rateRefresherFuture != null) { + rateRefresherFuture.cancel(true); + rateRefresherFuture = null; + } + transferRateLabel.setText(""); + return; + } + transferRateLabel.setText(transferRate.formatHumanTransferRate()); + }; + + /** + * @return true if fully halted/panic button pressed + */ + public static boolean isHalted() { + return ripper == null || ripper.isPanicked() || ripper.isCompleted(); + } + private void updateQueue(DefaultListModel model) { if (model == null) model = queueListModel; @@ -284,6 +314,8 @@ private void createUI(Container pane) { LOGGER.error("[!] Exception setting system theme:", e); } + Font monospaced = new Font(Font.MONOSPACED, Font.PLAIN, mainFrame.getContentPane().getFont().getSize()); + ripTextfield = new JTextField("", 20); ripTextfield.addMouseListener(new ContextMenuMouseListener(ripTextfield)); @@ -360,16 +392,26 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib statusLabel = new JLabel(Utils.getLocalizedString("inactive")); statusLabel.setHorizontalAlignment(JLabel.CENTER); + transferRateLabel.setHorizontalAlignment(JLabel.RIGHT); + transferRateLabel.setFont(monospaced); openButton = new JButton(); openButton.setVisible(false); JPanel statusPanel = new JPanel(new GridBagLayout()); statusPanel.setBorder(emptyBorder); gbc.gridx = 0; + gbc.weightx = 1; statusPanel.add(statusLabel, gbc); + gbc.gridx = 1; + gbc.weightx = 0; + statusPanel.add(transferRateLabel, gbc); + gbc.gridx = 0; + gbc.weightx = 1; + gbc.gridwidth = 2; gbc.gridy = 1; statusPanel.add(openButton, gbc); gbc.gridy = 0; + gbc.gridwidth = 1; JPanel progressPanel = new JPanel(new GridBagLayout()); progressPanel.setBorder(emptyBorder); @@ -1373,6 +1415,10 @@ private void ripNextAlbum() { return; } + if (rateRefresherFuture == null || rateRefresherFuture.isDone()) { + rateRefresherFuture = executor.scheduleAtFixedRate(rateRefresher, 0, TRANSFER_RATE_REFRESH_RATE, TimeUnit.MILLISECONDS); + } + String nextAlbum = (String) queueListModel.remove(0); updateQueue(); @@ -1422,6 +1468,7 @@ private Thread ripAlbum(String urlString) { statusProgress.setValue(100); openButton.setVisible(false); statusLabel.setVisible(true); + transferRateLabel.setVisible(true); pack(); boolean failed = false; try { @@ -1548,13 +1595,21 @@ public void run() { private synchronized void handleEvent(StatusEvent evt) { RipStatusMessage msg = evt.msg; + RipStatusMessage.STATUS status = msg.getStatus(); + + // CHUNK_BYTES is noisy, so handle it before any other computation + if (status == RipStatusMessage.STATUS.CHUNK_BYTES) { + transferRate.addChunk((Long) msg.getObject()); + transferRateLabel.setText(transferRate.formatHumanTransferRate()); + return; + } int completedPercent = evt.ripper.getCompletionPercentage(); statusProgress.setValue(completedPercent); statusProgress.setVisible(true); status(evt.ripper.getStatusText()); - switch (msg.getStatus()) { + switch (status) { case LOADING_RESOURCE: case DOWNLOAD_STARTED: if (LOGGER.isEnabled(Level.INFO)) { diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java index f589e9dbb..3d4c1644f 100644 --- a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java +++ b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java @@ -16,6 +16,7 @@ public enum STATUS { DOWNLOAD_SKIP("Download Skipped"), TOTAL_BYTES("Total bytes"), COMPLETED_BYTES("Completed bytes"), + CHUNK_BYTES("Transferred bytes in last chunk"), RIP_ERRORED("Rip Errored"), NO_ALBUM_OR_USER("No album or user"); diff --git a/src/main/java/com/rarchives/ripme/utils/TransferRate.java b/src/main/java/com/rarchives/ripme/utils/TransferRate.java new file mode 100644 index 000000000..9227160f3 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/utils/TransferRate.java @@ -0,0 +1,77 @@ +package com.rarchives.ripme.utils; + +import java.util.Deque; +import java.util.concurrent.ConcurrentLinkedDeque; + +public class TransferRate { + private final Deque transferQueue = new ConcurrentLinkedDeque<>(); + private int windowDurationMs = 10000; + + public void addChunk(long bytes) { + long now = System.currentTimeMillis(); + transferQueue.addFirst(new ChunkStamp(now, bytes)); + removeOldChunks(now); + } + + public double calculateBytesPerSecond() { + if (transferQueue.isEmpty()) { + return 0; + } + long totalBytes = 0; + ChunkStamp oldest = transferQueue.getLast(); + long now = System.currentTimeMillis(); + removeOldChunks(now); + if (transferQueue.isEmpty()) { + return 0; + } + for (ChunkStamp chunkStamp : transferQueue) { + totalBytes += chunkStamp.bytes; + } + long elapsedMs = now - oldest.timestampMs; + double elapsedSeconds = (double) elapsedMs / 1000; + if (elapsedSeconds <= 0) { + return 0; + } + return totalBytes / elapsedSeconds; + } + + public String formatHumanTransferRate() { + double bps = calculateBytesPerSecond(); + int giB = 1024 * 1024 * 1024; + int miB = 1024 * 1024; + int kiB = 1024; + if (bps >= giB) { + return String.format("%.2f GiB/s", bps / giB); + } else if (bps >= miB) { + return String.format("%.2f MiB/s", bps / miB); + } else if (bps >= kiB) { + return String.format("%.2f KiB/s", bps / kiB); + } else { + return String.format("%.2f B/s", bps); + } + } + + public void setWindowDurationMs(int windowDurationMs) { + this.windowDurationMs = windowDurationMs; + } + + public int getWindowDurationMs() { + return windowDurationMs; + } + + private void removeOldChunks(long now) { + while (!transferQueue.isEmpty() && transferQueue.getLast().timestampMs < now - windowDurationMs) { + transferQueue.removeLast(); + } + } + + private static class ChunkStamp { + long bytes; + long timestampMs; // millis + + ChunkStamp(long timestampMs, long bytes) { + this.bytes = bytes; + this.timestampMs = timestampMs; + } + } +} From 5c5feb78b6fc80245c5d7ec40bb4fbfa3f6946f3 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 10 Aug 2026 05:51:24 +0200 Subject: [PATCH 06/22] Handle no space left on device Detects ENOSPC locale-independently: IOException's message for ENOSPC is localized by the JVM, so a plain 'No space left on device' string match only works on English-locale systems. Falls back to checking the filesystem's usable space via FileStore when the message doesn't match. Co-Authored-By: Claude Sonnet 5 --- .../ripme/ripper/DownloadFileThread.java | 31 ++++++++++++++++++- src/main/resources/LabelsBundle.properties | 3 +- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index c1579582f..bf4a47d71 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -2,6 +2,7 @@ import java.io.*; import java.net.*; +import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; @@ -300,7 +301,16 @@ public void run() { "HTTP status code " + hse.getStatusCode() + " while downloading " + url.toExternalForm()); return; } - } catch (IOException | URISyntaxException e) { + } catch (IOException e) { + if (guessIsENOSPC(e, saveAs)) { + logger.debug("IOException", e); + observer.downloadErrored(url, Utils.getLocalizedString("device.nospace")); // TODO cancel all rips + return; + } + logger.debug("IOException", e); + logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + + e.getMessage()); + } catch (URISyntaxException e) { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); @@ -328,4 +338,23 @@ public void run() { logger.info("[+] Saved " + url + " as " + this.prettySaveAs); } + @SuppressWarnings("UnnecessaryLocalVariable") + private boolean guessIsENOSPC(IOException e, File saveAs) { + // The ENOSPC IOException message is localized in Java, so this only works on English locale systems. + if (e.getMessage() != null && e.getMessage().matches("No space left on device")) { + return true; + } + // Fallback: check usable space on the filesystem + try { + FileStore fs = Files.getFileStore(saveAs.toPath()); + // could check for 0 bytes, but 256 kilobytes is small enough + int downloadBufferSizeBytes = 1024 * 256; + boolean notEnoughUsableBytes = fs.getUsableSpace() < downloadBufferSizeBytes; + return notEnoughUsableBytes; + } catch (IOException ex) { + // unable to determine if no space left on device; fall through + } + return false; + } + } diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index 30aaffbab..444862664 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -87,4 +87,5 @@ http.status.exception = HTTP status exception exception.while.downloading.file = Exception while downloading file failed.to.download = Failed to download skipping = Skipping -file.already.exists = file already exists \ No newline at end of file +file.already.exists = file already exists +device.nospace = No space left on device \ No newline at end of file From 3dd1a817391f99852380d627005c8c64d511e50c Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Sun, 20 Jul 2025 02:37:49 -0400 Subject: [PATCH 07/22] Fix sorter usage warning --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 652430881..ace2baf2a 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1380,6 +1380,10 @@ private void loadHistory() throws IOException { }); } } + if (!HISTORY.isEmpty()) { + // Fix "WARNING: row index is bigger than sorter's row count. Most likely this is a wrong sorter usage" + historyTableModel.fireTableDataChanged(); + } } private void saveHistory() { From 124018f5ed807f7fbd683e295ae14c9ddb791432 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Sun, 20 Jul 2025 02:38:39 -0400 Subject: [PATCH 08/22] Set spammy log message to trace --- src/main/java/com/rarchives/ripme/utils/Utils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 36fa7273e..4e9786c75 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -809,7 +809,7 @@ public static String[] getSupportedLanguages() { } public static String getLocalizedString(String key) { - LOGGER.debug(String.format("Key %s in %s is: %s", key, getSelectedLanguage(), + LOGGER.trace(String.format("Key %s in %s is: %s", key, getSelectedLanguage(), resourceBundle.getString(key))); return resourceBundle.getString(key); } From c927b9ed32c92c26c9c3ccb542c8be0f9369d508 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 00:37:33 -0400 Subject: [PATCH 09/22] Clarify download try # log message --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 2 +- .../java/com/rarchives/ripme/ripper/DownloadVideoThread.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index bf4a47d71..46761f29d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -100,7 +100,7 @@ public void run() { do { tries += 1; try { - logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Retry #" + tries : "")); + logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Try #" + tries : "")); observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); // Setup HTTP request diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index b825598f1..3d070e8c2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -85,7 +85,7 @@ public void run() { byte[] data = new byte[1024 * 256]; int bytesRead; try { - logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : "")); + logger.info(" Downloading file: " + url + (tries > 0 ? " Try #" + tries+1 : "")); observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); // Setup HTTP request From 3ca98b60abf9cc388bab638d52164a8fed4c124e Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 00:42:13 -0400 Subject: [PATCH 10/22] Simplify expression --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 46761f29d..2ba04ae14 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -105,7 +105,7 @@ public void run() { // Setup HTTP request HttpURLConnection huc; - if (this.url.toString().startsWith("https")) { + if (url.getProtocol().equals("https")) { huc = (HttpsURLConnection) urlToDownload.openConnection(); } else { huc = (HttpURLConnection) urlToDownload.openConnection(); From 4a80c92aa55ee6ae747b972dd24154b3e8577637 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 10 Aug 2026 05:51:47 +0200 Subject: [PATCH 11/22] Fix download error handling Generic IOExceptions now notify downloadErrored and give up instead of silently falling through the retry loop forever. Redirects and retriable 5xx responses use continue to retry directly rather than the old throw-to-retry trick, which the above change broke (5xx responses stopped retrying after a single attempt since the generic IOException handler now returns immediately instead of falling through the loop). Co-Authored-By: Claude Sonnet 5 --- .../rarchives/ripme/ripper/DownloadFileThread.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 2ba04ae14..a759e7aa1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -153,8 +153,8 @@ public void run() { } String location = huc.getHeaderField("Location"); urlToDownload = new URI(location).toURL(); - // Throw exception so download can be retried - throw new IOException("Redirect status code " + statusCode + " - redirect to " + location); + logger.debug("Redirect status code {} - redirect to {}", statusCode, location); + continue; // retry } if (statusCode / 100 == 4) { // 4xx errors logger.error("[!] " + Utils.getLocalizedString("nonretriable.status.code") + " " + statusCode @@ -166,8 +166,8 @@ public void run() { if (statusCode / 100 == 5) { // 5xx errors observer.downloadErrored(url, Utils.getLocalizedString("retriable.status.code") + " " + statusCode + " while downloading " + url.toExternalForm()); - // Throw exception so download can be retried - throw new IOException(Utils.getLocalizedString("retriable.status.code") + " " + statusCode); + logger.debug("Retriable status code {} while downloading {}", statusCode, url); + continue; // retry } if (huc.getContentLength() == 503 && urlToDownload.getHost().endsWith("imgur.com")) { // Imgur image with 503 bytes is "404" @@ -310,10 +310,14 @@ public void run() { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); + observer.downloadErrored(url, e.getMessage()); + return; } catch (URISyntaxException e) { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); + observer.downloadErrored(url, Utils.getLocalizedString("exception.while.downloading.file")); + return; } catch (NullPointerException npe){ logger.error("[!] " + Utils.getLocalizedString("failed.to.download") + " for URL " + url); From 038cc6bc090e0e41268b4f758133cfe881c6d334 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 10 Aug 2026 05:52:07 +0200 Subject: [PATCH 12/22] Support fetching media from single-use token URLs Introduces TokenedUrlGetter and RipUrlId: some hosts serve media via signed or time-limited URLs, so downloads need a way to fetch a fresh URL for the same logical item rather than reusing a possibly-expired one, and a stable identity (RipUrlId) to track/dedupe that item across albums independent of which URL was used to fetch it. Co-Authored-By: Claude Sonnet 5 --- .../ripme/ripper/AbstractHTMLRipper.java | 81 ++++++------ .../ripme/ripper/AbstractJSONRipper.java | 76 +++++------ .../ripme/ripper/AbstractRipper.java | 32 +++-- .../rarchives/ripme/ripper/AlbumRipper.java | 72 ++++++----- .../ripme/ripper/DownloadFileThread.java | 122 +++++++++++++----- .../ripme/ripper/DownloadVideoThread.java | 104 ++++++++++----- .../com/rarchives/ripme/ripper/RipUrlId.java | 97 ++++++++++++++ .../ripme/ripper/TokenedUrlGetter.java | 14 ++ .../rarchives/ripme/ripper/VideoRipper.java | 55 ++++---- 9 files changed, 448 insertions(+), 205 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/ripper/RipUrlId.java create mode 100644 src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index aed3959a8..0c0b72731 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -13,11 +13,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -36,9 +32,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AbstractHTMLRipper.class); - private final Map itemsPending = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); + private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); + private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); Document cachedFirstPage; protected AbstractHTMLRipper(URL url) throws IOException { @@ -346,11 +342,10 @@ public int getCount() { return itemsCompleted.size() + itemsErrored.size(); } - @Override /* Queues multiple URLs of single images to download from a single Album URL */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); @@ -358,31 +353,42 @@ public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); + private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); + private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); protected AbstractJSONRipper(URL url) throws IOException { super(url); @@ -152,7 +149,7 @@ public int getCount() { /** * Queues multiple URLs of single images to download from a single Album URL */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); @@ -160,31 +157,41 @@ public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, - Boolean getFileExtFromMIME); + public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + TokenedUrlGetter tug = () -> url; + RipUrlId ripUrlId = new RipUrlId(getClass(), getHost(), url); + Path directory = saveAs.getParent(); + String filename = saveAs.getFileName().toString(); + return addURLToDownload(tug, ripUrlId, directory, filename, referrer, cookies, getFileExtFromMIME); + } + + protected boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String referrer, Map cookies, Boolean getFileExtFromMIME) { + return addURLToDownload(tug, ripUrlId, directory, null, referrer, cookies, getFileExtFromMIME); + } + + public abstract boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME); + /** * Queues image to be downloaded and saved. @@ -515,21 +529,21 @@ public void retrievingSource(String url) { /** * Notifies observers that a file download has completed. * - * @param url URL that was completed. - * @param saveAs Where the downloaded file is stored. + * @param ripUrlId The RipUrlId that was completed. + * @param saveAs Where the downloaded file is stored. */ - public abstract void downloadCompleted(URL url, Path saveAs); + public abstract void downloadCompleted(RipUrlId ripUrlId, Path saveAs); /** * Notifies observers that a file could not be downloaded (includes a reason). */ - public abstract void downloadErrored(URL url, String reason); + public abstract void downloadErrored(RipUrlId ripUrlId, String reason); /** * Notify observers that a download could not be completed, * but was not technically an "error". */ - public abstract void downloadExists(URL url, Path file); + public abstract void downloadExists(RipUrlId ripUrlId, Path file); /** * @return Number of files downloaded. @@ -805,7 +819,7 @@ protected boolean tryResumeDownload() { return false; } - protected boolean shouldIgnoreURL(URL url) { + protected static boolean shouldIgnoreExtension(URL url) { final String[] ignoredExtensions = Utils.getConfigStringArray("download.ignore_extensions"); if (ignoredExtensions == null || ignoredExtensions.length == 0) return false; // nothing ignored diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 870143ac7..6df442424 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -10,9 +10,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -32,9 +30,9 @@ public abstract class AlbumRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AlbumRipper.class); - private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); + private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); + private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); protected AlbumRipper(URL url) throws IOException { super(url); @@ -62,7 +60,7 @@ public int getCount() { /** * Queues multiple URLs of single images to download from a single Album URL */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); @@ -70,31 +68,42 @@ public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies = new HashMap<>(); - private final URL url; - private File saveAs; - private final String prettySaveAs; + private final Path directory; + private String filename; private final AbstractRipper observer; private final int retries; private final Boolean getFileExtFromMIME; @@ -38,11 +40,13 @@ class DownloadFileThread implements Runnable { private final int TIMEOUT; private final int retrySleep; - public DownloadFileThread(URL url, File saveAs, AbstractRipper observer, Boolean getFileExtFromMIME) { + + public DownloadFileThread(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, AbstractRipper observer, Boolean getFileExtFromMIME) { super(); - this.url = url; - this.saveAs = saveAs; - this.prettySaveAs = Utils.removeCWD(saveAs.toPath()); + this.tokenedUrlGetter = tug; + this.ripUrlId = ripUrlId; + this.directory = directory; + this.filename = filename; this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000); @@ -67,13 +71,47 @@ public void run() { if (observer.isStopped()) { // TODO add handler for graceful stop - observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("download.interrupted")); return; } + URL url = null; + try { + url = tokenedUrlGetter.getTokenedUrl(); + } catch (HttpStatusException e) { + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + logger.error("[!] Failed to get URL for " + ripUrlId); + return; // do not retry + } catch (IOException | URISyntaxException e) { + logger.error("[!] Failed to get URL for " + ripUrlId, e); + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + return; // do not retry + } + if (filename == null) { + // Strip token query parameters + filename = Path.of(url.getPath()).getFileName().toString(); + } // First thing we make sure the file name doesn't have any illegal chars in it - saveAs = new File( - saveAs.getParentFile().getAbsolutePath() + File.separator + Utils.sanitizeSaveAs(saveAs.getName())); + filename = Utils.sanitizeSaveAs(filename); + if (AbstractRipper.shouldIgnoreExtension(url)) { + observer.sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + return; + } + + if (!Files.exists(directory)) { + logger.info("[+] Creating directory: " + directory); + try { + Files.createDirectories(directory); + } catch (IOException e) { + logger.error("Error creating directory", e); + observer.downloadErrored(ripUrlId, "Error creating directory: " + directory + " ; " + e.getMessage()); + return; + } + } + + File saveAs = directory.resolve(filename).toFile(); + String prettySaveAs = Utils.removeCWD(saveAs.toPath()); + long fileSize = 0; int bytesTotal; int bytesDownloaded = 0; @@ -90,25 +128,26 @@ public void run() { } else { logger.info("[!] " + Utils.getLocalizedString("skipping") + " " + url + " -- " + Utils.getLocalizedString("file.already.exists") + ": " + prettySaveAs); - observer.downloadExists(url, saveAs.toPath()); + observer.downloadExists(ripUrlId, saveAs.toPath()); return; } } - URL urlToDownload = this.url; boolean redirected = false; int tries = 0; // Number of attempts to download do { tries += 1; try { - logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Try #" + tries : "")); - observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); + logger.info(" Downloading file: " + url + (tries > 0 ? " Try #" + tries : "")); + + String urlNoQuery = new URI(url.getProtocol(), url.getAuthority(), url.getPath(), null, null).toURL().toExternalForm(); + observer.sendUpdate(STATUS.DOWNLOAD_STARTED, urlNoQuery); // Setup HTTP request HttpURLConnection huc; if (url.getProtocol().equals("https")) { - huc = (HttpsURLConnection) urlToDownload.openConnection(); + huc = (HttpsURLConnection) url.openConnection(); } else { - huc = (HttpURLConnection) urlToDownload.openConnection(); + huc = (HttpURLConnection) url.openConnection(); } huc.setInstanceFollowRedirects(true); // It is important to set both ConnectTimeout and ReadTimeout. If you don't then @@ -143,7 +182,10 @@ public void run() { if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) { // TODO find a better way to handle servers that don't support resuming // downloads then just erroring out - throw new IOException(Utils.getLocalizedString("server.doesnt.support.resuming.downloads")); + observer.downloadErrored(ripUrlId, "Local file exists, resume attempted, but server does not support resuming downloads: " + + statusCode + " while downloading " + url.toExternalForm()); + //throw new IOException(Utils.getLocalizedString("server.doesnt.support.resuming.downloads")); + return; } if (statusCode / 100 == 3) { // 3xx Redirect if (!redirected) { @@ -152,27 +194,27 @@ public void run() { redirected = true; } String location = huc.getHeaderField("Location"); - urlToDownload = new URI(location).toURL(); + url = new URI(location).toURL(); // TODO fix redirect with TokenedUrlGetter logger.debug("Redirect status code {} - redirect to {}", statusCode, location); continue; // retry } if (statusCode / 100 == 4) { // 4xx errors logger.error("[!] " + Utils.getLocalizedString("nonretriable.status.code") + " " + statusCode + " while downloading from " + url); - observer.downloadErrored(url, Utils.getLocalizedString("nonretriable.status.code") + " " + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("nonretriable.status.code") + " " + statusCode + " while downloading " + url.toExternalForm()); return; // Not retriable, drop out. } if (statusCode / 100 == 5) { // 5xx errors - observer.downloadErrored(url, Utils.getLocalizedString("retriable.status.code") + " " + statusCode + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("retriable.status.code") + " " + statusCode + " while downloading " + url.toExternalForm()); logger.debug("Retriable status code {} while downloading {}", statusCode, url); continue; // retry } - if (huc.getContentLength() == 503 && urlToDownload.getHost().endsWith("imgur.com")) { + if (huc.getContentLength() == 503 && url.getHost().endsWith("imgur.com")) { // Imgur image with 503 bytes is "404" logger.error("[!] Imgur image is 404 (503 bytes long): " + url); - observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm()); + observer.downloadErrored(ripUrlId, "Imgur image is 404: " + url.toExternalForm()); return; } @@ -182,7 +224,7 @@ public void run() { bytesTotal = huc.getContentLength(); observer.setBytesTotal(bytesTotal); observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); + logger.debug("Size of file at " + url + " = " + bytesTotal + "b"); } // Save file @@ -257,7 +299,7 @@ public void run() { long bytesSinceLastProgressUpdate = 0; while ((bytesRead = bis.read(data)) != -1) { if (observer.isPanicked()) { - observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("download.interrupted")); return; } fos.write(data, 0, bytesRead); @@ -295,33 +337,33 @@ public void run() { break; } catch (HttpStatusException hse) { logger.debug(Utils.getLocalizedString("http.status.exception"), hse); - logger.error("[!] HTTP status " + hse.getStatusCode() + " while downloading from " + urlToDownload); + logger.error("[!] HTTP status " + hse.getStatusCode() + " while downloading from " + hse.getUrl()); if (hse.getStatusCode() == 404 && Utils.getConfigBoolean("errors.skip404", false)) { - observer.downloadErrored(url, + observer.downloadErrored(ripUrlId, "HTTP status code " + hse.getStatusCode() + " while downloading " + url.toExternalForm()); return; } } catch (IOException e) { if (guessIsENOSPC(e, saveAs)) { logger.debug("IOException", e); - observer.downloadErrored(url, Utils.getLocalizedString("device.nospace")); // TODO cancel all rips + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("device.nospace")); // TODO cancel all rips return; } logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); - observer.downloadErrored(url, e.getMessage()); + observer.downloadErrored(ripUrlId, e.getMessage()); return; } catch (URISyntaxException e) { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); - observer.downloadErrored(url, Utils.getLocalizedString("exception.while.downloading.file")); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("exception.while.downloading.file")); return; } catch (NullPointerException npe){ logger.error("[!] " + Utils.getLocalizedString("failed.to.download") + " for URL " + url); - observer.downloadErrored(url, + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); return; @@ -329,7 +371,7 @@ public void run() { if (tries > this.retries) { logger.error("[!] " + Utils.getLocalizedString("exceeded.maximum.retries") + " (" + this.retries + ") for URL " + url); - observer.downloadErrored(url, + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); return; } else { @@ -337,9 +379,23 @@ public void run() { Utils.sleep(retrySleep); } } + + // get fresh URL for the next attempt + try { + url = tokenedUrlGetter.getTokenedUrl(); + } catch (HttpStatusException e) { + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + logger.error("[!] Failed to get URL for " + ripUrlId); + return; // do not retry + } catch (IOException | URISyntaxException e) { + logger.error("[!] Failed to get URL for " + ripUrlId, e); + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + return; // do not retry + } + } while (true); - observer.downloadCompleted(url, saveAs.toPath()); - logger.info("[+] Saved " + url + " as " + this.prettySaveAs); + observer.downloadCompleted(ripUrlId, saveAs.toPath()); + logger.info("[+] Saved " + url + " as " + prettySaveAs); } @SuppressWarnings("UnnecessaryLocalVariable") diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index 3d070e8c2..9a3f6f7cf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -1,10 +1,8 @@ package com.rarchives.ripme.ripper; -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.net.HttpURLConnection; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -15,6 +13,7 @@ import com.rarchives.ripme.utils.Utils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jsoup.HttpStatusException; /** * Thread for downloading files. @@ -24,17 +23,19 @@ class DownloadVideoThread implements Runnable { private static final Logger logger = LogManager.getLogger(DownloadVideoThread.class); - private final URL url; - private final Path saveAs; - private final String prettySaveAs; + private final TokenedUrlGetter tokenedUrlGetter; // Some URLs may be valid for a limited time. This should get a fresh url + private final RipUrlId ripUrlId; + private final Path directory; + private String filename; private final AbstractRipper observer; private final int retries; - public DownloadVideoThread(URL url, Path saveAs, AbstractRipper observer) { + public DownloadVideoThread(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, AbstractRipper observer) { super(); - this.url = url; - this.saveAs = saveAs; - this.prettySaveAs = Utils.removeCWD(saveAs); + this.tokenedUrlGetter = tug; + this.ripUrlId = ripUrlId; + this.directory = directory; + this.filename = filename; this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); } @@ -45,13 +46,44 @@ public DownloadVideoThread(URL url, Path saveAs, AbstractRipper observer) { */ @Override public void run() { - try { - observer.stopCheck(); - } catch (IOException e) { + if (observer.isStopped()) { // TODO create status for gracefully-stopped download - observer.downloadErrored(url, "Download interrupted"); + observer.downloadErrored(ripUrlId, "Download interrupted"); return; } + URL url = null; + try { + url = tokenedUrlGetter.getTokenedUrl(); + } catch (HttpStatusException e) { + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + logger.error("[!] Failed to get URL for " + ripUrlId); + return; // do not retry + } catch (IOException | URISyntaxException e) { + logger.error("[!] Failed to get URL for " + ripUrlId, e); + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + return; // do not retry + } + if (filename == null) { + // Strip token query parameters + filename = Path.of(url.getPath()).getFileName().toString(); + } + if (AbstractRipper.shouldIgnoreExtension(url)) { + observer.sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + return; + } + if (!Files.exists(directory)) { + logger.info("[+] Creating directory: " + directory); + try { + Files.createDirectories(directory); + } catch (IOException e) { + logger.error("Error creating directory", e); + observer.downloadErrored(ripUrlId, "Error creating directory: " + directory + " ; " + e.getMessage()); + return; + } + } + Path saveAs = directory.resolve(filename); + String prettySaveAs = Utils.removeCWD(saveAs); + if (Files.exists(saveAs)) { if (Utils.getConfigBoolean("file.overwrite", false)) { logger.info("[!] Deleting existing file" + prettySaveAs); @@ -62,22 +94,22 @@ public void run() { } } else { logger.info("[!] Skipping " + url + " -- file already exists: " + prettySaveAs); - observer.downloadExists(url, saveAs); + observer.downloadExists(ripUrlId, saveAs); return; } } int bytesTotal, bytesDownloaded = 0; try { - bytesTotal = getTotalBytes(this.url); + bytesTotal = getTotalBytes(url); } catch (IOException e) { - logger.error("Failed to get file size at " + this.url, e); - observer.downloadErrored(this.url, "Failed to get file size of " + this.url); + logger.error("Failed to get file size at " + url, e); + observer.downloadErrored(ripUrlId, "Failed to get file size of " + url); return; } observer.setBytesTotal(bytesTotal); observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); + logger.debug("Size of file at " + url + " = " + bytesTotal + "b"); int tries = 0; // Number of attempts to download do { @@ -90,16 +122,16 @@ public void run() { // Setup HTTP request HttpURLConnection huc; - if (this.url.toString().startsWith("https")) { - huc = (HttpsURLConnection) this.url.openConnection(); + if (url.getProtocol().equals("https")) { + huc = (HttpsURLConnection) url.openConnection(); } else { - huc = (HttpURLConnection) this.url.openConnection(); + huc = (HttpURLConnection) url.openConnection(); } huc.setInstanceFollowRedirects(true); huc.setConnectTimeout(0); // Never timeout huc.setRequestProperty("accept", "*/*"); - huc.setRequestProperty("Referer", this.url.toExternalForm()); // Sic + huc.setRequestProperty("Referer", url.toExternalForm()); // Sic huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); tries += 1; logger.debug("Request properties: " + huc.getRequestProperties().toString()); @@ -109,7 +141,7 @@ public void run() { fos = Files.newOutputStream(saveAs); while ( (bytesRead = bis.read(data)) != -1) { if (observer.isPanicked()) { - observer.downloadErrored(url, "Download interrupted"); + observer.downloadErrored(ripUrlId, "Download interrupted"); return; } fos.write(data, 0, bytesRead); @@ -134,12 +166,26 @@ public void run() { } if (tries > this.retries) { logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url); - observer.downloadErrored(url, "Failed to download " + url.toExternalForm()); + observer.downloadErrored(ripUrlId, "Failed to download " + url.toExternalForm()); return; } + + // get fresh URL for the next attempt + try { + url = tokenedUrlGetter.getTokenedUrl(); + } catch (HttpStatusException e) { + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + logger.error("[!] Failed to get URL for " + ripUrlId); + return; // do not retry + } catch (IOException | URISyntaxException e) { + logger.error("[!] Failed to get URL for " + ripUrlId, e); + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + return; // do not retry + } + } while (true); - observer.downloadCompleted(url, saveAs); - logger.info("[+] Saved " + url + " as " + this.prettySaveAs); + observer.downloadCompleted(ripUrlId, saveAs); + logger.info("[+] Saved " + url + " as " + prettySaveAs); } /** @@ -152,7 +198,7 @@ private int getTotalBytes(URL url) throws IOException { HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("HEAD"); conn.setRequestProperty("accept", "*/*"); - conn.setRequestProperty("Referer", this.url.toExternalForm()); // Sic + conn.setRequestProperty("Referer", url.toExternalForm()); // Sic conn.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); return conn.getContentLength(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/RipUrlId.java b/src/main/java/com/rarchives/ripme/ripper/RipUrlId.java new file mode 100644 index 000000000..9e376bb2e --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/RipUrlId.java @@ -0,0 +1,97 @@ +package com.rarchives.ripme.ripper; + +import java.net.URL; +import java.util.Objects; + +/** + * RipUrlId represents a unique file on a host. + * Necessary because some files may be accessible from multiple URLs, for example: + * - a file in multiple albums, or + * - a file only accessible with a tokened URL. + */ +public class RipUrlId { + Class ripper; + String ripperHost; + String ripUrlId; + URL url; + + /** + * @param ripper The ripper associated with the id + * @param ripperHost The ripper's getHost(), because a ripper may support multiple hosts + * @param ripUrlId The unique identifier of the file fetchable by the ripper + */ + public RipUrlId(Class ripper, String ripperHost, String ripUrlId) { + if (ripper == null) { + throw new IllegalArgumentException("ripper cannot be null"); + } + if (ripperHost == null) { + throw new IllegalArgumentException("ripperHost cannot be null"); + } + if (ripUrlId == null) { + throw new IllegalArgumentException("ripUrlId cannot be null"); + } + this.ripper = ripper; + this.ripperHost = ripperHost; + this.ripUrlId = ripUrlId; + } + + /** + * Transitionary constructor for rippers that do not yet create an id + * + * @param ripper The ripper associated with the id + * @param ripperHost The ripper's getHost(), because a ripper may support multiple hosts + * @param url A URL fetchable by the ripper + * @deprecated The other constructor is preferable + */ + @Deprecated + public RipUrlId(Class ripper, String ripperHost, URL url) { + if (ripper == null) { + throw new IllegalArgumentException("ripper cannot be null"); + } + if (ripperHost == null) { + throw new IllegalArgumentException("ripperHost cannot be null"); + } + if (url == null) { + throw new IllegalArgumentException("url cannot be null"); + } + this.ripper = ripper; + this.ripperHost = ripperHost; + this.url = url; + } + + public Class getRipper() { + return ripper; + } + + public String getRipperHost() { + return ripperHost; + } + + public String getRipUrlId() { + return ripUrlId; + } + + public URL getUrl() { + return url; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + RipUrlId ripUrlId1 = (RipUrlId) o; + return Objects.equals(ripper, ripUrlId1.ripper) && Objects.equals(ripperHost, ripUrlId1.ripperHost) && Objects.equals(ripUrlId, ripUrlId1.ripUrlId) && Objects.equals(url, ripUrlId1.url); + } + + @Override + public int hashCode() { + return Objects.hash(ripper, ripperHost, ripUrlId, url); + } + + @Override + public String toString() { + if (url != null) { + return url.toString(); + } + return ripper.getSimpleName() + ": " + ripperHost + ": " + ripUrlId; + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java b/src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java new file mode 100644 index 000000000..706c7bc72 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java @@ -0,0 +1,14 @@ +package com.rarchives.ripme.ripper; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; + +public interface TokenedUrlGetter { + /** + * @return The URL of the file to fetch + * @throws IOException May be thrown if a tokened URI can't be fetched + * @throws URISyntaxException May be thrown if a URI can't be constructed + */ + URL getTokenedUrl() throws IOException, URISyntaxException; +} diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 67515e789..d717ef851 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -49,10 +49,21 @@ public String getAlbumTitle(URL url) { } @Override - public boolean addURLToDownload(URL url, Path saveAs) { + public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file String urlFile = this.workingDir + "/urls.txt"; + URL url = null; + try { + url = tug.getTokenedUrl(); + } catch (IOException | URISyntaxException e) { + logger.error("Unable to get URL for {}", ripUrlId, e); + return false; + } + if (AbstractRipper.shouldIgnoreExtension(url)) { + sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + return false; + } try (FileWriter fw = new FileWriter(urlFile, true)) { fw.write(url.toExternalForm()); @@ -64,28 +75,25 @@ public boolean addURLToDownload(URL url, Path saveAs) { logger.error("Error while writing to " + urlFile, e); return false; } + return true; } else { if (isThisATest()) { // Tests shouldn't download the whole video // Just change this.url to the download URL so the test knows we found it. logger.debug("Test rip, found URL: " + url); - this.url = url; + try { + this.url = tug.getTokenedUrl(); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } return true; } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - threadPool.addThread(new DownloadVideoThread(url, saveAs, this)); + + threadPool.addThread(new DownloadVideoThread(tug, ripUrlId, directory, filename, this)); } return true; } - @Override - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - return addURLToDownload(url, saveAs); - } - /** * Creates & sets working directory based on URL. * @@ -123,11 +131,11 @@ public int getCompletionPercentage() { /** * Runs if download successfully completed. * - * @param url Target URL - * @param saveAs Path to file, including filename. + * @param ripUrlId Target URL ID + * @param saveAs Path to file, including filename. */ @Override - public void downloadCompleted(URL url, Path saveAs) { + public void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { if (observer == null) { return; } @@ -146,31 +154,32 @@ public void downloadCompleted(URL url, Path saveAs) { /** * Runs if the download errored somewhere. * - * @param url Target URL - * @param reason Reason why the download failed. + * @param ripUrlId Target URL ID + * @param reason Reason why the download failed. */ @Override - public void downloadErrored(URL url, String reason) { + public void downloadErrored(RipUrlId ripUrlId, String reason) { if (observer == null) { return; } - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); checkIfComplete(); } /** * Runs if user tries to redownload an already existing File. - * @param url Target URL - * @param file Existing file + * + * @param ripUrlId Target URL ID + * @param file Existing file */ @Override - public void downloadExists(URL url, Path file) { + public void downloadExists(RipUrlId ripUrlId, Path file) { if (observer == null) { return; } - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); checkIfComplete(); } From 3e1484c71ea193b97badf6fa890a9710d535341c Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:09:20 -0400 Subject: [PATCH 13/22] Remove dead code --- .../com/rarchives/ripme/ripper/AlbumRipper.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 6df442424..1cbae7568 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -116,19 +116,6 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di return true; } - /** - * Queues image to be downloaded and saved. - * Uses filename from URL to decide filename. - * @param url - * URL to download - * @return - * True on success - */ - protected boolean addURLToDownload(URL url) { - // Use empty prefix and empty subdirectory - return addURLToDownload(url, "", ""); - } - @Override /** * Cleans up & tells user about successful download From 7335d02cdc54f41dc365aa1230826a5a09659fbd Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:46:59 -0400 Subject: [PATCH 14/22] Extract duplicate code --- .../ripme/ripper/AbstractHTMLRipper.java | 77 ------------------ .../ripme/ripper/AbstractJSONRipper.java | 78 ------------------- .../ripme/ripper/AbstractRipper.java | 65 +++++++++++++--- .../rarchives/ripme/ripper/AlbumRipper.java | 78 ------------------- .../rarchives/ripme/ripper/VideoRipper.java | 70 +---------------- 5 files changed, 54 insertions(+), 314 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 0c0b72731..4bee19e12 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -32,9 +32,6 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AbstractHTMLRipper.class); - private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); - private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); Document cachedFirstPage; protected AbstractHTMLRipper(URL url) throws IOException { @@ -334,14 +331,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /* - Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - /* Queues multiple URLs of single images to download from a single Album URL */ @@ -414,72 +403,6 @@ protected boolean addURLToDownload(URL url) { return addURLToDownload(url, "", ""); } - /* - Cleans up & tells user about successful download - */ - @Override - public void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - /* - * Cleans up & tells user about failed download. - */ - @Override - public void downloadErrored(RipUrlId ripUrlId, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(ripUrlId); - itemsErrored.put(ripUrlId, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); - - checkIfComplete(); - } - - /* - Tells user that a single file in the album they wish to download has - already been downloaded in the past. - */ - @Override - public void downloadExists(RipUrlId ripUrlId, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - notifyComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 7910bae77..3935fcc63 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -28,10 +28,6 @@ public abstract class AbstractJSONRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AbstractJSONRipper.class); - private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); - private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); - protected AbstractJSONRipper(URL url) throws IOException { super(url); } @@ -137,14 +133,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /** - * Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - @Override /** * Queues multiple URLs of single images to download from a single Album URL @@ -217,72 +205,6 @@ protected boolean addURLToDownload(URL url) { return addURLToDownload(url, "", ""); } - /** - * Cleans up & tells user about successful download - */ - @Override - public void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - /** - * Cleans up & tells user about failed download. - */ - @Override - public void downloadErrored(RipUrlId ripUrlId, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(ripUrlId); - itemsErrored.put(ripUrlId, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); - - checkIfComplete(); - } - - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. - */ - @Override - public void downloadExists(RipUrlId ripUrlId, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - notifyComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 72dbdbb81..7e8638850 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -14,12 +14,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Observable; -import java.util.Random; -import java.util.Scanner; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.logging.log4j.LogManager; @@ -45,6 +40,11 @@ public abstract class AbstractRipper implements RipperInterface, Runnable { private static final Logger logger = LogManager.getLogger(AbstractRipper.class); + + protected final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); + protected final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + protected final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); + private final String URLHistoryFile = Utils.getURLHistoryFile(); public static final String USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"; @@ -532,27 +532,68 @@ public void retrievingSource(String url) { * @param ripUrlId The RipUrlId that was completed. * @param saveAs Where the downloaded file is stored. */ - public abstract void downloadCompleted(RipUrlId ripUrlId, Path saveAs); + protected void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { + if (observer == null) { + return; + } + try { + String path = Utils.removeCWD(saveAs); + RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); + itemsPending.remove(ripUrlId); + itemsCompleted.put(ripUrlId, saveAs); + observer.update(this, msg); + + checkIfComplete(); + } catch (Exception e) { + logger.error("Exception while updating observer: ", e); + } + } /** * Notifies observers that a file could not be downloaded (includes a reason). */ - public abstract void downloadErrored(RipUrlId ripUrlId, String reason); + protected void downloadErrored(RipUrlId ripUrlId, String reason) { + if (observer == null) { + return; + } + itemsPending.remove(ripUrlId); + itemsErrored.put(ripUrlId, reason); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); + + checkIfComplete(); + } /** * Notify observers that a download could not be completed, * but was not technically an "error". */ - public abstract void downloadExists(RipUrlId ripUrlId, Path file); + protected void downloadExists(RipUrlId ripUrlId, Path file) { + if (observer == null) { + return; + } + + itemsPending.remove(ripUrlId); + itemsCompleted.put(ripUrlId, file); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); + + checkIfComplete(); + } /** * @return Number of files downloaded. */ - int getCount() { - return 1; + public int getCount() { + return itemsCompleted.size() + itemsErrored.size(); } - protected abstract void checkIfComplete(); + /** + * Checks if complete and notifies observers if complete + */ + protected void checkIfComplete() { + if (itemsPending.isEmpty()) { + notifyComplete(); + } + } /** * Notifies observers and updates state if all files have been ripped. diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 1cbae7568..8b40a82bf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -30,10 +30,6 @@ public abstract class AlbumRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AlbumRipper.class); - private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); - private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); - protected AlbumRipper(URL url) throws IOException { super(url); } @@ -48,14 +44,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /** - * Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - @Override /** * Queues multiple URLs of single images to download from a single Album URL @@ -116,72 +104,6 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di return true; } - @Override - /** - * Cleans up & tells user about successful download - */ - public void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - @Override - /** - * Cleans up & tells user about failed download. - */ - public void downloadErrored(RipUrlId ripUrlId, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(ripUrlId); - itemsErrored.put(ripUrlId, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); - - checkIfComplete(); - } - - @Override - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. - */ - public void downloadExists(RipUrlId ripUrlId, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - notifyComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index d717ef851..5279c66f2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -89,6 +89,7 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di return true; } + itemsPending.add(ripUrlId); threadPool.addThread(new DownloadVideoThread(tug, ripUrlId, directory, filename, this)); } return true; @@ -128,61 +129,6 @@ public int getCompletionPercentage() { return (int) (100 * (bytesCompleted / (float) bytesTotal)); } - /** - * Runs if download successfully completed. - * - * @param ripUrlId Target URL ID - * @param saveAs Path to file, including filename. - */ - @Override - public void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { - if (observer == null) { - return; - } - - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - /** - * Runs if the download errored somewhere. - * - * @param ripUrlId Target URL ID - * @param reason Reason why the download failed. - */ - @Override - public void downloadErrored(RipUrlId ripUrlId, String reason) { - if (observer == null) { - return; - } - - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); - checkIfComplete(); - } - - /** - * Runs if user tries to redownload an already existing File. - * - * @param ripUrlId Target URL ID - * @param file Existing file - */ - @Override - public void downloadExists(RipUrlId ripUrlId, Path file) { - if (observer == null) { - return; - } - - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); - checkIfComplete(); - } - /** * Gets the status and changes it to a human-readable form. * @@ -206,18 +152,4 @@ public URL sanitizeURL(URL url) throws MalformedURLException { return url; } - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - - if (bytesCompleted >= bytesTotal) { - notifyComplete(); - } - } - } From bd8e9ab61651190dbfa54552e24810390b5790ad Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:57:08 -0400 Subject: [PATCH 15/22] Replace DownloadVideoThread with DownloadFileThread --- .../ripme/ripper/DownloadVideoThread.java | 206 ------------------ .../rarchives/ripme/ripper/VideoRipper.java | 7 +- 2 files changed, 6 insertions(+), 207 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java deleted file mode 100644 index 9a3f6f7cf..000000000 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ /dev/null @@ -1,206 +0,0 @@ -package com.rarchives.ripme.ripper; - -import java.io.*; -import java.net.HttpURLConnection; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; - -import javax.net.ssl.HttpsURLConnection; - -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Utils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jsoup.HttpStatusException; - -/** - * Thread for downloading files. - * Includes retry logic, observer notifications, and other goodies. - */ -class DownloadVideoThread implements Runnable { - - private static final Logger logger = LogManager.getLogger(DownloadVideoThread.class); - - private final TokenedUrlGetter tokenedUrlGetter; // Some URLs may be valid for a limited time. This should get a fresh url - private final RipUrlId ripUrlId; - private final Path directory; - private String filename; - private final AbstractRipper observer; - private final int retries; - - public DownloadVideoThread(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, AbstractRipper observer) { - super(); - this.tokenedUrlGetter = tug; - this.ripUrlId = ripUrlId; - this.directory = directory; - this.filename = filename; - this.observer = observer; - this.retries = Utils.getConfigInteger("download.retries", 1); - } - - /** - * Attempts to download the file. Retries as needed. - * Notifies observers upon completion/error/warn. - */ - @Override - public void run() { - if (observer.isStopped()) { - // TODO create status for gracefully-stopped download - observer.downloadErrored(ripUrlId, "Download interrupted"); - return; - } - URL url = null; - try { - url = tokenedUrlGetter.getTokenedUrl(); - } catch (HttpStatusException e) { - observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); - logger.error("[!] Failed to get URL for " + ripUrlId); - return; // do not retry - } catch (IOException | URISyntaxException e) { - logger.error("[!] Failed to get URL for " + ripUrlId, e); - observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); - return; // do not retry - } - if (filename == null) { - // Strip token query parameters - filename = Path.of(url.getPath()).getFileName().toString(); - } - if (AbstractRipper.shouldIgnoreExtension(url)) { - observer.sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return; - } - if (!Files.exists(directory)) { - logger.info("[+] Creating directory: " + directory); - try { - Files.createDirectories(directory); - } catch (IOException e) { - logger.error("Error creating directory", e); - observer.downloadErrored(ripUrlId, "Error creating directory: " + directory + " ; " + e.getMessage()); - return; - } - } - Path saveAs = directory.resolve(filename); - String prettySaveAs = Utils.removeCWD(saveAs); - - if (Files.exists(saveAs)) { - if (Utils.getConfigBoolean("file.overwrite", false)) { - logger.info("[!] Deleting existing file" + prettySaveAs); - try { - Files.delete(saveAs); - } catch (IOException e) { - e.printStackTrace(); - } - } else { - logger.info("[!] Skipping " + url + " -- file already exists: " + prettySaveAs); - observer.downloadExists(ripUrlId, saveAs); - return; - } - } - - int bytesTotal, bytesDownloaded = 0; - try { - bytesTotal = getTotalBytes(url); - } catch (IOException e) { - logger.error("Failed to get file size at " + url, e); - observer.downloadErrored(ripUrlId, "Failed to get file size of " + url); - return; - } - observer.setBytesTotal(bytesTotal); - observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.debug("Size of file at " + url + " = " + bytesTotal + "b"); - - int tries = 0; // Number of attempts to download - do { - InputStream bis = null; OutputStream fos = null; - byte[] data = new byte[1024 * 256]; - int bytesRead; - try { - logger.info(" Downloading file: " + url + (tries > 0 ? " Try #" + tries+1 : "")); - observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); - - // Setup HTTP request - HttpURLConnection huc; - if (url.getProtocol().equals("https")) { - huc = (HttpsURLConnection) url.openConnection(); - } - else { - huc = (HttpURLConnection) url.openConnection(); - } - huc.setInstanceFollowRedirects(true); - huc.setConnectTimeout(0); // Never timeout - huc.setRequestProperty("accept", "*/*"); - huc.setRequestProperty("Referer", url.toExternalForm()); // Sic - huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); - tries += 1; - logger.debug("Request properties: " + huc.getRequestProperties().toString()); - huc.connect(); - // Check status code - bis = new BufferedInputStream(huc.getInputStream()); - fos = Files.newOutputStream(saveAs); - while ( (bytesRead = bis.read(data)) != -1) { - if (observer.isPanicked()) { - observer.downloadErrored(ripUrlId, "Download interrupted"); - return; - } - fos.write(data, 0, bytesRead); - observer.sendUpdate(STATUS.CHUNK_BYTES, bytesRead); - bytesDownloaded += bytesRead; - observer.setBytesCompleted(bytesDownloaded); - observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); - } - bis.close(); - fos.close(); - break; // Download successful: break out of infinite loop - } catch (IOException e) { - logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage(), e); - } finally { - // Close any open streams - try { - if (bis != null) { bis.close(); } - } catch (IOException ignored) { } - try { - if (fos != null) { fos.close(); } - } catch (IOException ignored) { } - } - if (tries > this.retries) { - logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url); - observer.downloadErrored(ripUrlId, "Failed to download " + url.toExternalForm()); - return; - } - - // get fresh URL for the next attempt - try { - url = tokenedUrlGetter.getTokenedUrl(); - } catch (HttpStatusException e) { - observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); - logger.error("[!] Failed to get URL for " + ripUrlId); - return; // do not retry - } catch (IOException | URISyntaxException e) { - logger.error("[!] Failed to get URL for " + ripUrlId, e); - observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); - return; // do not retry - } - - } while (true); - observer.downloadCompleted(ripUrlId, saveAs); - logger.info("[+] Saved " + url + " as " + prettySaveAs); - } - - /** - * @param url - * Target URL - * @return - * Returns connection length - */ - private int getTotalBytes(URL url) throws IOException { - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setRequestMethod("HEAD"); - conn.setRequestProperty("accept", "*/*"); - conn.setRequestProperty("Referer", url.toExternalForm()); // Sic - conn.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); - return conn.getContentLength(); - } - -} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 5279c66f2..2852fd973 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -48,6 +48,11 @@ public String getAlbumTitle(URL url) { return "videos"; } + @Override + protected boolean useByteProgessBar() { + return true; + } + @Override public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { if (Utils.getConfigBoolean("urls_only.save", false)) { @@ -90,7 +95,7 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di } itemsPending.add(ripUrlId); - threadPool.addThread(new DownloadVideoThread(tug, ripUrlId, directory, filename, this)); + threadPool.addThread(new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME)); } return true; } From b945fe41f354ca4a8b7382702849caa2689e67be Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 14:40:23 -0400 Subject: [PATCH 16/22] Extract duplicate code --- .../ripme/ripper/AbstractHTMLRipper.java | 59 ---------------- .../ripme/ripper/AbstractJSONRipper.java | 59 ---------------- .../ripme/ripper/AbstractRipper.java | 67 ++++++++++++++++++- .../rarchives/ripme/ripper/AlbumRipper.java | 60 ----------------- .../rarchives/ripme/ripper/VideoRipper.java | 48 +------------ 5 files changed, 67 insertions(+), 226 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 4bee19e12..a29a318cb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -331,65 +331,6 @@ protected boolean allowDuplicates() { return false; } - /* - Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.contains(ripUrlId) - || itemsCompleted.containsKey(ripUrlId) - || itemsErrored.containsKey(ripUrlId) )) { - // Item is already downloaded/downloading, skip it. - // TODO print path if in itemsCompleted or itemsErrored - logger.info("[!] Skipping " + ripUrlId + " -- already attempted: " + Utils.removeCWD(directory)); - return false; - } - - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - URL url = null; - try { - url = tug.getTokenedUrl(); - } catch (IOException | URISyntaxException e) { - logger.error("Unable to get URL for {}", ripUrlId, e); - itemsErrored.put(ripUrlId, e.getMessage()); - return false; - } - if (AbstractRipper.shouldIgnoreExtension(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(ripUrlId, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - return true; - } - else { - itemsPending.add(ripUrlId); - DownloadFileThread dft = new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - /** * Queues image to be downloaded and saved. * Uses filename from URL to decide filename. diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 3935fcc63..fb31aaa55 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -133,65 +133,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /** - * Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.contains(ripUrlId) - || itemsCompleted.containsKey(ripUrlId) - || itemsErrored.containsKey(ripUrlId) )) { - // Item is already downloaded/downloading, skip it. - // TODO print path if in itemsCompleted or itemsErrored - logger.info("[!] Skipping " + ripUrlId + " -- already attempted: " + Utils.removeCWD(directory)); - return false; - } - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - URL url = null; - try { - url = tug.getTokenedUrl(); - } catch (IOException | URISyntaxException e) { - logger.error("Unable to get URL for {}", ripUrlId, e); - itemsErrored.put(ripUrlId, e.getMessage()); - return false; - } - if (AbstractRipper.shouldIgnoreExtension(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(ripUrlId, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - return true; - } - else { - itemsPending.add(ripUrlId); - DownloadFileThread dft = new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - /** * Queues image to be downloaded and saved. * Uses filename from URL to decide filename. diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 7e8638850..e089951fe 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -11,9 +11,11 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -64,6 +66,8 @@ public abstract class AbstractRipper public abstract String getGID(URL url) throws MalformedURLException, URISyntaxException; + protected abstract boolean allowDuplicates(); + public boolean hasASAPRipping() { return false; } @@ -271,7 +275,66 @@ protected boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path return addURLToDownload(tug, ripUrlId, directory, null, referrer, cookies, getFileExtFromMIME); } - public abstract boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME); + /** + * Queues multiple URLs of single images to download from a single Album URL + */ + public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { + // Only download one file if this is a test. + if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { + stop(); + itemsPending.clear(); + return false; + } + if (!allowDuplicates() + && ( itemsPending.contains(ripUrlId) + || itemsCompleted.containsKey(ripUrlId) + || itemsErrored.containsKey(ripUrlId) )) { + // Item is already downloaded/downloading, skip it. + // TODO print path if in itemsCompleted or itemsErrored + logger.info("[!] Skipping " + ripUrlId + " -- already attempted: " + Utils.removeCWD(directory)); + return false; + } + + if (Utils.getConfigBoolean("urls_only.save", false)) { + // Output URL to file + Path urlFile = Paths.get(this.workingDir + "/urls.txt"); + URL url = null; + try { + url = tug.getTokenedUrl(); + } catch (IOException | URISyntaxException e) { + logger.error("Unable to get URL for {}", ripUrlId, e); + itemsErrored.put(ripUrlId, e.getMessage()); + return false; + } + if (AbstractRipper.shouldIgnoreExtension(url)) { + sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + return false; + } + String text = url.toExternalForm() + System.lineSeparator(); + try { + Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); + itemsCompleted.put(ripUrlId, urlFile); + } catch (IOException e) { + logger.error("Error while writing to " + urlFile, e); + return false; + } + return true; + } + else { + itemsPending.add(ripUrlId); + DownloadFileThread dft = new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME); + if (referrer != null) { + dft.setReferrer(referrer); + } + if (cookies != null) { + dft.setCookies(cookies); + } + threadPool.addThread(dft); + } + + return true; + } + /** @@ -590,7 +653,7 @@ public int getCount() { * Checks if complete and notifies observers if complete */ protected void checkIfComplete() { - if (itemsPending.isEmpty()) { + if (itemsPending.isEmpty()) { // TODO add itemsActive for current transfers notifyComplete(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 8b40a82bf..1570e11b0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -44,66 +44,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /** - * Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.contains(ripUrlId) - || itemsCompleted.containsKey(ripUrlId) - || itemsErrored.containsKey(ripUrlId) )) { - // Item is already downloaded/downloading, skip it. - // TODO print path if in itemsCompleted or itemsErrored - logger.info("[!] Skipping " + ripUrlId + " -- already attempted: " + Utils.removeCWD(directory)); - return false; - } - - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - URL url = null; - try { - url = tug.getTokenedUrl(); - } catch (IOException | URISyntaxException e) { - logger.error("Unable to get URL for {}", ripUrlId, e); - itemsErrored.put(ripUrlId, e.getMessage()); - return false; - } - if (AbstractRipper.shouldIgnoreExtension(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(ripUrlId, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - return true; - } - else { - itemsPending.add(ripUrlId); - DownloadFileThread dft = new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - /** * Sets directory to save all ripped files to. * @param url diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 2852fd973..d2904feb2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -12,8 +12,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.rarchives.ripme.ui.RipStatusMessage; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; public abstract class VideoRipper extends AbstractRipper { @@ -54,50 +52,8 @@ protected boolean useByteProgessBar() { } @Override - public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - String urlFile = this.workingDir + "/urls.txt"; - URL url = null; - try { - url = tug.getTokenedUrl(); - } catch (IOException | URISyntaxException e) { - logger.error("Unable to get URL for {}", ripUrlId, e); - return false; - } - if (AbstractRipper.shouldIgnoreExtension(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - - try (FileWriter fw = new FileWriter(urlFile, true)) { - fw.write(url.toExternalForm()); - fw.write("\n"); - - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, urlFile); - observer.update(this, msg); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - return false; - } - return true; - } else { - if (isThisATest()) { - // Tests shouldn't download the whole video - // Just change this.url to the download URL so the test knows we found it. - logger.debug("Test rip, found URL: " + url); - try { - this.url = tug.getTokenedUrl(); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); - } - return true; - } - - itemsPending.add(ripUrlId); - threadPool.addThread(new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME)); - } - return true; + protected boolean allowDuplicates() { + return false; } /** From 652b4d55f7c7330499c7749efb74731d676bbc07 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 10 Aug 2026 05:56:06 +0200 Subject: [PATCH 17/22] Unify thread pools, fix race condition and dupe handling Each ripper previously maintained its own separate DownloadThreadPool for resolving direct image URLs from intermediate "detail" pages (one per ripper subclass), in addition to the shared download pool -- now unified into a shared crawler pool and ripper pool per rip. Also fixes a real race: if all downloads on a page were skipped, the old itemsPending.isEmpty()-based completion check could fire before the crawler pool had finished discovering and queuing every item (e.g. a later page's items not yet queued), letting the next ripper in the queue start early. Completion is now tracked via itemsSeen vs. itemsTotal counters instead, with the crawler pool required to fully drain before the ripper pool's completion is even checked. Bundled fixes for three deadlocks the itemsSeen/itemsTotal approach introduced: - downloadCompleted/downloadErrored/downloadExists/downloadSkipped gated their itemsPending bookkeeping behind an observer-null check meant only for the GUI notification. Neither the CLI (App.java) nor the test harness ever set an observer, so itemsPending never emptied outside the GUI -- every non-GUI rip hung forever. - The crawler-pool wait polled for getScheduledThreadCount() to reach the crawl loop's item count, which is wrong for any ripper whose downloadURL() doesn't submit a crawler-pool thread per item (e.g. DribbbleRipper, which calls addURLToDownload() directly) -- the count never reached the target. Removed the wait: crawler threads are always submitted synchronously before this call, so there was nothing left to wait for scheduling-wise. - itemsSeen was only incremented past the URL-history "already downloaded, skip" check, deep in the call chain. In test mode this never mattered (only one item is ever processed), but in real CLI usage -- re-ripping an album where some or all items were already downloaded -- every skipped item bypassed the increment, so itemsSeen could never reach itemsTotal. Moved the increment to the top of both outer addURLToDownload entry chains so every attempted item is counted exactly once regardless of which internal path handles or skips it. Co-Authored-By: Claude Sonnet 5 --- .../ripme/ripper/AbstractHTMLRipper.java | 29 ++-- .../ripme/ripper/AbstractJSONRipper.java | 26 ++-- .../ripme/ripper/AbstractRipper.java | 142 +++++++++++++----- .../ripme/ripper/DownloadThreadPool.java | 82 ++++++++-- .../ripper/rippers/DeviantartRipper.java | 9 +- .../ripme/ripper/rippers/E621Ripper.java | 10 +- .../ripme/ripper/rippers/EHentaiRipper.java | 10 +- .../ripme/ripper/rippers/FlickrRipper.java | 8 - .../ripper/rippers/FuraffinityRipper.java | 9 -- .../ripme/ripper/rippers/HqpornerRipper.java | 9 +- .../ripme/ripper/rippers/ImagebamRipper.java | 10 +- .../ripper/rippers/ImagevenueRipper.java | 10 +- .../ripme/ripper/rippers/ImgurRipper.java | 2 +- .../ripme/ripper/rippers/ListalRipper.java | 10 +- .../ripper/rippers/MotherlessRipper.java | 10 +- .../ripme/ripper/rippers/NfsfwRipper.java | 12 +- .../ripme/ripper/rippers/NhentaiRipper.java | 8 - .../ripme/ripper/rippers/PornhubRipper.java | 11 +- .../ripme/ripper/rippers/RedditRipper.java | 2 +- .../ripme/ripper/rippers/TumblrRipper.java | 2 +- .../ripme/ripper/rippers/VkRipper.java | 2 +- .../rippers/video/CliphunterRipper.java | 2 +- .../rippers/video/MotherlessVideoRipper.java | 2 +- .../ripper/rippers/video/PornhubRipper.java | 2 +- .../rippers/video/TwitchVideoRipper.java | 2 +- .../ripper/rippers/video/ViddmeRipper.java | 2 +- .../ripper/rippers/video/VidearnRipper.java | 2 +- .../ripme/ripper/rippers/video/VkRipper.java | 2 +- .../ripper/rippers/video/YuvutuRipper.java | 2 +- .../com/rarchives/ripme/ui/MainWindow.java | 31 ++-- 30 files changed, 247 insertions(+), 213 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index a29a318cb..2ba66554e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -69,10 +69,6 @@ protected List getDescriptionsFromPage(Document doc) throws IOException protected abstract void downloadURL(URL url, int index); - protected DownloadThreadPool getThreadPool() { - return null; - } - protected boolean keepSortOrder() { return true; } @@ -114,7 +110,7 @@ protected boolean pageContainsAlbums(URL url) { @Override public void rip() throws IOException, URISyntaxException { - int index = 0; + int imageIndex = 0; int textindex = 0; logger.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); @@ -169,9 +165,9 @@ public void rip() throws IOException, URISyntaxException { } for (String imageURL : imageURLs) { - index += 1; - logger.debug("Found image url #" + index + ": '" + imageURL + "'"); - downloadURL(new URI(imageURL).toURL(), index); + imageIndex += 1; + logger.debug("Found image url #" + imageIndex + ": '" + imageURL + "'"); + downloadURL(new URI(imageURL).toURL(), imageIndex); if (isStopped() || isThisATest()) { break; } @@ -199,7 +195,7 @@ public void rip() throws IOException, URISyntaxException { workingDir.getCanonicalPath() + "" + File.separator - + getPrefix(index) + + getPrefix(imageIndex) + (tempDesc.length > 1 ? tempDesc[1] : filename) + ".txt").exists(); @@ -229,12 +225,17 @@ public void rip() throws IOException, URISyntaxException { } } - // If they're using a thread pool, wait for it. - if (getThreadPool() != null) { - logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); - getThreadPool().waitForThreads(); + logger.info("All items queued; total items: {}; url: {}", imageIndex, url); + + // Final total item count is now known + setItemsTotal(imageIndex); + + if (getCrawlerThreadPool() != null) { + logger.debug("Waiting for crawler threadpool: {}", url); + getCrawlerThreadPool().waitForThreads(imageIndex, shouldStop, url); } - waitForThreads(); + + waitForRipperThreads(); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index fb31aaa55..5ed3c637c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -42,9 +42,6 @@ protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxEx } protected abstract List getURLsFromJSON(JSONObject json); protected abstract void downloadURL(URL url, int index); - private DownloadThreadPool getThreadPool() { - return null; - } protected boolean keepSortOrder() { return true; @@ -62,7 +59,7 @@ public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException @Override public void rip() throws IOException, URISyntaxException { - int index = 0; + int imageIndex = 0; logger.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); JSONObject json = getFirstPage(); @@ -91,9 +88,9 @@ public void rip() throws IOException, URISyntaxException { break; } - index += 1; - logger.debug("Found image url #" + index+ ": " + imageURL); - downloadURL(new URI(imageURL).toURL(), index); + imageIndex += 1; + logger.debug("Found image url #" + imageIndex+ ": " + imageURL); + downloadURL(new URI(imageURL).toURL(), imageIndex); } if (isStopped() || isThisATest()) { @@ -109,12 +106,17 @@ public void rip() throws IOException, URISyntaxException { } } - // If they're using a thread pool, wait for it. - if (getThreadPool() != null) { - logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); - getThreadPool().waitForThreads(); + logger.info("All items queued; total items: {}; url: {}", imageIndex, url); + + // Final total item count is now known + setItemsTotal(imageIndex); + + if (getCrawlerThreadPool() != null) { + logger.debug("Waiting for crawler threadpool: {}", url); + getCrawlerThreadPool().waitForThreads(imageIndex, shouldStop, url); } - waitForThreads(); + + waitForRipperThreads(); } protected String getPrefix(int index) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index e089951fe..db27dc1a7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -18,6 +18,7 @@ import java.nio.file.StandardOpenOption; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -46,6 +47,23 @@ public abstract class AbstractRipper protected final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); protected final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); protected final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); + protected final Map itemsSkipped = Collections.synchronizedMap(new HashMap<>()); + + /** + * Rippers should set itemsTotal to the best known number of total items, + * if known at the start of the rip, e.g. in getFirstPage(). + * The best known number might be indicated on the album page, + * or calculated by the number of pages and the number of items per page. + * Once the last item is seen by the HTML or JSON crawler, the final value is set. + */ + private final AtomicInteger itemsTotal = new AtomicInteger(0); + + /** + * If an album has a duplicate RipUrlId (e.g. the same image linked twice), + * duplicates can't be counted by itemsPending, but {@link #waitForRipperThreads()} needs + * to know that the ripper has seen each link crawled. + */ + private final AtomicInteger itemsSeen = new AtomicInteger(0); private final String URLHistoryFile = Utils.getURLHistoryFile(); @@ -55,7 +73,8 @@ public abstract class AbstractRipper protected URL url; protected File workingDir; - DownloadThreadPool threadPool; + private DownloadThreadPool ripperThreadPool; + private DownloadThreadPool crawlerThreadPool; RipStatusHandler observer = null; private final AtomicBoolean completed = new AtomicBoolean(false); @@ -74,8 +93,8 @@ public boolean hasASAPRipping() { // Everytime addUrlToDownload skips a already downloaded url this increases by 1 public int alreadyDownloadedUrls = 0; - private final AtomicBoolean shouldStop = new AtomicBoolean(false); - private final AtomicBoolean shouldPanic = new AtomicBoolean(false); + protected final AtomicBoolean shouldStop = new AtomicBoolean(false); + protected final AtomicBoolean shouldPanic = new AtomicBoolean(false); private static boolean thisIsATest = false; public void stop() { @@ -97,16 +116,29 @@ public boolean isStopped() { return shouldStop.get(); } - public boolean isCompleted() { - return completed.get(); - } - protected void stopCheck() throws IOException { if (shouldStop.get()) { throw new IOException("Ripping interrupted"); } } + /** + * Used for file downloads. Used by {@link #addURLToDownload(TokenedUrlGetter, RipUrlId, Path, String, String, Map, Boolean)} + */ + protected DownloadThreadPool getRipperThreadPool() { + return ripperThreadPool; + } + + /** + * Used by Rippers to crawl file pages.
+ * After the last file page is crawled and all threads are queued to the ripper thread pool, + * {@link #rip()} terminates the crawler thread pool.
+ * After the crawler thread pool is finished, {@link #rip()} terminates the ripper thread pool. + */ + protected DownloadThreadPool getCrawlerThreadPool() { + return crawlerThreadPool; + } + /** * Adds a URL to the url history file * @@ -234,7 +266,8 @@ public void setup() throws IOException, URISyntaxException { // ctx.reconfigure(); // ctx.updateLoggers(); - this.threadPool = new DownloadThreadPool(); + this.ripperThreadPool = new DownloadThreadPool(getClass().getSimpleName() + "-ripper-" + getGID(url)); + this.crawlerThreadPool = new DownloadThreadPool(getClass().getSimpleName() + "-crawler"); } public void setObserver(RipStatusHandler obs) { @@ -264,6 +297,7 @@ public boolean addURLToDownload(URL url, Path saveAs) { * False if failed to download */ public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + itemsSeen.incrementAndGet(); TokenedUrlGetter tug = () -> url; RipUrlId ripUrlId = new RipUrlId(getClass(), getHost(), url); Path directory = saveAs.getParent(); @@ -285,6 +319,7 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di itemsPending.clear(); return false; } + if (!allowDuplicates() && ( itemsPending.contains(ripUrlId) || itemsCompleted.containsKey(ripUrlId) @@ -329,7 +364,7 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di if (cookies != null) { dft.setCookies(cookies); } - threadPool.addThread(dft); + getRipperThreadPool().addThread(dft); } return true; @@ -393,6 +428,7 @@ protected boolean addURLToDownload(URL url, Map options) { */ protected boolean addURLToDownload(URL url, String subdirectory, String referrer, Map cookies, String prefix, String fileName, String extension, Boolean getFileExtFromMIME) { + itemsSeen.incrementAndGet(); // A common bug is rippers adding urls that are just "http:". // This rejects said urls. if (url.toExternalForm().equals("http:") || url.toExternalForm().equals("https:")) { @@ -570,11 +606,24 @@ public static String getFileName(URL url, String prefix, String fileName, String /** * Waits for downloading threads to complete. */ - protected void waitForThreads() { - logger.debug("Waiting for threads to finish"); - completed.set(false); - threadPool.waitForThreads(); - checkIfComplete(); + protected void waitForRipperThreads() { + waitForRipperThreads(true); + } + + protected void waitForRipperThreads(boolean notifyComplete) { + logger.debug("Waiting for threads to finish; url: {}", url); + if (!notifyComplete) { + setItemsTotal(0); + } + ripperThreadPool.waitForThreads(() -> { + boolean finished = shouldStop.get() || (itemsSeen.get() >= itemsTotal.get() && itemsPending.isEmpty()); + logger.trace("ripperThreadPool: are threads finished? {} url: {} shouldStop: {}; itemsPending.size(): {}; itemsCompleted.size(): {}; itemsErrored.size(): {}; itemsSkipped.size(): {}; itemsTotal: {}; itemsSeen: {}", + finished, url, shouldStop, itemsPending.size(), itemsCompleted.size(), itemsErrored.size(), itemsSkipped.size(), itemsTotal, itemsSeen); + return finished; + }, url); + if (notifyComplete) { + notifyComplete(); + } } /** @@ -596,17 +645,17 @@ public void retrievingSource(String url) { * @param saveAs Where the downloaded file is stored. */ protected void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { + itemsPending.remove(ripUrlId); + itemsCompleted.put(ripUrlId, saveAs); if (observer == null) { return; } try { String path = Utils.removeCWD(saveAs); RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, saveAs); observer.update(this, msg); - checkIfComplete(); + //checkIfComplete(); } catch (Exception e) { logger.error("Exception while updating observer: ", e); } @@ -616,14 +665,29 @@ protected void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { * Notifies observers that a file could not be downloaded (includes a reason). */ protected void downloadErrored(RipUrlId ripUrlId, String reason) { + itemsPending.remove(ripUrlId); + itemsErrored.put(ripUrlId, reason); if (observer == null) { return; } - itemsPending.remove(ripUrlId); - itemsErrored.put(ripUrlId, reason); observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); - checkIfComplete(); + //checkIfComplete(); + } + + /** + * Notifies observers that a file could not be downloaded (includes a reason). + */ + protected void downloadSkipped(RipUrlId ripUrlId, String reason) { + itemsPending.remove(ripUrlId); + //itemsSkipped.put(ripUrlId, reason); + itemsCompleted.put(ripUrlId, null); // TODO use itemsSkipped and make the progress bar display it as completed + if (observer == null) { + return; + } + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_SKIP, ripUrlId + " : " + reason)); + + //checkIfComplete(); } /** @@ -631,15 +695,15 @@ protected void downloadErrored(RipUrlId ripUrlId, String reason) { * but was not technically an "error". */ protected void downloadExists(RipUrlId ripUrlId, Path file) { + itemsPending.remove(ripUrlId); + itemsCompleted.put(ripUrlId, file); if (observer == null) { return; } - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, file); observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); - checkIfComplete(); + //checkIfComplete(); } /** @@ -649,15 +713,6 @@ public int getCount() { return itemsCompleted.size() + itemsErrored.size(); } - /** - * Checks if complete and notifies observers if complete - */ - protected void checkIfComplete() { - if (itemsPending.isEmpty()) { // TODO add itemsActive for current transfers - notifyComplete(); - } - } - /** * Notifies observers and updates state if all files have been ripped. */ @@ -672,6 +727,7 @@ protected void notifyComplete() { RipStatusComplete rsc = new RipStatusComplete(workingDir.toPath(), getCount()); RipStatusMessage msg = new RipStatusMessage(STATUS.RIP_COMPLETE, rsc); + logger.debug("Sending RIP_COMPLETE: url: {}", getURL()); observer.update(this, msg); // we do not care if the rollingfileappender is active, @@ -805,11 +861,11 @@ public void run() { rip(); } catch (HttpStatusException e) { logger.error("Got exception while running ripper:", e); - waitForThreads(); + waitForRipperThreads(false); sendUpdate(STATUS.RIP_ERRORED, "HTTP status code " + e.getStatusCode() + " for URL " + e.getUrl()); } catch (Exception e) { logger.error("Got exception while running ripper:", e); - waitForThreads(); + waitForRipperThreads(false); sendUpdate(STATUS.RIP_ERRORED, e.getMessage()); } finally { cleanup(); @@ -938,4 +994,22 @@ protected static boolean shouldIgnoreExtension(URL url) { } return false; } + + /** + * Gets the asserted number of total items, or 0 if unknown. + * Possibly useful in rippers. + */ + protected int getItemsTotal() { + return itemsTotal.get(); + } + + /** + * For use in rippers to update the best estimate of total items. + */ + protected void setItemsTotal(int itemsTotal) { + if (itemsTotal < 0) { + throw new IllegalArgumentException("itemsTotal cannot be negative. Use 0 for unknown."); + } + this.itemsTotal.set(itemsTotal); + } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index 8ae43743f..cc0198a9f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -1,8 +1,12 @@ package com.rarchives.ripme.ripper; +import java.net.URL; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import com.rarchives.ripme.utils.Utils; import org.apache.logging.log4j.LogManager; @@ -15,42 +19,92 @@ public class DownloadThreadPool { private static final Logger logger = LogManager.getLogger(DownloadThreadPool.class); private ThreadPoolExecutor threadPool = null; - - public DownloadThreadPool() { - initialize("Main"); - } + private final AtomicLong scheduledThreadCount = new AtomicLong(0); + private final String name; public DownloadThreadPool(String threadPoolName) { - initialize(threadPoolName); - } - - /** - * Initializes the threadpool. - * @param threadPoolName Name of the threadpool. - */ - private void initialize(String threadPoolName) { int threads = Utils.getConfigInteger("threads.size", 10); logger.debug("Initializing " + threadPoolName + " thread pool with " + threads + " threads"); - threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads); + this.name = threadPoolName; + this.threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads); } + /** * For adding threads to execution pool. * @param t * Thread to be added. */ public void addThread(Runnable t) { + logger.trace("addThread called; name: {}, scheduledThreadCount: {}", name, scheduledThreadCount); + scheduledThreadCount.incrementAndGet(); threadPool.execute(t); } /** * Tries to shutdown threadpool. */ - public void waitForThreads() { + public void waitForThreads(Supplier isFinishedQueueing, URL url) { + logger.trace("waitForThreads called; name: {}; url: {}", name, url); + while (!isFinishedQueueing.get()) { + logger.trace("waiting for items to finish queueing; name: {}; url: {}", name, url); + try { + Thread.sleep(200); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.trace("sleep interrupted; name: {}; url: {}", name, url); + break; + } + } + + logger.trace("about to shutdown thread pool. name: {}; url: {}", name, url); threadPool.shutdown(); + logger.trace("thread pool shutdown. name: {}; url: {}", name, url); try { threadPool.awaitTermination(3600, TimeUnit.SECONDS); + logger.trace("thread pool terminated. name: {}; url: {}", name, url); } catch (InterruptedException e) { logger.error("[!] Interrupted while waiting for threads to finish: ", e); } } + + /** + * Tries to shutdown threadpool. + * + * expectedScheduledThreads/shouldStop are unused: crawler threads are always + * submitted synchronously in the same thread, before this is called, so + * there is nothing left to wait for them to be scheduled. Polling for + * getScheduledThreadCount() to reach expectedScheduledThreads hung forever + * for any ripper whose downloadURL() doesn't submit a crawler-pool thread + * per item (e.g. rippers that call addURLToDownload() directly), since the + * count would then never reach the expected value. + */ + public void waitForThreads(int expectedScheduledThreads, AtomicBoolean shouldStop, URL url) { + logger.trace("waitForThreads called; name: {}; url: {}", name, url); + logger.trace("about to shutdown thread pool. name: {}; url: {}", name, url); + threadPool.shutdown(); + logger.trace("thread pool shutdown. name: {}; url: {}", name, url); + try { + threadPool.awaitTermination(3600, TimeUnit.SECONDS); + logger.trace("thread pool terminated. name: {}; url: {}", name, url); + } catch (InterruptedException e) { + logger.error("[!] Interrupted while waiting for threads to finish: ", e); + } + } + + public int getPendingThreadCount() { + return threadPool.getQueue().size(); + } + + public int getActiveThreadCount() { + return threadPool.getActiveCount(); + } + + public long getCompletedThreadCount() { + return threadPool.getCompletedTaskCount(); + } + + public long getScheduledThreadCount() { + //return threadPool.getTaskCount(); // approximate, bad + return scheduledThreadCount.get(); + } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 98510250c..4bb197e77 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -30,7 +30,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -94,7 +93,6 @@ public class DeviantartRipper extends AbstractHTMLRipper { private boolean usingCatPath = false; private int downloadCount = 0; private Map cookies = new HashMap(); - private DownloadThreadPool deviantartThreadPool = new DownloadThreadPool("deviantart"); private ArrayList names = new ArrayList(); List allowedCookies = Arrays.asList("agegate_state", "userinfo", "auth", "auth_secure"); @@ -106,11 +104,6 @@ public class DeviantartRipper extends AbstractHTMLRipper { private final String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0"; private final String utilsKey = "DeviantartLogin.cookies"; //for config file - @Override - public DownloadThreadPool getThreadPool() { - return deviantartThreadPool; - } - public DeviantartRipper(URL url) throws IOException { super(url); } @@ -304,7 +297,7 @@ protected void downloadURL(URL url, int index) { // Start Thread and add to pool. DeviantartImageThread t = new DeviantartImageThread(url); - deviantartThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index 9b40f0542..d0fbbbb9e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -19,7 +19,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; @@ -37,8 +36,6 @@ public class E621Ripper extends AbstractHTMLRipper { private static Pattern gidPatternNew = null; private static Pattern gidPatternPoolNew = null; - private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621"); - private Map cookies = new HashMap(); private String userAgent = USER_AGENT; @@ -78,11 +75,6 @@ private Document getDocument(String url) throws IOException { return getDocument(url, 1); } - @Override - public DownloadThreadPool getThreadPool() { - return e621ThreadPool; - } - @Override public String getDomain() { return "e621.net"; @@ -136,7 +128,7 @@ public void downloadURL(final URL url, int index) { // rate limit sleep(3000); // addURLToDownload(url, getPrefix(index)); - e621ThreadPool.addThread(new E621FileThread(url, getPrefix(index))); + getCrawlerThreadPool().addThread(new E621FileThread(url, getPrefix(index))); } private String getTerm(URL url) throws MalformedURLException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index 5349f55c7..d58bcc4c9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -21,7 +21,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; @@ -44,8 +43,6 @@ public class EHentaiRipper extends AbstractHTMLRipper { } private String lastURL = null; - // Thread pool for finding direct image links from "image" pages (html) - private final DownloadThreadPool ehentaiThreadPool = new DownloadThreadPool("ehentai"); // Current HTML document private Document albumDoc = null; @@ -53,11 +50,6 @@ public EHentaiRipper(URL url) throws IOException { super(url); } - @Override - public DownloadThreadPool getThreadPool() { - return ehentaiThreadPool; - } - @Override public String getHost() { return "e-hentai"; @@ -194,7 +186,7 @@ public List getURLsFromPage(Document page) { @Override public void downloadURL(URL url, int index) { EHentaiImageThread t = new EHentaiImageThread(url, index, this.workingDir.toPath()); - ehentaiThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 1f1954207..37acde4ac 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -28,8 +28,6 @@ public class FlickrRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(FlickrRipper.class); - private final DownloadThreadPool flickrThreadPool; - private enum UrlType { USER, PHOTOSET @@ -45,11 +43,6 @@ private class Album { } } - @Override - public DownloadThreadPool getThreadPool() { - return flickrThreadPool; - } - @Override public boolean hasASAPRipping() { return true; @@ -57,7 +50,6 @@ public boolean hasASAPRipping() { public FlickrRipper(URL url) throws IOException { super(url); - flickrThreadPool = new DownloadThreadPool(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index ad9075d0d..cb7c49479 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -54,15 +54,6 @@ private void warnAboutSharedAccount(String loginCookies) { } } - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool furaffinityThreadPool - = new DownloadThreadPool( "furaffinity"); - - @Override - public DownloadThreadPool getThreadPool() { - return furaffinityThreadPool; - } - public FuraffinityRipper(URL url) throws IOException { super(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java index 7183f1d79..54a98f306 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java @@ -18,7 +18,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; public class HqpornerRipper extends AbstractHTMLRipper { @@ -30,7 +29,6 @@ public class HqpornerRipper extends AbstractHTMLRipper { private Pattern p1 = Pattern.compile("https?://hqporner.com/hdporn/([a-zA-Z0-9_-]*).html/?$"); // video pattern. private Pattern p2 = Pattern.compile("https://hqporner.com/([a-zA-Z0-9/_-]+)"); // category/top/actress/studio pattern. private Pattern p3 = Pattern.compile("https?://[A-Za-z0-9/.-_]+\\.mp4"); // to match links ending with .mp4 - private DownloadThreadPool hqpornerThreadPool = new DownloadThreadPool("hqpornerThreadPool"); private String subdirectory = ""; public HqpornerRipper(URL url) throws IOException { @@ -111,7 +109,7 @@ public boolean tryResumeDownload() { @Override public void downloadURL(URL url, int index) { - hqpornerThreadPool.addThread(new HqpornerDownloadThread(url, index, subdirectory)); + getCrawlerThreadPool().addThread(new HqpornerDownloadThread(url, index, subdirectory)); } @Override @@ -123,11 +121,6 @@ public Document getNextPage(Document doc) throws IOException { throw new IOException("No next page found."); } - @Override - public DownloadThreadPool getThreadPool() { - return hqpornerThreadPool; - } - @Override public boolean useByteProgessBar() { return true; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 7f58e1a24..ff2c9636b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -1,7 +1,6 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; import java.io.IOException; @@ -29,13 +28,6 @@ public class ImagebamRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(ImagebamRipper.class); - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool imagebamThreadPool = new DownloadThreadPool("imagebam"); - @Override - public DownloadThreadPool getThreadPool() { - return imagebamThreadPool; - } - public ImagebamRipper(URL url) throws IOException { super(url); } @@ -90,7 +82,7 @@ public List getURLsFromPage(Document doc) { @Override public void downloadURL(URL url, int index) { ImagebamImageThread t = new ImagebamImageThread(url, index); - imagebamThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); sleep(500); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java index 5df8e9f17..3de62ad50 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java @@ -17,7 +17,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -25,13 +24,6 @@ public class ImagevenueRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(ImagevenueRipper.class); - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool imagevenueThreadPool = new DownloadThreadPool("imagevenue"); - @Override - public DownloadThreadPool getThreadPool() { - return imagevenueThreadPool; - } - public ImagevenueRipper(URL url) throws IOException { super(url); } @@ -72,7 +64,7 @@ public List getURLsFromPage(Document doc) { public void downloadURL(URL url, int index) { ImagevenueImageThread t = new ImagevenueImageThread(url, index); - imagevenueThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 5bb7b0020..afbab931e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -205,7 +205,7 @@ public void rip() throws IOException { } catch (URISyntaxException e) { throw new IOException("Failed ripping " + this.url, e); } - waitForThreads(); + waitForRipperThreads(); } private void ripSingleImage(URL url) throws IOException, URISyntaxException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java index 7157e49dd..2cf37ef6e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java @@ -19,7 +19,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; /** @@ -38,8 +37,6 @@ public class ListalRipper extends AbstractHTMLRipper { private String listId = null; // listId to get more images via POST. private UrlType urlType = UrlType.UNKNOWN; - private DownloadThreadPool listalThreadPool = new DownloadThreadPool("listalThreadPool"); - public ListalRipper(URL url) throws IOException { super(url); } @@ -77,7 +74,7 @@ public List getURLsFromPage(Document page) { @Override public void downloadURL(URL url, int index) { - listalThreadPool.addThread(new ListalImageDownloadThread(url, index)); + getCrawlerThreadPool().addThread(new ListalImageDownloadThread(url, index)); } @Override @@ -137,11 +134,6 @@ public Document getNextPage(Document page) throws IOException, URISyntaxExceptio } - @Override - public DownloadThreadPool getThreadPool() { - return listalThreadPool; - } - /** * Returns the image urls for UrlType LIST. */ diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index 955e85a34..dc1fa3f67 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -31,11 +31,8 @@ public class MotherlessRipper extends AbstractHTMLRipper { private static final String DOMAIN = "motherless.com", HOST = "motherless"; - private DownloadThreadPool motherlessThreadPool; - public MotherlessRipper(URL url) throws IOException { super(url); - motherlessThreadPool = new DownloadThreadPool(); } @Override @@ -117,7 +114,7 @@ protected List getURLsFromPage(Document page) { protected void downloadURL(URL url, int index) { // Create thread for finding image at "url" page MotherlessImageRunnable mit = new MotherlessImageRunnable(url, index); - motherlessThreadPool.addThread(mit); + getCrawlerThreadPool().addThread(mit); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { @@ -155,11 +152,6 @@ public String getGID(URL url) throws MalformedURLException { throw new MalformedURLException("Expected URL format: https://motherless.com/GIXXXXXXX, got: " + url); } - @Override - protected DownloadThreadPool getThreadPool() { - return motherlessThreadPool; - } - /** * Helper class to find and download images found on "image" pages */ diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java index d6b17b02f..e69dbed81 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -17,7 +17,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; public class NfsfwRipper extends AbstractHTMLRipper { @@ -34,12 +33,8 @@ public class NfsfwRipper extends AbstractHTMLRipper { "https?://[wm.]*nfsfw.com/gallery/v/[^/]+/(.+)$" ); - // threads pool for downloading images from image pages - private DownloadThreadPool nfsfwThreadPool; - public NfsfwRipper(URL url) throws IOException { super(url); - nfsfwThreadPool = new DownloadThreadPool("NFSFW"); } @Override @@ -105,7 +100,7 @@ protected void downloadURL(URL url, int index) { index = ++this.index; } NfsfwImageThread t = new NfsfwImageThread(url, currentDir, index); - nfsfwThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); } @Override @@ -141,11 +136,6 @@ public String getGID(URL url) throws MalformedURLException { + " Got: " + url); } - @Override - public DownloadThreadPool getThreadPool() { - return nfsfwThreadPool; - } - @Override public boolean hasQueueSupport() { return true; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java index 41693a3ee..13df2aa74 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java @@ -27,9 +27,6 @@ public class NhentaiRipper extends AbstractHTMLRipper { private Document firstPage; - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool nhentaiThreadPool = new DownloadThreadPool("nhentai"); - @Override public boolean hasQueueSupport() { return true; @@ -51,11 +48,6 @@ public List getAlbumsToQueue(Document doc) { return urlsToAddToQueue; } - @Override - public DownloadThreadPool getThreadPool() { - return nhentaiThreadPool; - } - public NhentaiRipper(URL url) throws IOException { super(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java index 481ab1ede..c6279c3e0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java @@ -18,7 +18,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -31,9 +30,6 @@ public class PornhubRipper extends AbstractHTMLRipper { private static final String DOMAIN = "pornhub.com", HOST = "Pornhub"; - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool pornhubThreadPool = new DownloadThreadPool("pornhub"); - public PornhubRipper(URL url) throws IOException { super(url); } @@ -82,7 +78,7 @@ protected List getURLsFromPage(Document page) { @Override protected void downloadURL(URL url, int index) { PornhubImageThread t = new PornhubImageThread(url, index, this.workingDir.toPath()); - pornhubThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { @@ -119,11 +115,6 @@ public String getGID(URL url) throws MalformedURLException { + " Got: " + url); } - @Override - public DownloadThreadPool getThreadPool(){ - return pornhubThreadPool; - } - public boolean canRip(URL url) { return url.getHost().endsWith(DOMAIN) && url.getPath().startsWith("/album"); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 2419cd035..cb6b8eeb7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -114,7 +114,7 @@ public void rip() throws IOException { } catch (URISyntaxException e) { new IOException(e.getMessage()); } - waitForThreads(); + waitForRipperThreads(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 898a0ec62..56b695c66 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -236,7 +236,7 @@ public void rip() throws IOException { } } - waitForThreads(); + waitForRipperThreads(); } private boolean handleJSON(JSONObject json) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java index 59f79aab7..e36d22ff6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java @@ -158,7 +158,7 @@ public void rip() throws IOException, URISyntaxException { for (int index = 0; index < URLs.size(); index ++) { downloadURL(new URI(URLs.get(index)).toURL(), index); } - waitForThreads(); + waitForRipperThreads(); } else { RIP_TYPE = RipType.IMAGE; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java index 1e7a48f8e..76527ceea 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java @@ -78,6 +78,6 @@ public void rip() throws IOException, URISyntaxException { } } addURLToDownload(new URI(vidURL).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java index 0f95aaafc..c6cf3f123 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java @@ -70,6 +70,6 @@ public void rip() throws IOException, URISyntaxException { } String vidUrl = vidUrls.get(0); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java index 1857c309d..b7ed4ee85 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java @@ -154,6 +154,6 @@ public void rip() throws IOException, URISyntaxException { } addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + bestQuality + "p_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java index 076e90ca6..f6249e9d0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java @@ -75,6 +75,6 @@ public void rip() throws IOException, URISyntaxException { addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + title); } } - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java index a2cff267d..31c127d39 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java @@ -68,6 +68,6 @@ public void rip() throws IOException, URISyntaxException { String vidUrl = videos.first().attr("content"); vidUrl = vidUrl.replaceAll("&", "&"); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java index 00e77c427..4829cff86 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java @@ -63,6 +63,6 @@ public void rip() throws IOException, URISyntaxException { } String vidUrl = mp4s.get(0); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java index 4a7ea8ccd..1ab5c1ef4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java @@ -61,7 +61,7 @@ public void rip() throws IOException, URISyntaxException { logger.info(" Retrieving " + this.url); String videoURL = getVideoURLAtPage(this.url.toExternalForm()); addURLToDownload(new URI(videoURL).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } public static String getVideoURLAtPage(String url) throws IOException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java index 2fe0291e6..61762f3bb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java @@ -77,6 +77,6 @@ public void rip() throws IOException, URISyntaxException { addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); } } - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index ace2baf2a..528b854a9 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -137,6 +137,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static final AtomicBoolean gracefulStop = new AtomicBoolean(false); // Allow active transfers to finish, then stop ripping. private static final AtomicBoolean panicStop = new AtomicBoolean(false); // Immediately stop active transfers, then stop ripping. + private static final AtomicBoolean isRipperActive = new AtomicBoolean(false); public static final int TRANSFER_RATE_REFRESH_RATE = 200; private static final TransferRate transferRate = new TransferRate(); @@ -144,7 +145,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static final ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); private Future rateRefresherFuture = null; private final Runnable rateRefresher = () -> { - if (isHalted()) { + if (!isRipperActive.get()) { if (rateRefresherFuture != null) { rateRefresherFuture.cancel(true); rateRefresherFuture = null; @@ -155,13 +156,6 @@ public final class MainWindow implements Runnable, RipStatusHandler { transferRateLabel.setText(transferRate.formatHumanTransferRate()); }; - /** - * @return true if fully halted/panic button pressed - */ - public static boolean isHalted() { - return ripper == null || ripper.isPanicked() || ripper.isCompleted(); - } - private void updateQueue(DefaultListModel model) { if (model == null) model = queueListModel; @@ -1402,6 +1396,13 @@ private void saveHistory() { } private void ripNextAlbum() { + LOGGER.debug("ripNextAlbum called"); + if (isRipperActive.getAndSet(true)) { + // Already ripping + LOGGER.debug("already ripping"); + return; + } + // Save current state of queue to configuration. Utils.setConfigList("queue", queueListModel.elements()); @@ -1409,6 +1410,7 @@ private void ripNextAlbum() { boolean wasPanicStop = gracefulStop.getAndSet(false); if (wasGracefulStop || wasPanicStop) { // Stop requested + LOGGER.debug("wasGracefulStop or wasPanicStop"); ripFinishCleanup(); return; } @@ -1427,16 +1429,20 @@ private void ripNextAlbum() { updateQueue(); + LOGGER.debug("calling ripAlbum(\"{}\")", nextAlbum); Thread t = ripAlbum(nextAlbum); if (t == null) { + LOGGER.debug("ripAlbum() returned null"); try { Thread.sleep(500); } catch (InterruptedException ie) { LOGGER.error(Utils.getLocalizedString("interrupted.while.waiting.to.rip.next.album"), ie); } + isRipperActive.set(false); ripNextAlbum(); } else { + LOGGER.debug("Starting new ripper thread"); t.start(); } } @@ -1444,6 +1450,7 @@ private void ripNextAlbum() { private void ripFinishCleanup() { stopButton.setEnabled(false); panicButton.setEnabled(false); + isRipperActive.set(false); statusProgress.setValue(0); statusProgress.setVisible(false); } @@ -1476,6 +1483,7 @@ private Thread ripAlbum(String urlString) { pack(); boolean failed = false; try { + LOGGER.debug("Creating ripper for url {}", url); ripper = AbstractRipper.getRipper(url); ripper.setup(); } catch (Exception e) { @@ -1651,10 +1659,11 @@ private synchronized void handleEvent(StatusEvent evt) { if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } - ripFinishCleanup(); openButton.setVisible(false); + ripFinishCleanup(); pack(); statusWithColor("Error: " + msg.getObject(), Color.RED); + ripNextAlbum(); break; case RIP_COMPLETE: @@ -1683,7 +1692,6 @@ private synchronized void handleEvent(StatusEvent evt) { } saveHistory(); Utils.saveConfig(); - ripFinishCleanup(); openButton.setVisible(true); Path f = rsc.dir; String prettyFile = Utils.shortenPath(f); @@ -1740,6 +1748,7 @@ private synchronized void handleEvent(StatusEvent evt) { LOGGER.error(e); } }); + ripFinishCleanup(); pack(); ripNextAlbum(); break; @@ -1753,8 +1762,8 @@ private synchronized void handleEvent(StatusEvent evt) { if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } - ripFinishCleanup(); openButton.setVisible(false); + ripFinishCleanup(); pack(); statusWithColor("Error: " + msg.getObject(), Color.RED); break; From b61d0693f54072de5522263ed12f9f660bc15680 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 14:59:01 -0400 Subject: [PATCH 18/22] Reconfigure logger from new Configuration Log4j2 docs recommend against modifying Appenders, and recommend building new Configuration objects instead --- .../com/rarchives/ripme/ui/MainWindow.java | 24 +++----- .../java/com/rarchives/ripme/utils/Utils.java | 59 ++++++++++++------- 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 528b854a9..ad250e468 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1146,24 +1146,14 @@ public void intervalRemoved(ListDataEvent arg0) { } private void setLogLevel(String level) { - // default level is error, set in case something else is given. - Level newLevel = Level.ERROR; level = level.substring(level.lastIndexOf(' ') + 1); - switch (level) { - case "Debug": - newLevel = Level.DEBUG; - break; - case "Info": - newLevel = Level.INFO; - break; - case "Warn": - newLevel = Level.WARN; - } - LoggerContext ctx = (LoggerContext) LogManager.getContext(false); - Configuration config = ctx.getConfiguration(); - LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); - loggerConfig.setLevel(newLevel); - ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig. + Level newLevel = switch (level) { + case "Debug" -> Level.DEBUG; + case "Info" -> Level.INFO; + case "Warn" -> Level.WARN; + default -> Level.ERROR; + }; + Utils.configureLogger(newLevel); } private void setupTrayIcon() { diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 4e9786c75..7d1325b2a 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -39,9 +39,11 @@ import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.commons.configuration2.io.FileHandler; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.ConsoleAppender; import org.apache.logging.log4j.core.appender.RollingFileAppender; import org.apache.logging.log4j.core.appender.rolling.DefaultRolloverStrategy; import org.apache.logging.log4j.core.appender.rolling.SizeBasedTriggeringPolicy; @@ -50,6 +52,10 @@ import org.apache.logging.log4j.core.config.LoggerConfig; import com.rarchives.ripme.ripper.AbstractRipper; +import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; +import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory; +import org.apache.logging.log4j.core.config.builder.api.RootLoggerComponentBuilder; +import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; /** * Common utility functions used in various places throughout the project. @@ -603,36 +609,45 @@ public static void playSound(String filename) { } } + public static void configureLogger() { + configureLogger(Level.INFO); // default INFO level + } + /** * Configures root logger, either for FILE output or just console. */ - public static void configureLogger() { - LoggerContext ctx = (LoggerContext) LogManager.getContext(false); - Configuration config = ctx.getConfiguration(); - LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); + public static void configureLogger(Level level) { + ConfigurationBuilder builder = ConfigurationBuilderFactory.newConfigurationBuilder(); + + //builder.setStatusLevel(Level.DEBUG); + final String consoleAppenderName = "stdout"; + builder.add(builder.newAppender(consoleAppenderName, "CONSOLE") + .addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT) + .add(builder.newLayout("PatternLayout").addAttribute("pattern", "%-5level %c{1}: %msg%n%xEx")) + ); + + RootLoggerComponentBuilder rootLogger = builder.newRootLogger(level); + rootLogger.add(builder.newAppenderRef(consoleAppenderName)); // write to ripme.log file if checked in GUI boolean logSave = getConfigBoolean("log.save", false); if (logSave) { - LOGGER.debug("add rolling appender ripmelog"); - TriggeringPolicy tp = SizeBasedTriggeringPolicy.createPolicy("20M"); - DefaultRolloverStrategy rs = DefaultRolloverStrategy.newBuilder().withMax("2").build(); - RollingFileAppender rolling = RollingFileAppender.newBuilder() - .setName("ripmelog") - .withFileName("ripme.log") - .withFilePattern("%d{yyyy-MM-dd HH:mm:ss} %p %m%n") - .withPolicy(tp) - .withStrategy(rs) - .build(); - loggerConfig.addAppender(rolling, null, null); - } else { - LOGGER.debug("remove rolling appender ripmelog"); - if (config.getAppender("ripmelog") != null) { - config.getAppender("ripmelog").stop(); - } - loggerConfig.removeAppender("ripmelog"); + final String fileAppenderName = "rolling"; + builder.add(builder.newAppender(fileAppenderName, "RollingFile") + .addAttribute("fileName", "ripme.log") + .addAttribute("filePattern", "ripme-%d{yyyy-MM-dd}-%i.log.gz") + .add(builder.newLayout("PatternLayout").addAttribute("pattern", "%d %-5level %c{1}: %msg%n%xEx")) + .addComponent(builder.newComponent("Policies") + .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "20M"))) + ); + rootLogger.add(builder.newAppenderRef(fileAppenderName)); } - ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig. + + builder.add(rootLogger); + + Configuration configuration = builder.build(); + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + ctx.reconfigure(configuration); } /** From aece21e96edb4c0eaec98aa8111329c4f19a156b Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 24 Jul 2025 03:03:42 -0400 Subject: [PATCH 19/22] Set antialiasing hint --- src/main/java/com/rarchives/ripme/App.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 9b8431101..539e9a3f0 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -76,6 +76,9 @@ public static void main(String[] args) throws IOException { if (GraphicsEnvironment.isHeadless() || args.length > 0) { handleArguments(args); } else { + // Antialiasing hint, especially for Linux + System.setProperty("awt.useSystemAAFontSettings", "on"); + if (SystemUtils.IS_OS_MAC_OSX) { System.setProperty("apple.laf.useScreenMenuBar", "true"); System.setProperty("com.apple.mrj.application.apple.menu.about.name", "RipMe"); From 6ee6db10772bfacf3bdfe1ff1ed93547eb964108 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 24 Jul 2025 03:06:28 -0400 Subject: [PATCH 20/22] Reduce duplicate code --- .../com/rarchives/ripme/ui/MainWindow.java | 49 +------------------ 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index ad250e468..4e4a2f07c 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -731,8 +731,8 @@ private void checkAndUpdate() { return field; } - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JLabel thing1ToAdd, - JButton thing2ToAdd) { + private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JComponent thing1ToAdd, + JComponent thing2ToAdd) { gbc.gridy = gbcYValue; gbc.gridx = 0; configurationPanel.add(thing1ToAdd, gbc); @@ -740,51 +740,6 @@ private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYV configurationPanel.add(thing2ToAdd, gbc); } - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JLabel thing1ToAdd, - JTextField thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JCheckBox thing1ToAdd, - JCheckBox thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - @SuppressWarnings("rawtypes") - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JCheckBox thing1ToAdd, - JComboBox thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - @SuppressWarnings("rawtypes") - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JComboBox thing1ToAdd, - JButton thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - @SuppressWarnings({ "unused", "rawtypes" }) - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JComboBox thing1ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - } - private void changeLocale() { statusLabel.setText(Utils.getLocalizedString("inactive")); configUpdateButton.setText(Utils.getLocalizedString("check.for.updates")); From 37c07eaeb4847107e1972f9536bc19cf180127ac Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 24 Jul 2025 03:44:17 -0400 Subject: [PATCH 21/22] Log error with logger --- .../ripme/uiUtils/ContextActionProtections.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java b/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java index 9237fea90..57ea0aa1b 100644 --- a/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java +++ b/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.uiUtils; -import javax.swing.*; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import javax.swing.text.JTextComponent; import java.awt.*; import java.awt.datatransfer.Clipboard; @@ -10,6 +12,8 @@ import java.io.IOException; public class ContextActionProtections { + private static final Logger logger = LogManager.getLogger(ContextActionProtections.class); + public static void pasteFromClipboard(JTextComponent textComponent) { Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard(); Transferable transferable = clipboard.getContents(new Object()); @@ -24,8 +28,8 @@ public static void pasteFromClipboard(JTextComponent textComponent) { // } // Set the text in the JTextField textComponent.setText(clipboardContent); - } catch (UnsupportedFlavorException | IOException unable_to_modify_text_on_paste) { - unable_to_modify_text_on_paste.printStackTrace(); + } catch (UnsupportedFlavorException | IOException e) { + logger.error("Unable to paste from clipboard", e); } } } From 8bb2a32afee229b811f4883bf4dfc5ffb523c236 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 10 Aug 2026 05:56:35 +0200 Subject: [PATCH 22/22] Reduce extra UI borders and avoid layout shifting Moves the single EmptyBorder(5,5,5,5) to the outer content pane instead of applying it separately to every sub-panel, and fixes button preferred sizes so they don't shift position/size when their label text bolds/unbolds. Co-Authored-By: Claude Sonnet 5 --- .../com/rarchives/ripme/ui/MainWindow.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 4e4a2f07c..f3331d952 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -196,7 +196,7 @@ public MainWindow() throws IOException { mainFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); mainFrame.setLayout(new GridBagLayout()); - createUI(mainFrame.getContentPane()); + createUI((JPanel) mainFrame.getContentPane()); pack(); loadHistory(); @@ -282,7 +282,7 @@ private boolean isCollapsed() { && !configurationPanel.isVisible()); } - private void createUI(Container pane) { + private void createUI(JPanel pane) { // If creating the tray icon fails, ignore it. try { setupTrayIcon(); @@ -290,7 +290,7 @@ private void createUI(Container pane) { LOGGER.warn(e.getMessage()); } - EmptyBorder emptyBorder = new EmptyBorder(5, 5, 5, 5); + pane.setBorder(new EmptyBorder(5, 5, 5, 5)); GridBagConstraints gbc = new GridBagConstraints(); gbc.fill = GridBagConstraints.HORIZONTAL; gbc.weightx = 1; @@ -364,7 +364,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib } catch (Exception ignored) { } JPanel ripPanel = new JPanel(new GridBagLayout()); - ripPanel.setBorder(emptyBorder); gbc.fill = GridBagConstraints.BOTH; gbc.weightx = 0; @@ -391,7 +390,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib openButton = new JButton(); openButton.setVisible(false); JPanel statusPanel = new JPanel(new GridBagLayout()); - statusPanel.setBorder(emptyBorder); gbc.gridx = 0; gbc.weightx = 1; @@ -408,12 +406,10 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.gridwidth = 1; JPanel progressPanel = new JPanel(new GridBagLayout()); - progressPanel.setBorder(emptyBorder); statusProgress = new JProgressBar(0, 100); progressPanel.add(statusProgress, gbc); JPanel optionsPanel = new JPanel(new GridBagLayout()); - optionsPanel.setBorder(emptyBorder); optionLog = new JButton(Utils.getLocalizedString("Log")); optionHistory = new JButton(Utils.getLocalizedString("History")); optionQueue = new JButton(Utils.getLocalizedString("queue")); @@ -435,6 +431,13 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib } catch (Exception e) { LOGGER.warn(e.getMessage()); } + + // Prevent button sizes/positions from shifting when text bolds/unbolds + optionLog.setPreferredSize(optionLog.getPreferredSize()); + optionHistory.setPreferredSize(optionHistory.getPreferredSize()); + optionQueue.setPreferredSize(optionQueue.getPreferredSize()); + optionConfiguration.setPreferredSize(optionConfiguration.getPreferredSize()); + gbc.gridx = 0; optionsPanel.add(optionLog, gbc); gbc.gridx = 1; @@ -445,7 +448,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib optionsPanel.add(optionConfiguration, gbc); logPanel = new JPanel(new GridBagLayout()); - logPanel.setBorder(emptyBorder); logText = new JTextPane(); logText.setEditable(false); JScrollPane logTextScroll = new JScrollPane(logText); @@ -459,7 +461,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.weighty = 0; historyPanel = new JPanel(new GridBagLayout()); - historyPanel.setBorder(emptyBorder); historyPanel.setVisible(false); historyPanel.setPreferredSize(new Dimension(300, 250)); @@ -541,7 +542,6 @@ public void setValueAt(Object value, int row, int col) { gbc.ipady = 0; JPanel historyButtonPanel = new JPanel(new GridBagLayout()); historyButtonPanel.setSize(new Dimension(300, 10)); - historyButtonPanel.setBorder(emptyBorder); gbc.gridx = 0; historyButtonPanel.add(historyButtonRemove, gbc); gbc.gridx = 1; @@ -555,7 +555,6 @@ public void setValueAt(Object value, int row, int col) { historyPanel.add(historyButtonPanel, gbc); queuePanel = new JPanel(new GridBagLayout()); - queuePanel.setBorder(emptyBorder); queuePanel.setVisible(false); queuePanel.setPreferredSize(new Dimension(300, 250)); queueListModel = new DefaultListModel<>(); @@ -582,7 +581,6 @@ public void setValueAt(Object value, int row, int col) { gbc.ipady = 0; configurationPanel = new JPanel(new GridBagLayout()); - configurationPanel.setBorder(emptyBorder); configurationPanel.setVisible(false); // TODO Configuration components