diff --git a/build.gradle.kts b/build.gradle.kts index 2859514e2..b7883429f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -12,6 +12,7 @@ plugins { id("jacoco") id("java") id("maven-publish") + id("com.gradleup.shadow") version "9.0.0-rc1" } repositories { @@ -85,6 +86,10 @@ tasks.withType { }) } +tasks.shadowJar { + transform() +} + publishing { publications { create("maven") { diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 9b8431101..539e9a3f0 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -76,6 +76,9 @@ public static void main(String[] args) throws IOException { if (GraphicsEnvironment.isHeadless() || args.length > 0) { handleArguments(args); } else { + // Antialiasing hint, especially for Linux + System.setProperty("awt.useSystemAAFontSettings", "on"); + if (SystemUtils.IS_OS_MAC_OSX) { System.setProperty("apple.laf.useScreenMenuBar", "true"); System.setProperty("com.apple.mrj.application.apple.menu.about.name", "RipMe"); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 0740f62c4..2ba66554e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -13,11 +13,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -36,9 +32,6 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AbstractHTMLRipper.class); - private final Map itemsPending = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); Document cachedFirstPage; protected AbstractHTMLRipper(URL url) throws IOException { @@ -76,10 +69,6 @@ protected List getDescriptionsFromPage(Document doc) throws IOException protected abstract void downloadURL(URL url, int index); - protected DownloadThreadPool getThreadPool() { - return null; - } - protected boolean keepSortOrder() { return true; } @@ -121,7 +110,7 @@ protected boolean pageContainsAlbums(URL url) { @Override public void rip() throws IOException, URISyntaxException { - int index = 0; + int imageIndex = 0; int textindex = 0; logger.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); @@ -176,9 +165,9 @@ public void rip() throws IOException, URISyntaxException { } for (String imageURL : imageURLs) { - index += 1; - logger.debug("Found image url #" + index + ": '" + imageURL + "'"); - downloadURL(new URI(imageURL).toURL(), index); + imageIndex += 1; + logger.debug("Found image url #" + imageIndex + ": '" + imageURL + "'"); + downloadURL(new URI(imageURL).toURL(), imageIndex); if (isStopped() || isThisATest()) { break; } @@ -206,7 +195,7 @@ public void rip() throws IOException, URISyntaxException { workingDir.getCanonicalPath() + "" + File.separator - + getPrefix(index) + + getPrefix(imageIndex) + (tempDesc.length > 1 ? tempDesc[1] : filename) + ".txt").exists(); @@ -236,12 +225,17 @@ public void rip() throws IOException, URISyntaxException { } } - // If they're using a thread pool, wait for it. - if (getThreadPool() != null) { - logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); - getThreadPool().waitForThreads(); + logger.info("All items queued; total items: {}; url: {}", imageIndex, url); + + // Final total item count is now known + setItemsTotal(imageIndex); + + if (getCrawlerThreadPool() != null) { + logger.debug("Waiting for crawler threadpool: {}", url); + getCrawlerThreadPool().waitForThreads(imageIndex, shouldStop, url); } - waitForThreads(); + + waitForRipperThreads(); } /** @@ -338,68 +332,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /* - Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - - @Override - /* - Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.containsKey(url) - || itemsCompleted.containsKey(url) - || itemsErrored.containsKey(url) )) { - // Item is already downloaded/downloading, skip it. - logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); - return false; - } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(url, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - } - else { - itemsPending.put(url, saveAs.toFile()); - DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - - @Override - public boolean addURLToDownload(URL url, Path saveAs) { - return addURLToDownload(url, saveAs, null, null, false); - } - /** * Queues image to be downloaded and saved. * Uses filename from URL to decide filename. @@ -413,72 +345,6 @@ protected boolean addURLToDownload(URL url) { return addURLToDownload(url, "", ""); } - @Override - /* - Cleans up & tells user about successful download - */ - public void downloadCompleted(URL url, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(url); - itemsCompleted.put(url, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - @Override - /* - * Cleans up & tells user about failed download. - */ - public void downloadErrored(URL url, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(url); - itemsErrored.put(url, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); - - checkIfComplete(); - } - - @Override - /* - Tells user that a single file in the album they wish to download has - already been downloaded in the past. - */ - public void downloadExists(URL url, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(url); - itemsCompleted.put(url, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - super.checkIfComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index a49084c63..5ed3c637c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -11,10 +11,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -31,10 +28,6 @@ public abstract class AbstractJSONRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AbstractJSONRipper.class); - private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); - protected AbstractJSONRipper(URL url) throws IOException { super(url); } @@ -49,9 +42,6 @@ protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxEx } protected abstract List getURLsFromJSON(JSONObject json); protected abstract void downloadURL(URL url, int index); - private DownloadThreadPool getThreadPool() { - return null; - } protected boolean keepSortOrder() { return true; @@ -69,7 +59,7 @@ public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException @Override public void rip() throws IOException, URISyntaxException { - int index = 0; + int imageIndex = 0; logger.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); JSONObject json = getFirstPage(); @@ -98,9 +88,9 @@ public void rip() throws IOException, URISyntaxException { break; } - index += 1; - logger.debug("Found image url #" + index+ ": " + imageURL); - downloadURL(new URI(imageURL).toURL(), index); + imageIndex += 1; + logger.debug("Found image url #" + imageIndex+ ": " + imageURL); + downloadURL(new URI(imageURL).toURL(), imageIndex); } if (isStopped() || isThisATest()) { @@ -116,12 +106,17 @@ public void rip() throws IOException, URISyntaxException { } } - // If they're using a thread pool, wait for it. - if (getThreadPool() != null) { - logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); - getThreadPool().waitForThreads(); + logger.info("All items queued; total items: {}; url: {}", imageIndex, url); + + // Final total item count is now known + setItemsTotal(imageIndex); + + if (getCrawlerThreadPool() != null) { + logger.debug("Waiting for crawler threadpool: {}", url); + getCrawlerThreadPool().waitForThreads(imageIndex, shouldStop, url); } - waitForThreads(); + + waitForRipperThreads(); } protected String getPrefix(int index) { @@ -140,68 +135,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /** - * Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - - @Override - /** - * Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.containsKey(url) - || itemsCompleted.containsKey(url) - || itemsErrored.containsKey(url) )) { - // Item is already downloaded/downloading, skip it. - logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); - return false; - } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(url, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - } - else { - itemsPending.put(url, saveAs.toFile()); - DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - - @Override - public boolean addURLToDownload(URL url, Path saveAs) { - return addURLToDownload(url, saveAs, null, null, false); - } - /** * Queues image to be downloaded and saved. * Uses filename from URL to decide filename. @@ -215,72 +148,6 @@ protected boolean addURLToDownload(URL url) { return addURLToDownload(url, "", ""); } - @Override - /** - * Cleans up & tells user about successful download - */ - public void downloadCompleted(URL url, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(url); - itemsCompleted.put(url, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - @Override - /** - * Cleans up & tells user about failed download. - */ - public void downloadErrored(URL url, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(url); - itemsErrored.put(url, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); - - checkIfComplete(); - } - - @Override - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. - */ - public void downloadExists(URL url, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(url); - itemsCompleted.put(url, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - super.checkIfComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 8ccff6481..db27dc1a7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -11,16 +11,14 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Observable; -import java.util.Random; -import java.util.Scanner; +import java.nio.file.StandardOpenOption; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -45,6 +43,28 @@ public abstract class AbstractRipper implements RipperInterface, Runnable { private static final Logger logger = LogManager.getLogger(AbstractRipper.class); + + protected final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); + protected final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + protected final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); + protected final Map itemsSkipped = Collections.synchronizedMap(new HashMap<>()); + + /** + * Rippers should set itemsTotal to the best known number of total items, + * if known at the start of the rip, e.g. in getFirstPage(). + * The best known number might be indicated on the album page, + * or calculated by the number of pages and the number of items per page. + * Once the last item is seen by the HTML or JSON crawler, the final value is set. + */ + private final AtomicInteger itemsTotal = new AtomicInteger(0); + + /** + * If an album has a duplicate RipUrlId (e.g. the same image linked twice), + * duplicates can't be counted by itemsPending, but {@link #waitForRipperThreads()} needs + * to know that the ripper has seen each link crawled. + */ + private final AtomicInteger itemsSeen = new AtomicInteger(0); + private final String URLHistoryFile = Utils.getURLHistoryFile(); public static final String USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"; @@ -53,10 +73,11 @@ public abstract class AbstractRipper protected URL url; protected File workingDir; - DownloadThreadPool threadPool; + private DownloadThreadPool ripperThreadPool; + private DownloadThreadPool crawlerThreadPool; RipStatusHandler observer = null; - private boolean completed = true; + private final AtomicBoolean completed = new AtomicBoolean(false); public abstract void rip() throws IOException, URISyntaxException; @@ -64,13 +85,16 @@ public abstract class AbstractRipper public abstract String getGID(URL url) throws MalformedURLException, URISyntaxException; + protected abstract boolean allowDuplicates(); + public boolean hasASAPRipping() { return false; } // Everytime addUrlToDownload skips a already downloaded url this increases by 1 public int alreadyDownloadedUrls = 0; - private final AtomicBoolean shouldStop = new AtomicBoolean(false); + protected final AtomicBoolean shouldStop = new AtomicBoolean(false); + protected final AtomicBoolean shouldPanic = new AtomicBoolean(false); private static boolean thisIsATest = false; public void stop() { @@ -78,6 +102,16 @@ public void stop() { shouldStop.set(true); } + public void panic() { + logger.trace("panic()"); + shouldStop.set(true); + shouldPanic.set(true); + } + + public boolean isPanicked() { + return shouldPanic.get(); + } + public boolean isStopped() { return shouldStop.get(); } @@ -88,6 +122,23 @@ protected void stopCheck() throws IOException { } } + /** + * Used for file downloads. Used by {@link #addURLToDownload(TokenedUrlGetter, RipUrlId, Path, String, String, Map, Boolean)} + */ + protected DownloadThreadPool getRipperThreadPool() { + return ripperThreadPool; + } + + /** + * Used by Rippers to crawl file pages.
+ * After the last file page is crawled and all threads are queued to the ripper thread pool, + * {@link #rip()} terminates the crawler thread pool.
+ * After the crawler thread pool is finished, {@link #rip()} terminates the ripper thread pool. + */ + protected DownloadThreadPool getCrawlerThreadPool() { + return crawlerThreadPool; + } + /** * Adds a URL to the url history file * @@ -215,7 +266,8 @@ public void setup() throws IOException, URISyntaxException { // ctx.reconfigure(); // ctx.updateLoggers(); - this.threadPool = new DownloadThreadPool(); + this.ripperThreadPool = new DownloadThreadPool(getClass().getSimpleName() + "-ripper-" + getGID(url)); + this.crawlerThreadPool = new DownloadThreadPool(getClass().getSimpleName() + "-crawler"); } public void setObserver(RipStatusHandler obs) { @@ -229,7 +281,9 @@ public void setObserver(RipStatusHandler obs) { * @param saveAs Path of the local file to save the content to. * @return True on success, false on failure. */ - public abstract boolean addURLToDownload(URL url, Path saveAs); + public boolean addURLToDownload(URL url, Path saveAs) { + return addURLToDownload(url, saveAs, null, null, false); + } /** * Queues image to be downloaded and saved. @@ -242,8 +296,81 @@ public void setObserver(RipStatusHandler obs) { * @return True if downloaded successfully * False if failed to download */ - protected abstract boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, - Boolean getFileExtFromMIME); + public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + itemsSeen.incrementAndGet(); + TokenedUrlGetter tug = () -> url; + RipUrlId ripUrlId = new RipUrlId(getClass(), getHost(), url); + Path directory = saveAs.getParent(); + String filename = saveAs.getFileName().toString(); + return addURLToDownload(tug, ripUrlId, directory, filename, referrer, cookies, getFileExtFromMIME); + } + + protected boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String referrer, Map cookies, Boolean getFileExtFromMIME) { + return addURLToDownload(tug, ripUrlId, directory, null, referrer, cookies, getFileExtFromMIME); + } + + /** + * Queues multiple URLs of single images to download from a single Album URL + */ + public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { + // Only download one file if this is a test. + if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { + stop(); + itemsPending.clear(); + return false; + } + + if (!allowDuplicates() + && ( itemsPending.contains(ripUrlId) + || itemsCompleted.containsKey(ripUrlId) + || itemsErrored.containsKey(ripUrlId) )) { + // Item is already downloaded/downloading, skip it. + // TODO print path if in itemsCompleted or itemsErrored + logger.info("[!] Skipping " + ripUrlId + " -- already attempted: " + Utils.removeCWD(directory)); + return false; + } + + if (Utils.getConfigBoolean("urls_only.save", false)) { + // Output URL to file + Path urlFile = Paths.get(this.workingDir + "/urls.txt"); + URL url = null; + try { + url = tug.getTokenedUrl(); + } catch (IOException | URISyntaxException e) { + logger.error("Unable to get URL for {}", ripUrlId, e); + itemsErrored.put(ripUrlId, e.getMessage()); + return false; + } + if (AbstractRipper.shouldIgnoreExtension(url)) { + sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + return false; + } + String text = url.toExternalForm() + System.lineSeparator(); + try { + Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); + itemsCompleted.put(ripUrlId, urlFile); + } catch (IOException e) { + logger.error("Error while writing to " + urlFile, e); + return false; + } + return true; + } + else { + itemsPending.add(ripUrlId); + DownloadFileThread dft = new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME); + if (referrer != null) { + dft.setReferrer(referrer); + } + if (cookies != null) { + dft.setCookies(cookies); + } + getRipperThreadPool().addThread(dft); + } + + return true; + } + + /** * Queues image to be downloaded and saved. @@ -301,6 +428,7 @@ protected boolean addURLToDownload(URL url, Map options) { */ protected boolean addURLToDownload(URL url, String subdirectory, String referrer, Map cookies, String prefix, String fileName, String extension, Boolean getFileExtFromMIME) { + itemsSeen.incrementAndGet(); // A common bug is rippers adding urls that are just "http:". // This rejects said urls. if (url.toExternalForm().equals("http:") || url.toExternalForm().equals("https:")) { @@ -478,11 +606,24 @@ public static String getFileName(URL url, String prefix, String fileName, String /** * Waits for downloading threads to complete. */ - protected void waitForThreads() { - logger.debug("Waiting for threads to finish"); - completed = false; - threadPool.waitForThreads(); - checkIfComplete(); + protected void waitForRipperThreads() { + waitForRipperThreads(true); + } + + protected void waitForRipperThreads(boolean notifyComplete) { + logger.debug("Waiting for threads to finish; url: {}", url); + if (!notifyComplete) { + setItemsTotal(0); + } + ripperThreadPool.waitForThreads(() -> { + boolean finished = shouldStop.get() || (itemsSeen.get() >= itemsTotal.get() && itemsPending.isEmpty()); + logger.trace("ripperThreadPool: are threads finished? {} url: {} shouldStop: {}; itemsPending.size(): {}; itemsCompleted.size(): {}; itemsErrored.size(): {}; itemsSkipped.size(): {}; itemsTotal: {}; itemsSeen: {}", + finished, url, shouldStop, itemsPending.size(), itemsCompleted.size(), itemsErrored.size(), itemsSkipped.size(), itemsTotal, itemsSeen); + return finished; + }, url); + if (notifyComplete) { + notifyComplete(); + } } /** @@ -500,44 +641,93 @@ public void retrievingSource(String url) { /** * Notifies observers that a file download has completed. * - * @param url URL that was completed. - * @param saveAs Where the downloaded file is stored. + * @param ripUrlId The RipUrlId that was completed. + * @param saveAs Where the downloaded file is stored. + */ + protected void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { + itemsPending.remove(ripUrlId); + itemsCompleted.put(ripUrlId, saveAs); + if (observer == null) { + return; + } + try { + String path = Utils.removeCWD(saveAs); + RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); + observer.update(this, msg); + + //checkIfComplete(); + } catch (Exception e) { + logger.error("Exception while updating observer: ", e); + } + } + + /** + * Notifies observers that a file could not be downloaded (includes a reason). */ - public abstract void downloadCompleted(URL url, Path saveAs); + protected void downloadErrored(RipUrlId ripUrlId, String reason) { + itemsPending.remove(ripUrlId); + itemsErrored.put(ripUrlId, reason); + if (observer == null) { + return; + } + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); + + //checkIfComplete(); + } /** * Notifies observers that a file could not be downloaded (includes a reason). */ - public abstract void downloadErrored(URL url, String reason); + protected void downloadSkipped(RipUrlId ripUrlId, String reason) { + itemsPending.remove(ripUrlId); + //itemsSkipped.put(ripUrlId, reason); + itemsCompleted.put(ripUrlId, null); // TODO use itemsSkipped and make the progress bar display it as completed + if (observer == null) { + return; + } + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_SKIP, ripUrlId + " : " + reason)); + + //checkIfComplete(); + } /** * Notify observers that a download could not be completed, * but was not technically an "error". */ - public abstract void downloadExists(URL url, Path file); + protected void downloadExists(RipUrlId ripUrlId, Path file) { + itemsPending.remove(ripUrlId); + itemsCompleted.put(ripUrlId, file); + if (observer == null) { + return; + } + + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); + + //checkIfComplete(); + } /** * @return Number of files downloaded. */ - int getCount() { - return 1; + public int getCount() { + return itemsCompleted.size() + itemsErrored.size(); } /** * Notifies observers and updates state if all files have been ripped. */ - void checkIfComplete() { + protected void notifyComplete() { if (observer == null) { logger.debug("observer is null"); return; } - if (!completed) { - completed = true; + if (!completed.getAndSet(true)) { logger.info(" Rip completed!"); RipStatusComplete rsc = new RipStatusComplete(workingDir.toPath(), getCount()); RipStatusMessage msg = new RipStatusMessage(STATUS.RIP_COMPLETE, rsc); + logger.debug("Sending RIP_COMPLETE: url: {}", getURL()); observer.update(this, msg); // we do not care if the rollingfileappender is active, @@ -671,11 +861,11 @@ public void run() { rip(); } catch (HttpStatusException e) { logger.error("Got exception while running ripper:", e); - waitForThreads(); + waitForRipperThreads(false); sendUpdate(STATUS.RIP_ERRORED, "HTTP status code " + e.getStatusCode() + " for URL " + e.getUrl()); } catch (Exception e) { logger.error("Got exception while running ripper:", e); - waitForThreads(); + waitForRipperThreads(false); sendUpdate(STATUS.RIP_ERRORED, e.getMessage()); } finally { cleanup(); @@ -789,7 +979,7 @@ protected boolean tryResumeDownload() { return false; } - protected boolean shouldIgnoreURL(URL url) { + protected static boolean shouldIgnoreExtension(URL url) { final String[] ignoredExtensions = Utils.getConfigStringArray("download.ignore_extensions"); if (ignoredExtensions == null || ignoredExtensions.length == 0) return false; // nothing ignored @@ -804,4 +994,22 @@ protected boolean shouldIgnoreURL(URL url) { } return false; } + + /** + * Gets the asserted number of total items, or 0 if unknown. + * Possibly useful in rippers. + */ + protected int getItemsTotal() { + return itemsTotal.get(); + } + + /** + * For use in rippers to update the best estimate of total items. + */ + protected void setItemsTotal(int itemsTotal) { + if (itemsTotal < 0) { + throw new IllegalArgumentException("itemsTotal cannot be negative. Use 0 for unknown."); + } + this.itemsTotal.set(itemsTotal); + } } diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index bda3bf6fb..1570e11b0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -10,9 +10,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -32,10 +30,6 @@ public abstract class AlbumRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AlbumRipper.class); - private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); - protected AlbumRipper(URL url) throws IOException { super(url); } @@ -50,147 +44,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /** - * Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - - @Override - /** - * Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.containsKey(url) - || itemsCompleted.containsKey(url) - || itemsErrored.containsKey(url) )) { - // Item is already downloaded/downloading, skip it. - logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs)); - return false; - } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(url, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - } - else { - itemsPending.put(url, saveAs.toFile()); - DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - - @Override - public boolean addURLToDownload(URL url, Path saveAs) { - return addURLToDownload(url, saveAs, null, null, false); - } - - /** - * Queues image to be downloaded and saved. - * Uses filename from URL to decide filename. - * @param url - * URL to download - * @return - * True on success - */ - protected boolean addURLToDownload(URL url) { - // Use empty prefix and empty subdirectory - return addURLToDownload(url, "", ""); - } - - @Override - /** - * Cleans up & tells user about successful download - */ - public void downloadCompleted(URL url, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(url); - itemsCompleted.put(url, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - @Override - /** - * Cleans up & tells user about failed download. - */ - public void downloadErrored(URL url, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(url); - itemsErrored.put(url, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); - - checkIfComplete(); - } - - @Override - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. - */ - public void downloadExists(URL url, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(url); - itemsCompleted.put(url, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - super.checkIfComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index e9c6f2427..53b31fa37 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -2,7 +2,9 @@ import java.io.*; import java.net.*; +import java.nio.file.FileStore; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; @@ -23,13 +25,14 @@ */ class DownloadFileThread implements Runnable { private static final Logger logger = LogManager.getLogger(DownloadFileThread.class); + private final TokenedUrlGetter tokenedUrlGetter; // Some URLs may be valid for a limited time. This should get a fresh url + private final RipUrlId ripUrlId; private String referrer = ""; private Map cookies = new HashMap<>(); - private final URL url; - private File saveAs; - private final String prettySaveAs; + private final Path directory; + private String filename; private final AbstractRipper observer; private final int retries; private final Boolean getFileExtFromMIME; @@ -37,11 +40,13 @@ class DownloadFileThread implements Runnable { private final int TIMEOUT; private final int retrySleep; - public DownloadFileThread(URL url, File saveAs, AbstractRipper observer, Boolean getFileExtFromMIME) { + + public DownloadFileThread(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, AbstractRipper observer, Boolean getFileExtFromMIME) { super(); - this.url = url; - this.saveAs = saveAs; - this.prettySaveAs = Utils.removeCWD(saveAs.toPath()); + this.tokenedUrlGetter = tug; + this.ripUrlId = ripUrlId; + this.directory = directory; + this.filename = filename; this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000); @@ -63,21 +68,57 @@ public void setCookies(Map cookies) { */ @Override public void run() { + + if (observer.isStopped()) { + // TODO add handler for graceful stop + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("download.interrupted")); + return; + } + + URL url = null; + try { + url = tokenedUrlGetter.getTokenedUrl(); + } catch (HttpStatusException e) { + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + logger.error("[!] Failed to get URL for " + ripUrlId); + return; // do not retry + } catch (IOException | URISyntaxException e) { + logger.error("[!] Failed to get URL for " + ripUrlId, e); + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + return; // do not retry + } + if (filename == null) { + // Strip token query parameters + filename = Path.of(url.getPath()).getFileName().toString(); + } // First thing we make sure the file name doesn't have any illegal chars in it - saveAs = new File( - saveAs.getParentFile().getAbsolutePath() + File.separator + Utils.sanitizeSaveAs(saveAs.getName())); + filename = Utils.sanitizeSaveAs(filename); + if (AbstractRipper.shouldIgnoreExtension(url)) { + observer.sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + return; + } + + if (!Files.exists(directory)) { + logger.info("[+] Creating directory: " + directory); + try { + Files.createDirectories(directory); + } catch (IOException e) { + logger.error("Error creating directory", e); + observer.downloadErrored(ripUrlId, "Error creating directory: " + directory + " ; " + e.getMessage()); + return; + } + } + + File saveAs = directory.resolve(filename).toFile(); + String prettySaveAs = Utils.removeCWD(saveAs.toPath()); + long fileSize = 0; int bytesTotal; int bytesDownloaded = 0; if (saveAs.exists() && observer.tryResumeDownload()) { fileSize = saveAs.length(); } - try { - observer.stopCheck(); - } catch (IOException e) { - observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); - return; - } + if (saveAs.exists() && !observer.tryResumeDownload() && !getFileExtFromMIME || Utils.fuzzyExists(Paths.get(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME && !observer.tryResumeDownload()) { @@ -87,25 +128,26 @@ public void run() { } else { logger.info("[!] " + Utils.getLocalizedString("skipping") + " " + url + " -- " + Utils.getLocalizedString("file.already.exists") + ": " + prettySaveAs); - observer.downloadExists(url, saveAs.toPath()); + observer.downloadExists(ripUrlId, saveAs.toPath()); return; } } - URL urlToDownload = this.url; boolean redirected = false; int tries = 0; // Number of attempts to download do { tries += 1; try { - logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Retry #" + tries : "")); - observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); + logger.info(" Downloading file: " + url + (tries > 0 ? " Try #" + tries : "")); + + String urlNoQuery = new URI(url.getProtocol(), url.getAuthority(), url.getPath(), null, null).toURL().toExternalForm(); + observer.sendUpdate(STATUS.DOWNLOAD_STARTED, urlNoQuery); // Setup HTTP request HttpURLConnection huc; - if (this.url.toString().startsWith("https")) { - huc = (HttpsURLConnection) urlToDownload.openConnection(); + if (url.getProtocol().equals("https")) { + huc = (HttpsURLConnection) url.openConnection(); } else { - huc = (HttpURLConnection) urlToDownload.openConnection(); + huc = (HttpURLConnection) url.openConnection(); } huc.setInstanceFollowRedirects(true); // It is important to set both ConnectTimeout and ReadTimeout. If you don't then @@ -140,7 +182,10 @@ public void run() { if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) { // TODO find a better way to handle servers that don't support resuming // downloads then just erroring out - throw new IOException(Utils.getLocalizedString("server.doesnt.support.resuming.downloads")); + observer.downloadErrored(ripUrlId, "Local file exists, resume attempted, but server does not support resuming downloads: " + + statusCode + " while downloading " + url.toExternalForm()); + //throw new IOException(Utils.getLocalizedString("server.doesnt.support.resuming.downloads")); + return; } if (statusCode / 100 == 3) { // 3xx Redirect if (!redirected) { @@ -149,27 +194,27 @@ public void run() { redirected = true; } String location = huc.getHeaderField("Location"); - urlToDownload = new URI(location).toURL(); - // Throw exception so download can be retried - throw new IOException("Redirect status code " + statusCode + " - redirect to " + location); + url = new URI(location).toURL(); // TODO fix redirect with TokenedUrlGetter + logger.debug("Redirect status code {} - redirect to {}", statusCode, location); + continue; // retry } if (statusCode / 100 == 4) { // 4xx errors logger.error("[!] " + Utils.getLocalizedString("nonretriable.status.code") + " " + statusCode + " while downloading from " + url); - observer.downloadErrored(url, Utils.getLocalizedString("nonretriable.status.code") + " " + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("nonretriable.status.code") + " " + statusCode + " while downloading " + url.toExternalForm()); return; // Not retriable, drop out. } if (statusCode / 100 == 5) { // 5xx errors - observer.downloadErrored(url, Utils.getLocalizedString("retriable.status.code") + " " + statusCode + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("retriable.status.code") + " " + statusCode + " while downloading " + url.toExternalForm()); - // Throw exception so download can be retried - throw new IOException(Utils.getLocalizedString("retriable.status.code") + " " + statusCode); + logger.debug("Retriable status code {} while downloading {}", statusCode, url); + continue; // retry } - if (huc.getContentLength() == 503 && urlToDownload.getHost().endsWith("imgur.com")) { + if (huc.getContentLength() == 503 && url.getHost().endsWith("imgur.com")) { // Imgur image with 503 bytes is "404" logger.error("[!] Imgur image is 404 (503 bytes long): " + url); - observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm()); + observer.downloadErrored(ripUrlId, "Imgur image is 404: " + url.toExternalForm()); return; } @@ -179,7 +224,7 @@ public void run() { bytesTotal = huc.getContentLength(); observer.setBytesTotal(bytesTotal); observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); + logger.debug("Size of file at " + url + " = " + bytesTotal + "b"); } // Save file @@ -250,16 +295,33 @@ public void run() { if (shouldSkipFileDownload) { logger.debug("Not downloading whole file because it is over 10mb and this is a test"); } else { + long lastProgressUpdate = 0; + long bytesSinceLastProgressUpdate = 0; while ((bytesRead = bis.read(data)) != -1) { - try { - observer.stopCheck(); - } catch (IOException e) { - observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); + if (observer.isPanicked()) { + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("download.interrupted")); return; } fos.write(data, 0, bytesRead); + bytesSinceLastProgressUpdate += bytesRead; + long now = System.currentTimeMillis(); + if (now > lastProgressUpdate + 200) { + lastProgressUpdate = now; + observer.sendUpdate(STATUS.CHUNK_BYTES, bytesSinceLastProgressUpdate); + if (observer.useByteProgessBar()) { + bytesDownloaded += bytesSinceLastProgressUpdate; + observer.setBytesCompleted(bytesDownloaded); + observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); + } + bytesSinceLastProgressUpdate = 0; + } + } + if (bytesSinceLastProgressUpdate > 0) { + // Flush the remaining bytes read since the last throttled update, otherwise the + // tail of every download is silently dropped from the progress/transfer-rate totals. + observer.sendUpdate(STATUS.CHUNK_BYTES, bytesSinceLastProgressUpdate); if (observer.useByteProgessBar()) { - bytesDownloaded += bytesRead; + bytesDownloaded += bytesSinceLastProgressUpdate; observer.setBytesCompleted(bytesDownloaded); observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); } @@ -275,20 +337,33 @@ public void run() { break; } catch (HttpStatusException hse) { logger.debug(Utils.getLocalizedString("http.status.exception"), hse); - logger.error("[!] HTTP status " + hse.getStatusCode() + " while downloading from " + urlToDownload); + logger.error("[!] HTTP status " + hse.getStatusCode() + " while downloading from " + hse.getUrl()); if (hse.getStatusCode() == 404 && Utils.getConfigBoolean("errors.skip404", false)) { - observer.downloadErrored(url, + observer.downloadErrored(ripUrlId, "HTTP status code " + hse.getStatusCode() + " while downloading " + url.toExternalForm()); return; } - } catch (IOException | URISyntaxException e) { + } catch (IOException e) { + if (guessIsENOSPC(e, saveAs)) { + logger.debug("IOException", e); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("device.nospace")); // TODO cancel all rips + return; + } + logger.debug("IOException", e); + logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + + e.getMessage()); + observer.downloadErrored(ripUrlId, e.getMessage()); + return; + } catch (URISyntaxException e) { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("exception.while.downloading.file")); + return; } catch (NullPointerException npe){ logger.error("[!] " + Utils.getLocalizedString("failed.to.download") + " for URL " + url); - observer.downloadErrored(url, + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); return; @@ -296,7 +371,7 @@ public void run() { if (tries > this.retries) { logger.error("[!] " + Utils.getLocalizedString("exceeded.maximum.retries") + " (" + this.retries + ") for URL " + url); - observer.downloadErrored(url, + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); return; } else { @@ -304,9 +379,42 @@ public void run() { Utils.sleep(retrySleep); } } + + // get fresh URL for the next attempt + try { + url = tokenedUrlGetter.getTokenedUrl(); + } catch (HttpStatusException e) { + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + logger.error("[!] Failed to get URL for " + ripUrlId); + return; // do not retry + } catch (IOException | URISyntaxException e) { + logger.error("[!] Failed to get URL for " + ripUrlId, e); + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + return; // do not retry + } + } while (true); - observer.downloadCompleted(url, saveAs.toPath()); - logger.info("[+] Saved " + url + " as " + this.prettySaveAs); + observer.downloadCompleted(ripUrlId, saveAs.toPath()); + logger.info("[+] Saved " + url + " as " + prettySaveAs); + } + + @SuppressWarnings("UnnecessaryLocalVariable") + private boolean guessIsENOSPC(IOException e, File saveAs) { + // The ENOSPC IOException message is localized in Java, so this only works on English locale systems. + if (e.getMessage() != null && e.getMessage().matches("No space left on device")) { + return true; + } + // Fallback: check usable space on the filesystem + try { + FileStore fs = Files.getFileStore(saveAs.toPath()); + // could check for 0 bytes, but 256 kilobytes is small enough + int downloadBufferSizeBytes = 1024 * 256; + boolean notEnoughUsableBytes = fs.getUsableSpace() < downloadBufferSizeBytes; + return notEnoughUsableBytes; + } catch (IOException ex) { + // unable to determine if no space left on device; fall through + } + return false; } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index 8ae43743f..cc0198a9f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -1,8 +1,12 @@ package com.rarchives.ripme.ripper; +import java.net.URL; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import com.rarchives.ripme.utils.Utils; import org.apache.logging.log4j.LogManager; @@ -15,42 +19,92 @@ public class DownloadThreadPool { private static final Logger logger = LogManager.getLogger(DownloadThreadPool.class); private ThreadPoolExecutor threadPool = null; - - public DownloadThreadPool() { - initialize("Main"); - } + private final AtomicLong scheduledThreadCount = new AtomicLong(0); + private final String name; public DownloadThreadPool(String threadPoolName) { - initialize(threadPoolName); - } - - /** - * Initializes the threadpool. - * @param threadPoolName Name of the threadpool. - */ - private void initialize(String threadPoolName) { int threads = Utils.getConfigInteger("threads.size", 10); logger.debug("Initializing " + threadPoolName + " thread pool with " + threads + " threads"); - threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads); + this.name = threadPoolName; + this.threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads); } + /** * For adding threads to execution pool. * @param t * Thread to be added. */ public void addThread(Runnable t) { + logger.trace("addThread called; name: {}, scheduledThreadCount: {}", name, scheduledThreadCount); + scheduledThreadCount.incrementAndGet(); threadPool.execute(t); } /** * Tries to shutdown threadpool. */ - public void waitForThreads() { + public void waitForThreads(Supplier isFinishedQueueing, URL url) { + logger.trace("waitForThreads called; name: {}; url: {}", name, url); + while (!isFinishedQueueing.get()) { + logger.trace("waiting for items to finish queueing; name: {}; url: {}", name, url); + try { + Thread.sleep(200); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.trace("sleep interrupted; name: {}; url: {}", name, url); + break; + } + } + + logger.trace("about to shutdown thread pool. name: {}; url: {}", name, url); threadPool.shutdown(); + logger.trace("thread pool shutdown. name: {}; url: {}", name, url); try { threadPool.awaitTermination(3600, TimeUnit.SECONDS); + logger.trace("thread pool terminated. name: {}; url: {}", name, url); } catch (InterruptedException e) { logger.error("[!] Interrupted while waiting for threads to finish: ", e); } } + + /** + * Tries to shutdown threadpool. + * + * expectedScheduledThreads/shouldStop are unused: crawler threads are always + * submitted synchronously in the same thread, before this is called, so + * there is nothing left to wait for them to be scheduled. Polling for + * getScheduledThreadCount() to reach expectedScheduledThreads hung forever + * for any ripper whose downloadURL() doesn't submit a crawler-pool thread + * per item (e.g. rippers that call addURLToDownload() directly), since the + * count would then never reach the expected value. + */ + public void waitForThreads(int expectedScheduledThreads, AtomicBoolean shouldStop, URL url) { + logger.trace("waitForThreads called; name: {}; url: {}", name, url); + logger.trace("about to shutdown thread pool. name: {}; url: {}", name, url); + threadPool.shutdown(); + logger.trace("thread pool shutdown. name: {}; url: {}", name, url); + try { + threadPool.awaitTermination(3600, TimeUnit.SECONDS); + logger.trace("thread pool terminated. name: {}; url: {}", name, url); + } catch (InterruptedException e) { + logger.error("[!] Interrupted while waiting for threads to finish: ", e); + } + } + + public int getPendingThreadCount() { + return threadPool.getQueue().size(); + } + + public int getActiveThreadCount() { + return threadPool.getActiveCount(); + } + + public long getCompletedThreadCount() { + return threadPool.getCompletedTaskCount(); + } + + public long getScheduledThreadCount() { + //return threadPool.getTaskCount(); // approximate, bad + return scheduledThreadCount.get(); + } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java deleted file mode 100644 index 9430adce3..000000000 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ /dev/null @@ -1,160 +0,0 @@ -package com.rarchives.ripme.ripper; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; - -import javax.net.ssl.HttpsURLConnection; - -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Utils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -/** - * Thread for downloading files. - * Includes retry logic, observer notifications, and other goodies. - */ -class DownloadVideoThread implements Runnable { - - private static final Logger logger = LogManager.getLogger(DownloadVideoThread.class); - - private final URL url; - private final Path saveAs; - private final String prettySaveAs; - private final AbstractRipper observer; - private final int retries; - - public DownloadVideoThread(URL url, Path saveAs, AbstractRipper observer) { - super(); - this.url = url; - this.saveAs = saveAs; - this.prettySaveAs = Utils.removeCWD(saveAs); - this.observer = observer; - this.retries = Utils.getConfigInteger("download.retries", 1); - } - - /** - * Attempts to download the file. Retries as needed. - * Notifies observers upon completion/error/warn. - */ - @Override - public void run() { - try { - observer.stopCheck(); - } catch (IOException e) { - observer.downloadErrored(url, "Download interrupted"); - return; - } - if (Files.exists(saveAs)) { - if (Utils.getConfigBoolean("file.overwrite", false)) { - logger.info("[!] Deleting existing file" + prettySaveAs); - try { - Files.delete(saveAs); - } catch (IOException e) { - e.printStackTrace(); - } - } else { - logger.info("[!] Skipping " + url + " -- file already exists: " + prettySaveAs); - observer.downloadExists(url, saveAs); - return; - } - } - - int bytesTotal, bytesDownloaded = 0; - try { - bytesTotal = getTotalBytes(this.url); - } catch (IOException e) { - logger.error("Failed to get file size at " + this.url, e); - observer.downloadErrored(this.url, "Failed to get file size of " + this.url); - return; - } - observer.setBytesTotal(bytesTotal); - observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); - - int tries = 0; // Number of attempts to download - do { - InputStream bis = null; OutputStream fos = null; - byte[] data = new byte[1024 * 256]; - int bytesRead; - try { - logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : "")); - observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); - - // Setup HTTP request - HttpURLConnection huc; - if (this.url.toString().startsWith("https")) { - huc = (HttpsURLConnection) this.url.openConnection(); - } - else { - huc = (HttpURLConnection) this.url.openConnection(); - } - huc.setInstanceFollowRedirects(true); - huc.setConnectTimeout(0); // Never timeout - huc.setRequestProperty("accept", "*/*"); - huc.setRequestProperty("Referer", this.url.toExternalForm()); // Sic - huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); - tries += 1; - logger.debug("Request properties: " + huc.getRequestProperties().toString()); - huc.connect(); - // Check status code - bis = new BufferedInputStream(huc.getInputStream()); - fos = Files.newOutputStream(saveAs); - while ( (bytesRead = bis.read(data)) != -1) { - try { - observer.stopCheck(); - } catch (IOException e) { - observer.downloadErrored(url, "Download interrupted"); - return; - } - fos.write(data, 0, bytesRead); - bytesDownloaded += bytesRead; - observer.setBytesCompleted(bytesDownloaded); - observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); - } - bis.close(); - fos.close(); - break; // Download successful: break out of infinite loop - } catch (IOException e) { - logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage(), e); - } finally { - // Close any open streams - try { - if (bis != null) { bis.close(); } - } catch (IOException ignored) { } - try { - if (fos != null) { fos.close(); } - } catch (IOException ignored) { } - } - if (tries > this.retries) { - logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url); - observer.downloadErrored(url, "Failed to download " + url.toExternalForm()); - return; - } - } while (true); - observer.downloadCompleted(url, saveAs); - logger.info("[+] Saved " + url + " as " + this.prettySaveAs); - } - - /** - * @param url - * Target URL - * @return - * Returns connection length - */ - private int getTotalBytes(URL url) throws IOException { - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setRequestMethod("HEAD"); - conn.setRequestProperty("accept", "*/*"); - conn.setRequestProperty("Referer", this.url.toExternalForm()); // Sic - conn.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); - return conn.getContentLength(); - } - -} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/RipUrlId.java b/src/main/java/com/rarchives/ripme/ripper/RipUrlId.java new file mode 100644 index 000000000..9e376bb2e --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/RipUrlId.java @@ -0,0 +1,97 @@ +package com.rarchives.ripme.ripper; + +import java.net.URL; +import java.util.Objects; + +/** + * RipUrlId represents a unique file on a host. + * Necessary because some files may be accessible from multiple URLs, for example: + * - a file in multiple albums, or + * - a file only accessible with a tokened URL. + */ +public class RipUrlId { + Class ripper; + String ripperHost; + String ripUrlId; + URL url; + + /** + * @param ripper The ripper associated with the id + * @param ripperHost The ripper's getHost(), because a ripper may support multiple hosts + * @param ripUrlId The unique identifier of the file fetchable by the ripper + */ + public RipUrlId(Class ripper, String ripperHost, String ripUrlId) { + if (ripper == null) { + throw new IllegalArgumentException("ripper cannot be null"); + } + if (ripperHost == null) { + throw new IllegalArgumentException("ripperHost cannot be null"); + } + if (ripUrlId == null) { + throw new IllegalArgumentException("ripUrlId cannot be null"); + } + this.ripper = ripper; + this.ripperHost = ripperHost; + this.ripUrlId = ripUrlId; + } + + /** + * Transitionary constructor for rippers that do not yet create an id + * + * @param ripper The ripper associated with the id + * @param ripperHost The ripper's getHost(), because a ripper may support multiple hosts + * @param url A URL fetchable by the ripper + * @deprecated The other constructor is preferable + */ + @Deprecated + public RipUrlId(Class ripper, String ripperHost, URL url) { + if (ripper == null) { + throw new IllegalArgumentException("ripper cannot be null"); + } + if (ripperHost == null) { + throw new IllegalArgumentException("ripperHost cannot be null"); + } + if (url == null) { + throw new IllegalArgumentException("url cannot be null"); + } + this.ripper = ripper; + this.ripperHost = ripperHost; + this.url = url; + } + + public Class getRipper() { + return ripper; + } + + public String getRipperHost() { + return ripperHost; + } + + public String getRipUrlId() { + return ripUrlId; + } + + public URL getUrl() { + return url; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + RipUrlId ripUrlId1 = (RipUrlId) o; + return Objects.equals(ripper, ripUrlId1.ripper) && Objects.equals(ripperHost, ripUrlId1.ripperHost) && Objects.equals(ripUrlId, ripUrlId1.ripUrlId) && Objects.equals(url, ripUrlId1.url); + } + + @Override + public int hashCode() { + return Objects.hash(ripper, ripperHost, ripUrlId, url); + } + + @Override + public String toString() { + if (url != null) { + return url.toString(); + } + return ripper.getSimpleName() + ": " + ripperHost + ": " + ripUrlId; + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java b/src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java new file mode 100644 index 000000000..706c7bc72 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java @@ -0,0 +1,14 @@ +package com.rarchives.ripme.ripper; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; + +public interface TokenedUrlGetter { + /** + * @return The URL of the file to fetch + * @throws IOException May be thrown if a tokened URI can't be fetched + * @throws URISyntaxException May be thrown if a URI can't be constructed + */ + URL getTokenedUrl() throws IOException, URISyntaxException; +} diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 785f3d92b..d2904feb2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -12,8 +12,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.rarchives.ripme.ui.RipStatusMessage; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; public abstract class VideoRipper extends AbstractRipper { @@ -49,41 +47,13 @@ public String getAlbumTitle(URL url) { } @Override - public boolean addURLToDownload(URL url, Path saveAs) { - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - String urlFile = this.workingDir + "/urls.txt"; - - try (FileWriter fw = new FileWriter(urlFile, true)) { - fw.write(url.toExternalForm()); - fw.write("\n"); - - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, urlFile); - observer.update(this, msg); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - return false; - } - } else { - if (isThisATest()) { - // Tests shouldn't download the whole video - // Just change this.url to the download URL so the test knows we found it. - logger.debug("Test rip, found URL: " + url); - this.url = url; - return true; - } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - threadPool.addThread(new DownloadVideoThread(url, saveAs, this)); - } + protected boolean useByteProgessBar() { return true; } @Override - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - return addURLToDownload(url, saveAs); + protected boolean allowDuplicates() { + return false; } /** @@ -120,60 +90,6 @@ public int getCompletionPercentage() { return (int) (100 * (bytesCompleted / (float) bytesTotal)); } - /** - * Runs if download successfully completed. - * - * @param url Target URL - * @param saveAs Path to file, including filename. - */ - @Override - public void downloadCompleted(URL url, Path saveAs) { - if (observer == null) { - return; - } - - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - /** - * Runs if the download errored somewhere. - * - * @param url Target URL - * @param reason Reason why the download failed. - */ - @Override - public void downloadErrored(URL url, String reason) { - if (observer == null) { - return; - } - - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); - checkIfComplete(); - } - - /** - * Runs if user tries to redownload an already existing File. - * @param url Target URL - * @param file Existing file - */ - @Override - public void downloadExists(URL url, Path file) { - if (observer == null) { - return; - } - - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - checkIfComplete(); - } - /** * Gets the status and changes it to a human-readable form. * @@ -197,18 +113,4 @@ public URL sanitizeURL(URL url) throws MalformedURLException { return url; } - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - - if (bytesCompleted >= bytesTotal) { - super.checkIfComplete(); - } - } - } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 98510250c..4bb197e77 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -30,7 +30,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -94,7 +93,6 @@ public class DeviantartRipper extends AbstractHTMLRipper { private boolean usingCatPath = false; private int downloadCount = 0; private Map cookies = new HashMap(); - private DownloadThreadPool deviantartThreadPool = new DownloadThreadPool("deviantart"); private ArrayList names = new ArrayList(); List allowedCookies = Arrays.asList("agegate_state", "userinfo", "auth", "auth_secure"); @@ -106,11 +104,6 @@ public class DeviantartRipper extends AbstractHTMLRipper { private final String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0"; private final String utilsKey = "DeviantartLogin.cookies"; //for config file - @Override - public DownloadThreadPool getThreadPool() { - return deviantartThreadPool; - } - public DeviantartRipper(URL url) throws IOException { super(url); } @@ -304,7 +297,7 @@ protected void downloadURL(URL url, int index) { // Start Thread and add to pool. DeviantartImageThread t = new DeviantartImageThread(url); - deviantartThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index 9b40f0542..d0fbbbb9e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -19,7 +19,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; @@ -37,8 +36,6 @@ public class E621Ripper extends AbstractHTMLRipper { private static Pattern gidPatternNew = null; private static Pattern gidPatternPoolNew = null; - private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621"); - private Map cookies = new HashMap(); private String userAgent = USER_AGENT; @@ -78,11 +75,6 @@ private Document getDocument(String url) throws IOException { return getDocument(url, 1); } - @Override - public DownloadThreadPool getThreadPool() { - return e621ThreadPool; - } - @Override public String getDomain() { return "e621.net"; @@ -136,7 +128,7 @@ public void downloadURL(final URL url, int index) { // rate limit sleep(3000); // addURLToDownload(url, getPrefix(index)); - e621ThreadPool.addThread(new E621FileThread(url, getPrefix(index))); + getCrawlerThreadPool().addThread(new E621FileThread(url, getPrefix(index))); } private String getTerm(URL url) throws MalformedURLException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index 5349f55c7..d58bcc4c9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -21,7 +21,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; @@ -44,8 +43,6 @@ public class EHentaiRipper extends AbstractHTMLRipper { } private String lastURL = null; - // Thread pool for finding direct image links from "image" pages (html) - private final DownloadThreadPool ehentaiThreadPool = new DownloadThreadPool("ehentai"); // Current HTML document private Document albumDoc = null; @@ -53,11 +50,6 @@ public EHentaiRipper(URL url) throws IOException { super(url); } - @Override - public DownloadThreadPool getThreadPool() { - return ehentaiThreadPool; - } - @Override public String getHost() { return "e-hentai"; @@ -194,7 +186,7 @@ public List getURLsFromPage(Document page) { @Override public void downloadURL(URL url, int index) { EHentaiImageThread t = new EHentaiImageThread(url, index, this.workingDir.toPath()); - ehentaiThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 1f1954207..37acde4ac 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -28,8 +28,6 @@ public class FlickrRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(FlickrRipper.class); - private final DownloadThreadPool flickrThreadPool; - private enum UrlType { USER, PHOTOSET @@ -45,11 +43,6 @@ private class Album { } } - @Override - public DownloadThreadPool getThreadPool() { - return flickrThreadPool; - } - @Override public boolean hasASAPRipping() { return true; @@ -57,7 +50,6 @@ public boolean hasASAPRipping() { public FlickrRipper(URL url) throws IOException { super(url); - flickrThreadPool = new DownloadThreadPool(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index ad9075d0d..cb7c49479 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -54,15 +54,6 @@ private void warnAboutSharedAccount(String loginCookies) { } } - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool furaffinityThreadPool - = new DownloadThreadPool( "furaffinity"); - - @Override - public DownloadThreadPool getThreadPool() { - return furaffinityThreadPool; - } - public FuraffinityRipper(URL url) throws IOException { super(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java index 7183f1d79..54a98f306 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java @@ -18,7 +18,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; public class HqpornerRipper extends AbstractHTMLRipper { @@ -30,7 +29,6 @@ public class HqpornerRipper extends AbstractHTMLRipper { private Pattern p1 = Pattern.compile("https?://hqporner.com/hdporn/([a-zA-Z0-9_-]*).html/?$"); // video pattern. private Pattern p2 = Pattern.compile("https://hqporner.com/([a-zA-Z0-9/_-]+)"); // category/top/actress/studio pattern. private Pattern p3 = Pattern.compile("https?://[A-Za-z0-9/.-_]+\\.mp4"); // to match links ending with .mp4 - private DownloadThreadPool hqpornerThreadPool = new DownloadThreadPool("hqpornerThreadPool"); private String subdirectory = ""; public HqpornerRipper(URL url) throws IOException { @@ -111,7 +109,7 @@ public boolean tryResumeDownload() { @Override public void downloadURL(URL url, int index) { - hqpornerThreadPool.addThread(new HqpornerDownloadThread(url, index, subdirectory)); + getCrawlerThreadPool().addThread(new HqpornerDownloadThread(url, index, subdirectory)); } @Override @@ -123,11 +121,6 @@ public Document getNextPage(Document doc) throws IOException { throw new IOException("No next page found."); } - @Override - public DownloadThreadPool getThreadPool() { - return hqpornerThreadPool; - } - @Override public boolean useByteProgessBar() { return true; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 7f58e1a24..ff2c9636b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -1,7 +1,6 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; import java.io.IOException; @@ -29,13 +28,6 @@ public class ImagebamRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(ImagebamRipper.class); - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool imagebamThreadPool = new DownloadThreadPool("imagebam"); - @Override - public DownloadThreadPool getThreadPool() { - return imagebamThreadPool; - } - public ImagebamRipper(URL url) throws IOException { super(url); } @@ -90,7 +82,7 @@ public List getURLsFromPage(Document doc) { @Override public void downloadURL(URL url, int index) { ImagebamImageThread t = new ImagebamImageThread(url, index); - imagebamThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); sleep(500); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java index 5df8e9f17..3de62ad50 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java @@ -17,7 +17,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -25,13 +24,6 @@ public class ImagevenueRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(ImagevenueRipper.class); - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool imagevenueThreadPool = new DownloadThreadPool("imagevenue"); - @Override - public DownloadThreadPool getThreadPool() { - return imagevenueThreadPool; - } - public ImagevenueRipper(URL url) throws IOException { super(url); } @@ -72,7 +64,7 @@ public List getURLsFromPage(Document doc) { public void downloadURL(URL url, int index) { ImagevenueImageThread t = new ImagevenueImageThread(url, index); - imagevenueThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 5bb7b0020..afbab931e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -205,7 +205,7 @@ public void rip() throws IOException { } catch (URISyntaxException e) { throw new IOException("Failed ripping " + this.url, e); } - waitForThreads(); + waitForRipperThreads(); } private void ripSingleImage(URL url) throws IOException, URISyntaxException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java index 7157e49dd..2cf37ef6e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java @@ -19,7 +19,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; /** @@ -38,8 +37,6 @@ public class ListalRipper extends AbstractHTMLRipper { private String listId = null; // listId to get more images via POST. private UrlType urlType = UrlType.UNKNOWN; - private DownloadThreadPool listalThreadPool = new DownloadThreadPool("listalThreadPool"); - public ListalRipper(URL url) throws IOException { super(url); } @@ -77,7 +74,7 @@ public List getURLsFromPage(Document page) { @Override public void downloadURL(URL url, int index) { - listalThreadPool.addThread(new ListalImageDownloadThread(url, index)); + getCrawlerThreadPool().addThread(new ListalImageDownloadThread(url, index)); } @Override @@ -137,11 +134,6 @@ public Document getNextPage(Document page) throws IOException, URISyntaxExceptio } - @Override - public DownloadThreadPool getThreadPool() { - return listalThreadPool; - } - /** * Returns the image urls for UrlType LIST. */ diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index 955e85a34..dc1fa3f67 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -31,11 +31,8 @@ public class MotherlessRipper extends AbstractHTMLRipper { private static final String DOMAIN = "motherless.com", HOST = "motherless"; - private DownloadThreadPool motherlessThreadPool; - public MotherlessRipper(URL url) throws IOException { super(url); - motherlessThreadPool = new DownloadThreadPool(); } @Override @@ -117,7 +114,7 @@ protected List getURLsFromPage(Document page) { protected void downloadURL(URL url, int index) { // Create thread for finding image at "url" page MotherlessImageRunnable mit = new MotherlessImageRunnable(url, index); - motherlessThreadPool.addThread(mit); + getCrawlerThreadPool().addThread(mit); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { @@ -155,11 +152,6 @@ public String getGID(URL url) throws MalformedURLException { throw new MalformedURLException("Expected URL format: https://motherless.com/GIXXXXXXX, got: " + url); } - @Override - protected DownloadThreadPool getThreadPool() { - return motherlessThreadPool; - } - /** * Helper class to find and download images found on "image" pages */ diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java index d6b17b02f..e69dbed81 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -17,7 +17,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; public class NfsfwRipper extends AbstractHTMLRipper { @@ -34,12 +33,8 @@ public class NfsfwRipper extends AbstractHTMLRipper { "https?://[wm.]*nfsfw.com/gallery/v/[^/]+/(.+)$" ); - // threads pool for downloading images from image pages - private DownloadThreadPool nfsfwThreadPool; - public NfsfwRipper(URL url) throws IOException { super(url); - nfsfwThreadPool = new DownloadThreadPool("NFSFW"); } @Override @@ -105,7 +100,7 @@ protected void downloadURL(URL url, int index) { index = ++this.index; } NfsfwImageThread t = new NfsfwImageThread(url, currentDir, index); - nfsfwThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); } @Override @@ -141,11 +136,6 @@ public String getGID(URL url) throws MalformedURLException { + " Got: " + url); } - @Override - public DownloadThreadPool getThreadPool() { - return nfsfwThreadPool; - } - @Override public boolean hasQueueSupport() { return true; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java index 41693a3ee..13df2aa74 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java @@ -27,9 +27,6 @@ public class NhentaiRipper extends AbstractHTMLRipper { private Document firstPage; - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool nhentaiThreadPool = new DownloadThreadPool("nhentai"); - @Override public boolean hasQueueSupport() { return true; @@ -51,11 +48,6 @@ public List getAlbumsToQueue(Document doc) { return urlsToAddToQueue; } - @Override - public DownloadThreadPool getThreadPool() { - return nhentaiThreadPool; - } - public NhentaiRipper(URL url) throws IOException { super(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java index 481ab1ede..c6279c3e0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java @@ -18,7 +18,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -31,9 +30,6 @@ public class PornhubRipper extends AbstractHTMLRipper { private static final String DOMAIN = "pornhub.com", HOST = "Pornhub"; - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool pornhubThreadPool = new DownloadThreadPool("pornhub"); - public PornhubRipper(URL url) throws IOException { super(url); } @@ -82,7 +78,7 @@ protected List getURLsFromPage(Document page) { @Override protected void downloadURL(URL url, int index) { PornhubImageThread t = new PornhubImageThread(url, index, this.workingDir.toPath()); - pornhubThreadPool.addThread(t); + getCrawlerThreadPool().addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { @@ -119,11 +115,6 @@ public String getGID(URL url) throws MalformedURLException { + " Got: " + url); } - @Override - public DownloadThreadPool getThreadPool(){ - return pornhubThreadPool; - } - public boolean canRip(URL url) { return url.getHost().endsWith(DOMAIN) && url.getPath().startsWith("/album"); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 2419cd035..cb6b8eeb7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -114,7 +114,7 @@ public void rip() throws IOException { } catch (URISyntaxException e) { new IOException(e.getMessage()); } - waitForThreads(); + waitForRipperThreads(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 898a0ec62..56b695c66 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -236,7 +236,7 @@ public void rip() throws IOException { } } - waitForThreads(); + waitForRipperThreads(); } private boolean handleJSON(JSONObject json) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java index 59f79aab7..e36d22ff6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java @@ -158,7 +158,7 @@ public void rip() throws IOException, URISyntaxException { for (int index = 0; index < URLs.size(); index ++) { downloadURL(new URI(URLs.get(index)).toURL(), index); } - waitForThreads(); + waitForRipperThreads(); } else { RIP_TYPE = RipType.IMAGE; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java index 1e7a48f8e..76527ceea 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java @@ -78,6 +78,6 @@ public void rip() throws IOException, URISyntaxException { } } addURLToDownload(new URI(vidURL).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java index 0f95aaafc..c6cf3f123 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java @@ -70,6 +70,6 @@ public void rip() throws IOException, URISyntaxException { } String vidUrl = vidUrls.get(0); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java index 1857c309d..b7ed4ee85 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java @@ -154,6 +154,6 @@ public void rip() throws IOException, URISyntaxException { } addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + bestQuality + "p_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java index 076e90ca6..f6249e9d0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java @@ -75,6 +75,6 @@ public void rip() throws IOException, URISyntaxException { addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + title); } } - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java index a2cff267d..31c127d39 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java @@ -68,6 +68,6 @@ public void rip() throws IOException, URISyntaxException { String vidUrl = videos.first().attr("content"); vidUrl = vidUrl.replaceAll("&", "&"); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java index 00e77c427..4829cff86 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java @@ -63,6 +63,6 @@ public void rip() throws IOException, URISyntaxException { } String vidUrl = mp4s.get(0); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java index 4a7ea8ccd..1ab5c1ef4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java @@ -61,7 +61,7 @@ public void rip() throws IOException, URISyntaxException { logger.info(" Retrieving " + this.url); String videoURL = getVideoURLAtPage(this.url.toExternalForm()); addURLToDownload(new URI(videoURL).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } public static String getVideoURLAtPage(String url) throws IOException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java index 2fe0291e6..61762f3bb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java @@ -77,6 +77,6 @@ public void rip() throws IOException, URISyntaxException { addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); } } - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index f41e695e8..f3331d952 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -15,9 +15,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; -import java.util.Date; +import java.util.*; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import javax.imageio.ImageIO; @@ -40,6 +44,7 @@ import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.uiUtils.ContextActionProtections; import com.rarchives.ripme.utils.RipUtils; +import com.rarchives.ripme.utils.TransferRate; import com.rarchives.ripme.utils.Utils; /** @@ -49,15 +54,14 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static final Logger LOGGER = LogManager.getLogger(MainWindow.class); - /* not static! */ - private boolean isRipping = false; // Flag to indicate if we're ripping something - private static JFrame mainFrame; private static JTextField ripTextfield; private static JButton ripButton, stopButton; + private static JButton panicButton; private static JLabel statusLabel; + private static final JLabel transferRateLabel = new JLabel(); private static JButton openButton; private static JProgressBar statusProgress; @@ -69,6 +73,8 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JButton optionLog; private static JPanel logPanel; private static JTextPane logText; + private static final Queue logLineLengths = new LinkedList<>(); + private static final int MAX_LOG_PANE_LINES = 1000; // History private static JButton optionHistory; @@ -129,6 +135,27 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static AbstractRipper ripper; + private static final AtomicBoolean gracefulStop = new AtomicBoolean(false); // Allow active transfers to finish, then stop ripping. + private static final AtomicBoolean panicStop = new AtomicBoolean(false); // Immediately stop active transfers, then stop ripping. + private static final AtomicBoolean isRipperActive = new AtomicBoolean(false); + + public static final int TRANSFER_RATE_REFRESH_RATE = 200; + private static final TransferRate transferRate = new TransferRate(); + + private static final ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + private Future rateRefresherFuture = null; + private final Runnable rateRefresher = () -> { + if (!isRipperActive.get()) { + if (rateRefresherFuture != null) { + rateRefresherFuture.cancel(true); + rateRefresherFuture = null; + } + transferRateLabel.setText(""); + return; + } + transferRateLabel.setText(transferRate.formatHumanTransferRate()); + }; + private void updateQueue(DefaultListModel model) { if (model == null) model = queueListModel; @@ -169,7 +196,7 @@ public MainWindow() throws IOException { mainFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); mainFrame.setLayout(new GridBagLayout()); - createUI(mainFrame.getContentPane()); + createUI((JPanel) mainFrame.getContentPane()); pack(); loadHistory(); @@ -255,7 +282,7 @@ private boolean isCollapsed() { && !configurationPanel.isVisible()); } - private void createUI(Container pane) { + private void createUI(JPanel pane) { // If creating the tray icon fails, ignore it. try { setupTrayIcon(); @@ -263,7 +290,7 @@ private void createUI(Container pane) { LOGGER.warn(e.getMessage()); } - EmptyBorder emptyBorder = new EmptyBorder(5, 5, 5, 5); + pane.setBorder(new EmptyBorder(5, 5, 5, 5)); GridBagConstraints gbc = new GridBagConstraints(); gbc.fill = GridBagConstraints.HORIZONTAL; gbc.weightx = 1; @@ -281,6 +308,8 @@ private void createUI(Container pane) { LOGGER.error("[!] Exception setting system theme:", e); } + Font monospaced = new Font(Font.MONOSPACED, Font.PLAIN, mainFrame.getContentPane().getFont().getSize()); + ripTextfield = new JTextField("", 20); ripTextfield.addMouseListener(new ContextMenuMouseListener(ripTextfield)); @@ -327,13 +356,14 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib ripButton = new JButton("Rip", ripIcon); stopButton = new JButton("Stop"); stopButton.setEnabled(false); + panicButton = new JButton("Panic!"); + panicButton.setEnabled(false); try { Image stopIcon = ImageIO.read(getClass().getClassLoader().getResource("stop.png")); stopButton.setIcon(new ImageIcon(stopIcon)); } catch (Exception ignored) { } JPanel ripPanel = new JPanel(new GridBagLayout()); - ripPanel.setBorder(emptyBorder); gbc.fill = GridBagConstraints.BOTH; gbc.weightx = 0; @@ -349,28 +379,37 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib ripPanel.add(ripButton, gbc); gbc.gridx = 3; ripPanel.add(stopButton, gbc); + gbc.gridx = 4; + ripPanel.add(panicButton, gbc); gbc.weightx = 1; statusLabel = new JLabel(Utils.getLocalizedString("inactive")); statusLabel.setHorizontalAlignment(JLabel.CENTER); + transferRateLabel.setHorizontalAlignment(JLabel.RIGHT); + transferRateLabel.setFont(monospaced); openButton = new JButton(); openButton.setVisible(false); JPanel statusPanel = new JPanel(new GridBagLayout()); - statusPanel.setBorder(emptyBorder); gbc.gridx = 0; + gbc.weightx = 1; statusPanel.add(statusLabel, gbc); + gbc.gridx = 1; + gbc.weightx = 0; + statusPanel.add(transferRateLabel, gbc); + gbc.gridx = 0; + gbc.weightx = 1; + gbc.gridwidth = 2; gbc.gridy = 1; statusPanel.add(openButton, gbc); gbc.gridy = 0; + gbc.gridwidth = 1; JPanel progressPanel = new JPanel(new GridBagLayout()); - progressPanel.setBorder(emptyBorder); statusProgress = new JProgressBar(0, 100); progressPanel.add(statusProgress, gbc); JPanel optionsPanel = new JPanel(new GridBagLayout()); - optionsPanel.setBorder(emptyBorder); optionLog = new JButton(Utils.getLocalizedString("Log")); optionHistory = new JButton(Utils.getLocalizedString("History")); optionQueue = new JButton(Utils.getLocalizedString("queue")); @@ -392,6 +431,13 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib } catch (Exception e) { LOGGER.warn(e.getMessage()); } + + // Prevent button sizes/positions from shifting when text bolds/unbolds + optionLog.setPreferredSize(optionLog.getPreferredSize()); + optionHistory.setPreferredSize(optionHistory.getPreferredSize()); + optionQueue.setPreferredSize(optionQueue.getPreferredSize()); + optionConfiguration.setPreferredSize(optionConfiguration.getPreferredSize()); + gbc.gridx = 0; optionsPanel.add(optionLog, gbc); gbc.gridx = 1; @@ -402,7 +448,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib optionsPanel.add(optionConfiguration, gbc); logPanel = new JPanel(new GridBagLayout()); - logPanel.setBorder(emptyBorder); logText = new JTextPane(); logText.setEditable(false); JScrollPane logTextScroll = new JScrollPane(logText); @@ -416,7 +461,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.weighty = 0; historyPanel = new JPanel(new GridBagLayout()); - historyPanel.setBorder(emptyBorder); historyPanel.setVisible(false); historyPanel.setPreferredSize(new Dimension(300, 250)); @@ -498,7 +542,6 @@ public void setValueAt(Object value, int row, int col) { gbc.ipady = 0; JPanel historyButtonPanel = new JPanel(new GridBagLayout()); historyButtonPanel.setSize(new Dimension(300, 10)); - historyButtonPanel.setBorder(emptyBorder); gbc.gridx = 0; historyButtonPanel.add(historyButtonRemove, gbc); gbc.gridx = 1; @@ -512,7 +555,6 @@ public void setValueAt(Object value, int row, int col) { historyPanel.add(historyButtonPanel, gbc); queuePanel = new JPanel(new GridBagLayout()); - queuePanel.setBorder(emptyBorder); queuePanel.setVisible(false); queuePanel.setPreferredSize(new Dimension(300, 250)); queueListModel = new DefaultListModel<>(); @@ -539,7 +581,6 @@ public void setValueAt(Object value, int row, int col) { gbc.ipady = 0; configurationPanel = new JPanel(new GridBagLayout()); - configurationPanel.setBorder(emptyBorder); configurationPanel.setVisible(false); // TODO Configuration components @@ -688,46 +729,8 @@ private void checkAndUpdate() { return field; } - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JLabel thing1ToAdd, - JButton thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JLabel thing1ToAdd, - JTextField thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JCheckBox thing1ToAdd, - JCheckBox thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - @SuppressWarnings("rawtypes") - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JCheckBox thing1ToAdd, - JComboBox thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - @SuppressWarnings("rawtypes") - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JComboBox thing1ToAdd, - JButton thing2ToAdd) { + private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JComponent thing1ToAdd, + JComponent thing2ToAdd) { gbc.gridy = gbcYValue; gbc.gridx = 0; configurationPanel.add(thing1ToAdd, gbc); @@ -735,13 +738,6 @@ private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYV configurationPanel.add(thing2ToAdd, gbc); } - @SuppressWarnings({ "unused", "rawtypes" }) - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JComboBox thing1ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - } - private void changeLocale() { statusLabel.setText(Utils.getLocalizedString("inactive")); configUpdateButton.setText(Utils.getLocalizedString("check.for.updates")); @@ -811,13 +807,32 @@ private void update() { stopButton.addActionListener(event -> { if (ripper != null) { ripper.stop(); - isRipping = false; + gracefulStop.set(true); + queueListModel.add(0, ripper.getURL().toString()); stopButton.setEnabled(false); statusProgress.setValue(0); statusProgress.setVisible(false); pack(); statusProgress.setValue(0); - status(Utils.getLocalizedString("download.interrupted")); + //status(Utils.getLocalizedString("download.interrupted")); + status("Rip gracefully stopping"); + appendLog("Download interrupted", Color.RED); + } + }); + + panicButton.addActionListener(event -> { + if (ripper != null) { + ripper.stop(); + ripper.panic(); + panicStop.set(true); + queueListModel.add(0, ripper.getURL().toString()); + stopButton.setEnabled(false); + panicButton.setEnabled(false); + statusProgress.setValue(0); + statusProgress.setVisible(false); + pack(); + statusProgress.setValue(0); + status("Rip interrupted"); // TODO localize appendLog("Download interrupted", Color.RED); } }); @@ -1070,10 +1085,7 @@ public void mouseClicked(MouseEvent e) { @Override public void intervalAdded(ListDataEvent arg0) { updateQueue(); - - if (!isRipping) { - ripNextAlbum(); - } + ripNextAlbum(); } @Override @@ -1087,24 +1099,14 @@ public void intervalRemoved(ListDataEvent arg0) { } private void setLogLevel(String level) { - // default level is error, set in case something else is given. - Level newLevel = Level.ERROR; level = level.substring(level.lastIndexOf(' ') + 1); - switch (level) { - case "Debug": - newLevel = Level.DEBUG; - break; - case "Info": - newLevel = Level.INFO; - break; - case "Warn": - newLevel = Level.WARN; - } - LoggerContext ctx = (LoggerContext) LogManager.getContext(false); - Configuration config = ctx.getConfiguration(); - LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); - loggerConfig.setLevel(newLevel); - ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig. + Level newLevel = switch (level) { + case "Debug" -> Level.DEBUG; + case "Info" -> Level.INFO; + case "Warn" -> Level.WARN; + default -> Level.ERROR; + }; + Utils.configureLogger(newLevel); } private void setupTrayIcon() { @@ -1257,7 +1259,11 @@ private void appendLog(final String text, final Color color) { StyledDocument sd = logText.getStyledDocument(); try { synchronized (this) { + if (logLineLengths.size() > MAX_LOG_PANE_LINES) { + sd.remove(0, logLineLengths.remove()); + } sd.insertString(sd.getLength(), text + "\n", sas); + logLineLengths.add(text.length() + 1); } } catch (BadLocationException e) { LOGGER.warn(e.getMessage()); @@ -1311,6 +1317,10 @@ private void loadHistory() throws IOException { }); } } + if (!HISTORY.isEmpty()) { + // Fix "WARNING: row index is bigger than sorter's row count. Most likely this is a wrong sorter usage" + historyTableModel.fireTableDataChanged(); + } } private void saveHistory() { @@ -1329,35 +1339,65 @@ private void saveHistory() { } private void ripNextAlbum() { - isRipping = true; + LOGGER.debug("ripNextAlbum called"); + if (isRipperActive.getAndSet(true)) { + // Already ripping + LOGGER.debug("already ripping"); + return; + } // Save current state of queue to configuration. Utils.setConfigList("queue", queueListModel.elements()); + boolean wasGracefulStop = gracefulStop.getAndSet(false); + boolean wasPanicStop = gracefulStop.getAndSet(false); + if (wasGracefulStop || wasPanicStop) { + // Stop requested + LOGGER.debug("wasGracefulStop or wasPanicStop"); + ripFinishCleanup(); + return; + } + if (queueListModel.isEmpty()) { // End of queue - isRipping = false; + ripFinishCleanup(); return; } + if (rateRefresherFuture == null || rateRefresherFuture.isDone()) { + rateRefresherFuture = executor.scheduleAtFixedRate(rateRefresher, 0, TRANSFER_RATE_REFRESH_RATE, TimeUnit.MILLISECONDS); + } + String nextAlbum = (String) queueListModel.remove(0); updateQueue(); + LOGGER.debug("calling ripAlbum(\"{}\")", nextAlbum); Thread t = ripAlbum(nextAlbum); if (t == null) { + LOGGER.debug("ripAlbum() returned null"); try { Thread.sleep(500); } catch (InterruptedException ie) { LOGGER.error(Utils.getLocalizedString("interrupted.while.waiting.to.rip.next.album"), ie); } + isRipperActive.set(false); ripNextAlbum(); } else { + LOGGER.debug("Starting new ripper thread"); t.start(); } } + private void ripFinishCleanup() { + stopButton.setEnabled(false); + panicButton.setEnabled(false); + isRipperActive.set(false); + statusProgress.setValue(0); + statusProgress.setVisible(false); + } + private Thread ripAlbum(String urlString) { if (!logPanel.isVisible()) { optionLog.doClick(); @@ -1378,12 +1418,15 @@ private Thread ripAlbum(String urlString) { return null; } stopButton.setEnabled(true); + panicButton.setEnabled(true); statusProgress.setValue(100); openButton.setVisible(false); statusLabel.setVisible(true); + transferRateLabel.setVisible(true); pack(); boolean failed = false; try { + LOGGER.debug("Creating ripper for url {}", url); ripper = AbstractRipper.getRipper(url); ripper.setup(); } catch (Exception e) { @@ -1487,9 +1530,7 @@ public void actionPerformed(ActionEvent event) { mainWindow.statusWithColor("This URL is already in queue: " + url, Color.ORANGE); ripTextfield.setText(""); } - else if(!mainWindow.isRipping){ - mainWindow.ripNextAlbum(); - } + mainWindow.ripNextAlbum(); } } @@ -1508,18 +1549,22 @@ public void run() { } private synchronized void handleEvent(StatusEvent evt) { - if (ripper.isStopped()) { + RipStatusMessage msg = evt.msg; + RipStatusMessage.STATUS status = msg.getStatus(); + + // CHUNK_BYTES is noisy, so handle it before any other computation + if (status == RipStatusMessage.STATUS.CHUNK_BYTES) { + transferRate.addChunk((Long) msg.getObject()); + transferRateLabel.setText(transferRate.formatHumanTransferRate()); return; } - RipStatusMessage msg = evt.msg; - int completedPercent = evt.ripper.getCompletionPercentage(); statusProgress.setValue(completedPercent); statusProgress.setVisible(true); status(evt.ripper.getStatusText()); - switch (msg.getStatus()) { + switch (status) { case LOADING_RESOURCE: case DOWNLOAD_STARTED: if (LOGGER.isEnabled(Level.INFO)) { @@ -1557,12 +1602,11 @@ private synchronized void handleEvent(StatusEvent evt) { if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } - stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); openButton.setVisible(false); + ripFinishCleanup(); pack(); statusWithColor("Error: " + msg.getObject(), Color.RED); + ripNextAlbum(); break; case RIP_COMPLETE: @@ -1590,9 +1634,7 @@ private synchronized void handleEvent(StatusEvent evt) { Utils.playSound("camera.wav"); } saveHistory(); - stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); + Utils.saveConfig(); openButton.setVisible(true); Path f = rsc.dir; String prettyFile = Utils.shortenPath(f); @@ -1649,6 +1691,7 @@ private synchronized void handleEvent(StatusEvent evt) { LOGGER.error(e); } }); + ripFinishCleanup(); pack(); ripNextAlbum(); break; @@ -1662,10 +1705,8 @@ private synchronized void handleEvent(StatusEvent evt) { if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } - stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); openButton.setVisible(false); + ripFinishCleanup(); pack(); statusWithColor("Error: " + msg.getObject(), Color.RED); break; diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java index f589e9dbb..3d4c1644f 100644 --- a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java +++ b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java @@ -16,6 +16,7 @@ public enum STATUS { DOWNLOAD_SKIP("Download Skipped"), TOTAL_BYTES("Total bytes"), COMPLETED_BYTES("Completed bytes"), + CHUNK_BYTES("Transferred bytes in last chunk"), RIP_ERRORED("Rip Errored"), NO_ALBUM_OR_USER("No album or user"); diff --git a/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java b/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java index 9237fea90..57ea0aa1b 100644 --- a/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java +++ b/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.uiUtils; -import javax.swing.*; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import javax.swing.text.JTextComponent; import java.awt.*; import java.awt.datatransfer.Clipboard; @@ -10,6 +12,8 @@ import java.io.IOException; public class ContextActionProtections { + private static final Logger logger = LogManager.getLogger(ContextActionProtections.class); + public static void pasteFromClipboard(JTextComponent textComponent) { Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard(); Transferable transferable = clipboard.getContents(new Object()); @@ -24,8 +28,8 @@ public static void pasteFromClipboard(JTextComponent textComponent) { // } // Set the text in the JTextField textComponent.setText(clipboardContent); - } catch (UnsupportedFlavorException | IOException unable_to_modify_text_on_paste) { - unable_to_modify_text_on_paste.printStackTrace(); + } catch (UnsupportedFlavorException | IOException e) { + logger.error("Unable to paste from clipboard", e); } } } diff --git a/src/main/java/com/rarchives/ripme/utils/TransferRate.java b/src/main/java/com/rarchives/ripme/utils/TransferRate.java new file mode 100644 index 000000000..9227160f3 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/utils/TransferRate.java @@ -0,0 +1,77 @@ +package com.rarchives.ripme.utils; + +import java.util.Deque; +import java.util.concurrent.ConcurrentLinkedDeque; + +public class TransferRate { + private final Deque transferQueue = new ConcurrentLinkedDeque<>(); + private int windowDurationMs = 10000; + + public void addChunk(long bytes) { + long now = System.currentTimeMillis(); + transferQueue.addFirst(new ChunkStamp(now, bytes)); + removeOldChunks(now); + } + + public double calculateBytesPerSecond() { + if (transferQueue.isEmpty()) { + return 0; + } + long totalBytes = 0; + ChunkStamp oldest = transferQueue.getLast(); + long now = System.currentTimeMillis(); + removeOldChunks(now); + if (transferQueue.isEmpty()) { + return 0; + } + for (ChunkStamp chunkStamp : transferQueue) { + totalBytes += chunkStamp.bytes; + } + long elapsedMs = now - oldest.timestampMs; + double elapsedSeconds = (double) elapsedMs / 1000; + if (elapsedSeconds <= 0) { + return 0; + } + return totalBytes / elapsedSeconds; + } + + public String formatHumanTransferRate() { + double bps = calculateBytesPerSecond(); + int giB = 1024 * 1024 * 1024; + int miB = 1024 * 1024; + int kiB = 1024; + if (bps >= giB) { + return String.format("%.2f GiB/s", bps / giB); + } else if (bps >= miB) { + return String.format("%.2f MiB/s", bps / miB); + } else if (bps >= kiB) { + return String.format("%.2f KiB/s", bps / kiB); + } else { + return String.format("%.2f B/s", bps); + } + } + + public void setWindowDurationMs(int windowDurationMs) { + this.windowDurationMs = windowDurationMs; + } + + public int getWindowDurationMs() { + return windowDurationMs; + } + + private void removeOldChunks(long now) { + while (!transferQueue.isEmpty() && transferQueue.getLast().timestampMs < now - windowDurationMs) { + transferQueue.removeLast(); + } + } + + private static class ChunkStamp { + long bytes; + long timestampMs; // millis + + ChunkStamp(long timestampMs, long bytes) { + this.bytes = bytes; + this.timestampMs = timestampMs; + } + } +} diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 36fa7273e..7d1325b2a 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -39,9 +39,11 @@ import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.commons.configuration2.ex.ConfigurationException; import org.apache.commons.configuration2.io.FileHandler; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.ConsoleAppender; import org.apache.logging.log4j.core.appender.RollingFileAppender; import org.apache.logging.log4j.core.appender.rolling.DefaultRolloverStrategy; import org.apache.logging.log4j.core.appender.rolling.SizeBasedTriggeringPolicy; @@ -50,6 +52,10 @@ import org.apache.logging.log4j.core.config.LoggerConfig; import com.rarchives.ripme.ripper.AbstractRipper; +import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; +import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory; +import org.apache.logging.log4j.core.config.builder.api.RootLoggerComponentBuilder; +import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; /** * Common utility functions used in various places throughout the project. @@ -603,36 +609,45 @@ public static void playSound(String filename) { } } + public static void configureLogger() { + configureLogger(Level.INFO); // default INFO level + } + /** * Configures root logger, either for FILE output or just console. */ - public static void configureLogger() { - LoggerContext ctx = (LoggerContext) LogManager.getContext(false); - Configuration config = ctx.getConfiguration(); - LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); + public static void configureLogger(Level level) { + ConfigurationBuilder builder = ConfigurationBuilderFactory.newConfigurationBuilder(); + + //builder.setStatusLevel(Level.DEBUG); + final String consoleAppenderName = "stdout"; + builder.add(builder.newAppender(consoleAppenderName, "CONSOLE") + .addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT) + .add(builder.newLayout("PatternLayout").addAttribute("pattern", "%-5level %c{1}: %msg%n%xEx")) + ); + + RootLoggerComponentBuilder rootLogger = builder.newRootLogger(level); + rootLogger.add(builder.newAppenderRef(consoleAppenderName)); // write to ripme.log file if checked in GUI boolean logSave = getConfigBoolean("log.save", false); if (logSave) { - LOGGER.debug("add rolling appender ripmelog"); - TriggeringPolicy tp = SizeBasedTriggeringPolicy.createPolicy("20M"); - DefaultRolloverStrategy rs = DefaultRolloverStrategy.newBuilder().withMax("2").build(); - RollingFileAppender rolling = RollingFileAppender.newBuilder() - .setName("ripmelog") - .withFileName("ripme.log") - .withFilePattern("%d{yyyy-MM-dd HH:mm:ss} %p %m%n") - .withPolicy(tp) - .withStrategy(rs) - .build(); - loggerConfig.addAppender(rolling, null, null); - } else { - LOGGER.debug("remove rolling appender ripmelog"); - if (config.getAppender("ripmelog") != null) { - config.getAppender("ripmelog").stop(); - } - loggerConfig.removeAppender("ripmelog"); + final String fileAppenderName = "rolling"; + builder.add(builder.newAppender(fileAppenderName, "RollingFile") + .addAttribute("fileName", "ripme.log") + .addAttribute("filePattern", "ripme-%d{yyyy-MM-dd}-%i.log.gz") + .add(builder.newLayout("PatternLayout").addAttribute("pattern", "%d %-5level %c{1}: %msg%n%xEx")) + .addComponent(builder.newComponent("Policies") + .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "20M"))) + ); + rootLogger.add(builder.newAppenderRef(fileAppenderName)); } - ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig. + + builder.add(rootLogger); + + Configuration configuration = builder.build(); + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + ctx.reconfigure(configuration); } /** @@ -809,7 +824,7 @@ public static String[] getSupportedLanguages() { } public static String getLocalizedString(String key) { - LOGGER.debug(String.format("Key %s in %s is: %s", key, getSelectedLanguage(), + LOGGER.trace(String.format("Key %s in %s is: %s", key, getSelectedLanguage(), resourceBundle.getString(key))); return resourceBundle.getString(key); } diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index 30aaffbab..444862664 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -87,4 +87,5 @@ http.status.exception = HTTP status exception exception.while.downloading.file = Exception while downloading file failed.to.download = Failed to download skipping = Skipping -file.already.exists = file already exists \ No newline at end of file +file.already.exists = file already exists +device.nospace = No space left on device \ No newline at end of file