package main.java.org.wikidata.analyzer.Fetcher;

import org.wikidata.wdtk.dumpfiles.MwDumpFile;
import org.wikidata.wdtk.dumpfiles.MwLocalDumpFile;
import org.wikidata.wdtk.dumpfiles.wmf.JsonOnlineDumpFile;
import org.wikidata.wdtk.util.DirectoryManager;
import org.wikidata.wdtk.util.DirectoryManagerImpl;
import org.wikidata.wdtk.util.WebResourceFetcher;
import org.wikidata.wdtk.util.WebResourceFetcherImpl;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

/**
 * @author Addshore
 */
public class DumpFetcher {

    protected File dataDirectory;

    public DumpFetcher(File dataDirectory) {
        this.dataDirectory = dataDirectory;
    }

    /**
     * Look for the most recent dump date online and try to retrieve as dump object with fallback:
     * 1 - Look for local dump copies (in a collection of locations)
     * 2 - Look online & download dumps
     *
     * @return MwDumpFile
     * @throws IOException
     */
    public MwDumpFile getMostRecentDump() throws IOException {
        DumpDateFetcher dateFetcher = new DumpDateFetcher();
        String latestDumpDate = dateFetcher.getLatestOnlineDumpDate();
        System.out.println("Latest dump date stamp is " + latestDumpDate);

        // Look for the dump in a list of possible local locations
        List<String> locationList = new ArrayList<>();
        //Local data dir location
        locationList.add(this.dataDirectory + "/dumpfiles/json-" + latestDumpDate + "/" + latestDumpDate + "-all.json.gz");
        //Labs dump location
        locationList.add("/public/dumps/public/wikidatawiki/entities/" + latestDumpDate + "/wikidata-" + latestDumpDate + "-all.json.gz");
        //Stat1002 dump location
        locationList.add("/mnt/data/xmldatadumps/public/wikidatawiki/entities/" + latestDumpDate + "/wikidata-" + latestDumpDate + "-all.json.gz");
        for (String dumpLocation: locationList) {
            if (Files.exists(Paths.get(dumpLocation)) && Files.isReadable(Paths.get(dumpLocation))) {
                MwLocalDumpFile localDumpFile = new MwLocalDumpFile( dumpLocation );
                if( localDumpFile.isAvailable() ) {
                    System.out.println("Using dump file from: " + dumpLocation);
                    return localDumpFile;
                }
            }
        }

        // Fallback to downloading the dump ourselves
        DirectoryManager localDirectoryManager = new DirectoryManagerImpl(
                Paths.get(this.dataDirectory.getAbsolutePath() + File.separator + "dumpfiles"),
                false
        );
        WebResourceFetcher fetcher = new WebResourceFetcherImpl();
        JsonOnlineDumpFile onlineDumpFile = new JsonOnlineDumpFile(
                latestDumpDate,
                "wikidatawiki",
                fetcher,
                localDirectoryManager
        );
        if (onlineDumpFile.isAvailable()) {
            System.out.println("Using online dump file");
            return onlineDumpFile;
        }

        throw new IOException("Failed to get most recent dump from any sources");
    }
}

@@ -769,208 +769,80 @@
   * 
-Look for the most recent dump date online and try to retrieve as dump object with fallback:%0A     * 1 - Look for local dump copies (in a collection of locations)%0A     * 2 - Look online & download dumps
+Get the latest dump using the dump data at the bottom of the online list
 %0A   
@@ -1113,65 +1113,464 @@
     
-System.out.println(%22Latest dump date stamp is %22 + latestD
+return this.getDump( latestDumpDate );%0A    %7D%0A%0A    /**%0A     * Look for the most recent dump date online and try to retrieve as dump object with fallback:%0A     * 1 - Look for local dump copies (in a collection of locations)%0A     * 2 - Look online & download dumps%0A     *%0A     * @return MwDumpFile%0A     * @throws IOException%0A     */%0A    public MwDumpFile getDump( String dumpDate ) throws IOException %7B%0A        System.out.println(%22Getting dump with date %22 + d
 umpD
@@ -1575,17 +1575,16 @@
 pDate);%0A
-%0A
         
@@ -1794,31 +1794,25 @@
 es/json-%22 + 
-latestD
+d
 umpDate + %22/
@@ -1811,31 +1811,25 @@
 ate + %22/%22 + 
-latestD
+d
 umpDate + %22-
@@ -1937,39 +1937,33 @@
 ki/entities/%22 + 
-latestD
+d
 umpDate + %22/wiki
@@ -1963,39 +1963,33 @@
  %22/wikidata-%22 + 
-latestD
+d
 umpDate + %22-all.
@@ -2110,31 +2110,25 @@
 ntities/%22 + 
-latestD
+d
 umpDate + %22/
@@ -2140,23 +2140,17 @@
 ata-%22 + 
-latestD
+d
 umpDate 
@@ -3032,23 +3032,17 @@
         
-latestD
+d
 umpDate,
@@ -3341,28 +3341,16 @@
  to get 
-most recent 
 dump fro

Allow getting a dump of a specific date