[lgogdownloader] 01/05: Imported Upstream version 2.24
Stephen Kitt
skitt at moszumanska.debian.org
Fri Jul 17 18:46:39 UTC 2015
This is an automated email from the git hooks/post-receive script.
skitt pushed a commit to branch master
in repository lgogdownloader.
commit 8511a808e7089d4f45d0398f6ffdbbeb1a81f5c3
Author: Stephen Kitt <steve at sk2.org>
Date: Tue Jul 14 14:00:20 2015 +0200
Imported Upstream version 2.24
---
Makefile | 4 +
include/config.h | 1 +
include/downloader.h | 6 +-
include/globalconstants.h | 48 +++---
include/util.h | 5 +-
main.cpp | 5 +-
man/lgogdownloader.supplemental.groff | 9 +-
src/api.cpp | 14 +-
src/downloader.cpp | 286 ++++++++++++++++++++++++++--------
src/util.cpp | 17 +-
10 files changed, 283 insertions(+), 112 deletions(-)
diff --git a/Makefile b/Makefile
index dadd708..9e7d4d7 100644
--- a/Makefile
+++ b/Makefile
@@ -75,6 +75,8 @@ debug: before_debug out_debug after_debug
out_debug: $(OBJ_DEBUG) $(DEP_DEBUG)
$(LD) $(LDFLAGS_DEBUG) $(LIBDIR_DEBUG) $(OBJ_DEBUG) $(LIB_DEBUG) -o $(OUT_DEBUG)
+$(OBJ_DEBUG): | before_debug
+
$(OBJDIR_DEBUG)/main.o: main.cpp
$(CXX) $(CFLAGS_DEBUG) $(VERSION) $(INC_DEBUG) -c main.cpp -o $(OBJDIR_DEBUG)/main.o
@@ -123,6 +125,8 @@ release: before_release out_release after_release
out_release: $(OBJ_RELEASE) $(DEP_RELEASE)
$(LD) $(LDFLAGS_RELEASE) $(LIBDIR_RELEASE) $(OBJ_RELEASE) $(LIB_RELEASE) -o $(OUT_RELEASE)
+$(OBJ_RELEASE): | before_release
+
$(OBJDIR_RELEASE)/main.o: main.cpp
$(CXX) $(CFLAGS_RELEASE) $(VERSION) $(INC_RELEASE) -c main.cpp -o $(OBJDIR_RELEASE)/main.o
diff --git a/include/config.h b/include/config.h
index 12e15a9..0e9f068 100644
--- a/include/config.h
+++ b/include/config.h
@@ -69,6 +69,7 @@ class Config
std::string sFileIdString;
std::string sLanguagePriority;
std::string sPlatformPriority;
+ std::string sIgnoreDLCCountRegex;
std::vector<unsigned int> vLanguagePriority;
std::vector<unsigned int> vPlatformPriority;
diff --git a/include/downloader.h b/include/downloader.h
index 3f676da..f305645 100644
--- a/include/downloader.h
+++ b/include/downloader.h
@@ -74,7 +74,7 @@ class Downloader
Timer timer;
Config config;
ProgressBar* progressbar;
- std::deque< std::pair<time_t, size_t> > TimeAndSize;
+ std::deque< std::pair<time_t, uintmax_t> > TimeAndSize;
protected:
private:
CURLcode downloadFile(const std::string& url, const std::string& filepath, const std::string& xml_data = std::string(), const std::string& gamename = std::string());
@@ -82,7 +82,7 @@ class Downloader
int downloadCovers(const std::string& gamename, const std::string& directory, const std::string& cover_xml_data);
int getGameDetails();
void getGameList();
- size_t getResumePosition();
+ uintmax_t getResumePosition();
CURLcode beginDownload();
std::string getResponse(const std::string& url);
std::string getLocalFileHash(const std::string& filepath, const std::string& gamename = std::string());
@@ -109,7 +109,7 @@ class Downloader
std::vector<gameDetails> games;
std::string coverXML;
- size_t resume_position;
+ off_t resume_position;
int retries;
std::ofstream report_ofs;
};
diff --git a/include/globalconstants.h b/include/globalconstants.h
index 811c3bf..975218f 100644
--- a/include/globalconstants.h
+++ b/include/globalconstants.h
@@ -13,25 +13,26 @@
namespace GlobalConstants
{
// Language constants
- const unsigned int LANGUAGE_EN = 1;
- const unsigned int LANGUAGE_DE = 2;
- const unsigned int LANGUAGE_FR = 4;
- const unsigned int LANGUAGE_PL = 8;
- const unsigned int LANGUAGE_RU = 16;
- const unsigned int LANGUAGE_CN = 32;
- const unsigned int LANGUAGE_CZ = 64;
- const unsigned int LANGUAGE_ES = 128;
- const unsigned int LANGUAGE_HU = 256;
- const unsigned int LANGUAGE_IT = 512;
- const unsigned int LANGUAGE_JP = 1024;
- const unsigned int LANGUAGE_TR = 2048;
- const unsigned int LANGUAGE_PT = 4096;
- const unsigned int LANGUAGE_KO = 8192;
- const unsigned int LANGUAGE_NL = 16384;
- const unsigned int LANGUAGE_SV = 32768;
- const unsigned int LANGUAGE_NO = 65536;
- const unsigned int LANGUAGE_DA = 131072;
- const unsigned int LANGUAGE_FI = 262144;
+ const unsigned int LANGUAGE_EN = 1 << 0;
+ const unsigned int LANGUAGE_DE = 1 << 1;
+ const unsigned int LANGUAGE_FR = 1 << 2;
+ const unsigned int LANGUAGE_PL = 1 << 3;
+ const unsigned int LANGUAGE_RU = 1 << 4;
+ const unsigned int LANGUAGE_CN = 1 << 5;
+ const unsigned int LANGUAGE_CZ = 1 << 6;
+ const unsigned int LANGUAGE_ES = 1 << 7;
+ const unsigned int LANGUAGE_HU = 1 << 8;
+ const unsigned int LANGUAGE_IT = 1 << 9;
+ const unsigned int LANGUAGE_JP = 1 << 10;
+ const unsigned int LANGUAGE_TR = 1 << 11;
+ const unsigned int LANGUAGE_PT = 1 << 12;
+ const unsigned int LANGUAGE_KO = 1 << 13;
+ const unsigned int LANGUAGE_NL = 1 << 14;
+ const unsigned int LANGUAGE_SV = 1 << 15;
+ const unsigned int LANGUAGE_NO = 1 << 16;
+ const unsigned int LANGUAGE_DA = 1 << 17;
+ const unsigned int LANGUAGE_FI = 1 << 18;
+ const unsigned int LANGUAGE_PT_BR = 1 << 19;
struct languageStruct {const unsigned int languageId; const std::string languageCode; const std::string languageString;};
const std::vector<languageStruct> LANGUAGES =
@@ -54,13 +55,14 @@ namespace GlobalConstants
{ LANGUAGE_SV, "sv", "Swedish" },
{ LANGUAGE_NO, "no", "Norwegian" },
{ LANGUAGE_DA, "da", "Danish" },
- { LANGUAGE_FI, "fi", "Finnish" }
+ { LANGUAGE_FI, "fi", "Finnish" },
+ { LANGUAGE_PT_BR, "br", "Brazilian Portuguese" }
};
// Platform constants
- const unsigned int PLATFORM_WINDOWS = 1;
- const unsigned int PLATFORM_MAC = 2;
- const unsigned int PLATFORM_LINUX = 4;
+ const unsigned int PLATFORM_WINDOWS = 1 << 0;
+ const unsigned int PLATFORM_MAC = 1 << 1;
+ const unsigned int PLATFORM_LINUX = 1 << 2;
struct platformStruct {const unsigned int platformId; const std::string platformCode; const std::string platformString;};
const std::vector<platformStruct> PLATFORMS =
diff --git a/include/util.h b/include/util.h
index 980460d..841b663 100644
--- a/include/util.h
+++ b/include/util.h
@@ -24,6 +24,7 @@ struct gameSpecificConfig
unsigned int iInstallerType;
unsigned int iInstallerLanguage;
bool bDLC;
+ bool bIgnoreDLCCount;
};
namespace Util
@@ -31,8 +32,8 @@ namespace Util
std::string makeFilepath(const std::string& directory, const std::string& path, const std::string& gamename, std::string subdirectory = "", const unsigned int& platformId = 0, const std::string& dlcname = "");
std::string makeRelativeFilepath(const std::string& path, const std::string& gamename, std::string subdirectory = "");
std::string getFileHash(const std::string& filename, unsigned hash_id);
- std::string getChunkHash(unsigned char* chunk, size_t chunk_size, unsigned hash_id);
- int createXML(std::string filepath, size_t chunk_size, std::string xml_dir = std::string());
+ std::string getChunkHash(unsigned char* chunk, uintmax_t chunk_size, unsigned hash_id);
+ int createXML(std::string filepath, uintmax_t chunk_size, std::string xml_dir = std::string());
int getGameSpecificConfig(std::string gamename, gameSpecificConfig* conf, std::string directory = std::string());
int replaceString(std::string& str, const std::string& to_replace, const std::string& replace_with);
void filepathReplaceReservedStrings(std::string& str, const std::string& gamename, const unsigned int& platformId = 0, const std::string& dlcname = "");
diff --git a/main.cpp b/main.cpp
index 81e1bc5..a59245d 100644
--- a/main.cpp
+++ b/main.cpp
@@ -13,7 +13,7 @@
#include <boost/filesystem.hpp>
#include <boost/program_options.hpp>
-#define VERSION_NUMBER "2.23"
+#define VERSION_NUMBER "2.24"
#ifndef VERSION_STRING
# define VERSION_STRING "LGOGDownloader " VERSION_NUMBER
@@ -143,7 +143,7 @@ int main(int argc, char *argv[])
("create-xml", bpo::value<std::string>(&config.sXMLFile)->default_value(""), "Create GOG XML for file\n\"automatic\" to enable automatic XML creation")
("update-check", bpo::value<bool>(&config.bUpdateCheck)->zero_tokens()->default_value(false), "Check for update notifications")
("check-orphans", bpo::value<std::string>(&config.sOrphanRegex)->implicit_value(""), check_orphans_text.c_str())
- ("status", bpo::value<bool>(&config.bCheckStatus)->zero_tokens()->default_value(false), "Show status of files\n\nOutput format:\nstatuscode gamename filename filesize filehash\n\nStatus codes:\nOK - File is OK\nND - File is not downloaded\nMD5 - MD5 mismatch, different version")
+ ("status", bpo::value<bool>(&config.bCheckStatus)->zero_tokens()->default_value(false), "Show status of files\n\nOutput format:\nstatuscode gamename filename filesize filehash\n\nStatus codes:\nOK - File is OK\nND - File is not downloaded\nMD5 - MD5 mismatch, different version\nFS - File size mismatch, incomplete download")
("save-config", bpo::value<bool>(&config.bSaveConfig)->zero_tokens()->default_value(false), "Create config file with current settings")
("reset-config", bpo::value<bool>(&config.bResetConfig)->zero_tokens()->default_value(false), "Reset config settings to default")
("report", bpo::value<std::string>(&config.sReportFilePath)->implicit_value("lgogdownloader-report.log"), "Save report of downloaded/repaired files to specified file\nDefault filename: lgogdownloader-report.log")
@@ -191,6 +191,7 @@ int main(int argc, char *argv[])
("language-priority", bpo::value<std::string>(&config.sLanguagePriority)->default_value(""), ("Set priority of systems" + priority_help_text + ", like \"4,1\" for French first, then English if no French version").c_str())
("platform-priority", bpo::value<std::string>(&config.sPlatformPriority)->default_value(""), ("Set priority of platforms" + priority_help_text + ", like \"4,1\" for Linux first, then Windows if no Linux version").c_str())
("save-serials", bpo::value<bool>(&config.bSaveSerials)->zero_tokens()->default_value(false), "Save serial numbers when downloading")
+ ("ignore-dlc-count", bpo::value<std::string>(&config.sIgnoreDLCCountRegex)->implicit_value(".*"), "Set regular expression filter for games to ignore DLC count information\nIgnoring DLC count information helps in situations where the account page doesn't provide accurate information about DLCs")
;
// Options read from config file
options_cfg_only.add_options()
diff --git a/man/lgogdownloader.supplemental.groff b/man/lgogdownloader.supplemental.groff
index 5f5ad1d..418e980 100644
--- a/man/lgogdownloader.supplemental.groff
+++ b/man/lgogdownloader.supplemental.groff
@@ -7,7 +7,7 @@ An open-source GOG.com downloader for Linux users which uses the same API as the
.PP
LGOGDownloader can download purchased games, query GOG.com to see if game files have changed, as well as downloading extras such as artwork and manuals. It is capable of downloading language-specific installers for games where they exist.
.PP
-These games are currently offered only for the Microsoft Windows\[rg] and Apple OS X\[rg] operating systems. To play these games under GNU/Linux will require a compatibility layer such as Wine. Usage of such a program is outside the scope of this document.
+These games are currently offered only for the Microsoft Windows\[rg] and Apple OS X\[rg] operating systems. To play these games under GNU/Linux will require a compatibility layer such as Wine. Usage of such a program is outside the scope of this document.
/--update-check/
.nf
@@ -70,7 +70,7 @@ It doesn't have to exist, but if it does exist, it must be readable to lgogdownl
\fI$XDG_CONFIG_HOME/lgogdownloader/gamename.conf\fP
JSON formatted file. Sets game specific settings for \fBgamename\fP.
.br
-Allowed settings are \fBlanguage\fP, \fBplatform\fP and \fBdlc\fP.
+Allowed settings are \fBlanguage\fP, \fBplatform\fP, \fBdlc\fP and \fBignore-dlc-count\fP.
.br
The \fBdlc\fP option is limited to disabling DLC for specific game. It can't enable DLC listing/downloading if \fB--no-dlc\fP option is used.
.br
@@ -79,7 +79,8 @@ Must be in the following format:
{
"language" : <int>,
"platform" : <int>,
- "dlc" : <bool>
+ "dlc" : <bool>,
+ "ignore-dlc-count" : <bool>
.br
}
@@ -87,7 +88,7 @@ Must be in the following format:
For both languages and platforms, the default behavior is to download all enabled ones.
The \fBlanguage-priority\fB and \fBplatform-priority\fB switches enable a priority-based mode: only the first matching one will be downloaded.
.PP
-For example, setting \fBlanguage\fB to 5 means both French and English will be downloaded (if available) for all games. Setting \fBlanguage-priority\fB to 4,1 means that the French version (and only that one) will be downloaded if available, and if not, the English version will be downloaded.
+For example, setting \fBlanguage\fB to 5 means both French and English will be downloaded (if available) for all games. Setting \fBlanguage-priority\fB to 4,1 means that the French version (and only that one) will be downloaded if available, and if not, the English version will be downloaded.
.PP
You're allowed to "stack" codes in the priority string if needed. If you set \fBlanguage-priority\fB 132,1 it means it'll download both Spanish (128) and French (4) versions if they are available, and the English (1) one only if none of French and Spanish are available.
.PP
diff --git a/src/api.cpp b/src/api.cpp
index d8b6eb8..28af5b1 100644
--- a/src/api.cpp
+++ b/src/api.cpp
@@ -22,7 +22,7 @@
size_t writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp) {
std::ostringstream *stream = (std::ostringstream*)userp;
- size_t count = size * nmemb;
+ std::streamsize count = (std::streamsize) size * nmemb;
stream->write(ptr, count);
return count;
}
@@ -90,7 +90,7 @@ int API::getAPIConfig()
#ifdef DEBUG
std::cerr << "DEBUG INFO (API::getAPIConfig)" << std::endl << json << std::endl;
#endif
- this->setError(jsonparser->getFormatedErrorMessages());
+ this->setError(jsonparser->getFormattedErrorMessages());
res = 0;
}
delete jsonparser;
@@ -195,7 +195,7 @@ int API::getUserDetails()
#ifdef DEBUG
std::cerr << "DEBUG INFO (API::getUserDetails)" << std::endl << json << std::endl;
#endif
- this->setError(jsonparser->getFormatedErrorMessages());
+ this->setError(jsonparser->getFormattedErrorMessages());
res = 0;
}
delete jsonparser;
@@ -554,7 +554,7 @@ gameDetails API::getGameDetails(const std::string& game_name, const unsigned int
#ifdef DEBUG
std::cerr << "DEBUG INFO (API::getGameDetails)" << std::endl << json << std::endl;
#endif
- this->setError(jsonparser->getFormatedErrorMessages());
+ this->setError(jsonparser->getFormattedErrorMessages());
}
delete jsonparser;
}
@@ -591,7 +591,7 @@ std::string API::getInstallerLink(const std::string& game_name, const std::strin
#ifdef DEBUG
std::cerr << "DEBUG INFO (API::getInstallerLink)" << std::endl << json << std::endl;
#endif
- this->setError(jsonparser->getFormatedErrorMessages());
+ this->setError(jsonparser->getFormattedErrorMessages());
}
delete jsonparser;
}
@@ -627,7 +627,7 @@ std::string API::getExtraLink(const std::string& game_name, const std::string& i
#ifdef DEBUG
std::cerr << "DEBUG INFO (API::getExtraLink)" << std::endl << json << std::endl;
#endif
- this->setError(jsonparser->getFormatedErrorMessages());
+ this->setError(jsonparser->getFormattedErrorMessages());
}
delete jsonparser;
}
@@ -676,7 +676,7 @@ std::string API::getXML(const std::string& game_name, const std::string& id)
#ifdef DEBUG
std::cerr << "DEBUG INFO (API::getXML)" << std::endl << json << std::endl;
#endif
- this->setError(jsonparser->getFormatedErrorMessages());
+ this->setError(jsonparser->getFormattedErrorMessages());
}
delete jsonparser;
}
diff --git a/src/downloader.cpp b/src/downloader.cpp
index 7574059..b86026e 100644
--- a/src/downloader.cpp
+++ b/src/downloader.cpp
@@ -30,6 +30,9 @@ namespace bptime = boost::posix_time;
Downloader::Downloader(Config &conf)
{
this->config = conf;
+ if (config.bLogin && boost::filesystem::exists(config.sCookiePath))
+ if (!boost::filesystem::remove(config.sCookiePath))
+ std::cout << "Failed to delete " << config.sCookiePath << std::endl;
}
Downloader::~Downloader()
@@ -73,6 +76,10 @@ int Downloader::init()
curl_easy_setopt(curlhandle, CURLOPT_PROGRESSFUNCTION, Downloader::progressCallback);
curl_easy_setopt(curlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, config.iDownloadRate);
+ // Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds
+ curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_TIME, 30);
+ curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_LIMIT, 200);
+
// Create new API handle and set curl options for the API
gogAPI = new API(config.sToken, config.sSecret);
gogAPI->curlSetOpt(CURLOPT_VERBOSE, config.bVerbose);
@@ -175,31 +182,13 @@ void Downloader::updateCheck()
void Downloader::getGameList()
{
- gameItems = this->getGames();
-
- // Filter the game list
- if (!config.sGameRegex.empty())
+ if (config.sGameRegex == "free")
{
- // GameRegex filter aliases
- if (config.sGameRegex == "all")
- config.sGameRegex = ".*";
-
- if (config.sGameRegex == "free")
- {
- gameItems = this->getFreeGames();
- }
- else
- { // Filter the names
- std::vector<gameItem> gameItemsFiltered;
- boost::regex expression(config.sGameRegex);
- boost::match_results<std::string::const_iterator> what;
- for (unsigned int i = 0; i < gameItems.size(); ++i)
- {
- if (boost::regex_search(gameItems[i].name, what, expression)) // Check if name matches the specified regex
- gameItemsFiltered.push_back(gameItems[i]);
- }
- gameItems = gameItemsFiltered;
- }
+ gameItems = this->getFreeGames();
+ }
+ else
+ {
+ gameItems = this->getGames();
}
}
@@ -249,12 +238,13 @@ int Downloader::getGameDetails()
gameSpecificConfig conf;
conf.bDLC = config.bDLC;
+ conf.bIgnoreDLCCount = false;
conf.iInstallerLanguage = config.iInstallerLanguage;
conf.iInstallerType = config.iInstallerType;
if (!config.bUpdateCache) // Disable game specific config files for cache update
{
if (Util::getGameSpecificConfig(gameItems[i].name, &conf) > 0)
- std::cout << std::endl << gameItems[i].name << " - Language: " << conf.iInstallerLanguage << ", Platform: " << conf.iInstallerType << ", DLC: " << (conf.bDLC ? "true" : "false") << std::endl;
+ std::cout << std::endl << gameItems[i].name << " - Language: " << conf.iInstallerLanguage << ", Platform: " << conf.iInstallerType << ", DLC: " << (conf.bDLC ? "true" : "false") << ", Ignore DLC count: " << (conf.bIgnoreDLCCount ? "true" : "false") << std::endl;
}
game = gogAPI->getGameDetails(gameItems[i].name, conf.iInstallerType, conf.iInstallerLanguage, config.bDuplicateHandler);
@@ -274,6 +264,17 @@ int Downloader::getGameDetails()
gameDetailsJSON = this->getGameDetailsJSON(gameItems[i].id);
game.serials = this->getSerialsFromJSON(gameDetailsJSON);
}
+
+ // Ignore DLC count and try to get DLCs from JSON
+ if (game.dlcs.empty() && !bHasDLC && conf.bDLC && conf.bIgnoreDLCCount)
+ {
+ if (gameDetailsJSON.empty())
+ gameDetailsJSON = this->getGameDetailsJSON(gameItems[i].id);
+
+ gameItems[i].dlcnames = Util::getDLCNamesFromJSON(gameDetailsJSON["dlcs"]);
+ bHasDLC = !gameItems[i].dlcnames.empty();
+ }
+
if (game.dlcs.empty() && bHasDLC && conf.bDLC)
{
for (unsigned int j = 0; j < gameItems[i].dlcnames.size(); ++j)
@@ -1160,7 +1161,7 @@ CURLcode Downloader::downloadFile(const std::string& url, const std::string& fil
CURLcode res = CURLE_RECV_ERROR; // assume network error
bool bResume = false;
FILE *outfile;
- size_t offset=0;
+ off_t offset=0;
// Get directory from filepath
boost::filesystem::path pathname = filepath;
@@ -1227,8 +1228,9 @@ CURLcode Downloader::downloadFile(const std::string& url, const std::string& fil
{
bResume = true;
fseek(outfile, 0, SEEK_END);
- offset = ftell(outfile);
- curl_easy_setopt(curlhandle, CURLOPT_RESUME_FROM, offset);
+ // use ftello to support large files on 32 bit platforms
+ offset = ftello(outfile);
+ curl_easy_setopt(curlhandle, CURLOPT_RESUME_FROM_LARGE, offset);
this->resume_position = offset;
}
else
@@ -1341,9 +1343,18 @@ CURLcode Downloader::downloadFile(const std::string& url, const std::string& fil
}
// Retry partially downloaded file
- if (res == CURLE_PARTIAL_FILE && (this->retries < config.iRetries) )
+ // Retry if we aborted the transfer due to low speed limit
+ if ((res == CURLE_PARTIAL_FILE || res == CURLE_OPERATION_TIMEDOUT) && (this->retries < config.iRetries) )
{
this->retries++;
+
+ std::cerr << std::endl << "Retry " << this->retries << "/" << config.iRetries;
+ if (res == CURLE_PARTIAL_FILE)
+ std::cerr << " (partial download)";
+ else if (res == CURLE_OPERATION_TIMEDOUT)
+ std::cerr << " (timeout)";
+ std::cerr << std::endl;
+
res = this->downloadFile(url, filepath, xml_data, gamename);
}
else
@@ -1359,10 +1370,10 @@ int Downloader::repairFile(const std::string& url, const std::string& filepath,
{
int res = 0;
FILE *outfile;
- size_t offset=0, from_offset, to_offset, filesize;
+ off_t offset=0, from_offset, to_offset, filesize;
std::string filehash;
int chunks;
- std::vector<size_t> chunk_from, chunk_to;
+ std::vector<off_t> chunk_from, chunk_to;
std::vector<std::string> chunk_hash;
bool bParsingFailed = false;
@@ -1468,7 +1479,8 @@ int Downloader::repairFile(const std::string& url, const std::string& filepath,
if ((outfile = fopen(filepath.c_str(), "r+"))!=NULL )
{
fseek(outfile, 0, SEEK_END);
- offset = ftell(outfile);
+ // use ftello to support large files on 32 bit platforms
+ offset = ftello(outfile);
}
else
{
@@ -1496,7 +1508,8 @@ int Downloader::repairFile(const std::string& url, const std::string& filepath,
}
return res;
}
-
+
+ // check if file sizes match
if (offset != filesize)
{
std::cout << "Filesizes don't match" << std::endl
@@ -1519,12 +1532,32 @@ int Downloader::repairFile(const std::string& url, const std::string& filepath,
}
else
{
+ if (bLocalXMLExists)
+ {
+ std::cout << "Deleting old XML data" << std::endl;
+ boost::filesystem::remove(xml_file, ec); // Delete old XML data
+ if (ec)
+ {
+ std::cout << "Failed to delete " << xml_file << std::endl;
+ }
+ }
+
CURLcode result = this->downloadFile(url, filepath, xml_data, gamename);
std::cout << std::endl;
if (result == CURLE_OK)
+ {
+ bLocalXMLExists = boost::filesystem::exists(xml_file); // Check to see if downloadFile saved XML data
+ if (!bLocalXMLExists)
+ {
+ std::cout << "Starting automatic XML creation" << std::endl;
+ Util::createXML(filepath, config.iChunkSize, xml_directory);
+ }
res = 1;
+ }
else
+ {
res = 0;
+ }
}
}
return res;
@@ -1534,13 +1567,14 @@ int Downloader::repairFile(const std::string& url, const std::string& filepath,
int iChunksRepaired = 0;
for (int i=0; i<chunks; i++)
{
- size_t chunk_begin = chunk_from.at(i);
- size_t chunk_end = chunk_to.at(i);
- size_t size=0, chunk_size = chunk_end - chunk_begin + 1;
+ off_t chunk_begin = chunk_from.at(i);
+ off_t chunk_end = chunk_to.at(i);
+ off_t size=0, chunk_size = chunk_end - chunk_begin + 1;
std::string range = std::to_string(chunk_begin) + "-" + std::to_string(chunk_end); // Download range string for curl
std::cout << "\033[0K\rChunk " << i << " (" << chunk_size << " bytes): ";
- fseek(outfile, chunk_begin, SEEK_SET);
+ // use fseeko to support large files on 32 bit platforms
+ fseeko(outfile, chunk_begin, SEEK_SET);
unsigned char *chunk = (unsigned char *) malloc(chunk_size * sizeof(unsigned char *));
if (chunk == NULL)
{
@@ -1560,7 +1594,8 @@ int Downloader::repairFile(const std::string& url, const std::string& filepath,
if (hash != chunk_hash.at(i))
{
std::cout << "Failed - downloading chunk" << std::endl;
- fseek(outfile, chunk_begin, SEEK_SET);
+ // use fseeko to support large files on 32 bit platforms
+ fseeko(outfile, chunk_begin, SEEK_SET);
curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str());
curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, outfile);
curl_easy_setopt(curlhandle, CURLOPT_RANGE, range.c_str()); //download range
@@ -1772,14 +1807,14 @@ int Downloader::progressCallback(void *clientp, double dltotal, double dlnow, do
// 10 second average download speed
// Don't use static value of 10 seconds because update interval depends on when and how often progress callback is called
- downloader->TimeAndSize.push_back(std::make_pair(time(NULL), static_cast<size_t>(dlnow)));
+ downloader->TimeAndSize.push_back(std::make_pair(time(NULL), static_cast<uintmax_t>(dlnow)));
if (downloader->TimeAndSize.size() > 100) // 100 * 100ms = 10s
{
downloader->TimeAndSize.pop_front();
time_t time_first = downloader->TimeAndSize.front().first;
- size_t size_first = downloader->TimeAndSize.front().second;
+ uintmax_t size_first = downloader->TimeAndSize.front().second;
time_t time_last = downloader->TimeAndSize.back().first;
- size_t size_last = downloader->TimeAndSize.back().second;
+ uintmax_t size_last = downloader->TimeAndSize.back().second;
rate = (size_last - size_first) / static_cast<double>((time_last - time_first));
}
@@ -1860,7 +1895,7 @@ size_t Downloader::readData(void *ptr, size_t size, size_t nmemb, FILE *stream)
return fread(ptr, size, nmemb, stream);
}
-size_t Downloader::getResumePosition()
+uintmax_t Downloader::getResumePosition()
{
return this->resume_position;
}
@@ -2076,8 +2111,16 @@ std::vector<gameItem> Downloader::getGames()
#ifdef DEBUG
std::cerr << "DEBUG INFO (Downloader::getGames)" << std::endl << response << std::endl;
#endif
- std::cout << jsonparser->getFormatedErrorMessages();
+ std::cout << jsonparser->getFormattedErrorMessages();
delete jsonparser;
+ if (!response.empty())
+ {
+ if(response[0] != '{')
+ {
+ // Response was not JSON. Assume that cookies have expired.
+ std::cerr << "Response was not JSON. Cookies have most likely expired. Try --login first." << std::endl;
+ }
+ }
exit(1);
}
#ifdef DEBUG
@@ -2106,10 +2149,47 @@ std::vector<gameItem> Downloader::getGames()
if (config.bPlatformDetection && !(platform & config.iInstallerType))
continue;
+ // Filter the game list
+ if (!config.sGameRegex.empty())
+ {
+ // GameRegex filter aliases
+ if (config.sGameRegex == "all")
+ config.sGameRegex = ".*";
+
+ boost::regex expression(config.sGameRegex);
+ boost::match_results<std::string::const_iterator> what;
+ if (!boost::regex_search(game.name, what, expression)) // Check if name matches the specified regex
+ continue;
+ }
+
if (config.bDLC)
{
int dlcCount = product["dlcCount"].asInt();
- if (dlcCount != 0)
+
+ bool bDownloadDLCInfo = (dlcCount != 0);
+
+ if (!bDownloadDLCInfo && !config.sIgnoreDLCCountRegex.empty())
+ {
+ boost::regex expression(config.sIgnoreDLCCountRegex);
+ boost::match_results<std::string::const_iterator> what;
+ if (boost::regex_search(game.name, what, expression)) // Check if name matches the specified regex
+ {
+ bDownloadDLCInfo = true;
+ }
+ }
+
+ if (bDownloadDLCInfo && !config.sGameRegex.empty())
+ {
+ // don't download unnecessary info if user is only interested in a subset of his account
+ boost::regex expression(config.sGameRegex);
+ boost::match_results<std::string::const_iterator> what;
+ if (!boost::regex_search(game.name, what, expression))
+ {
+ bDownloadDLCInfo = false;
+ }
+ }
+
+ if (bDownloadDLCInfo)
{
std::string gameinfo = this->getResponse("https://www.gog.com/account/gameDetails/" + game.id + ".json");
Json::Value info;
@@ -2118,7 +2198,7 @@ std::vector<gameItem> Downloader::getGames()
#ifdef DEBUG
std::cerr << "DEBUG INFO (Downloader::getGames)" << std::endl << gameinfo << std::endl;
#endif
- std::cout << jsonparser->getFormatedErrorMessages();
+ std::cout << jsonparser->getFormattedErrorMessages();
delete jsonparser;
exit(1);
}
@@ -2156,7 +2236,7 @@ std::vector<gameItem> Downloader::getFreeGames()
#ifdef DEBUG
std::cerr << "DEBUG INFO (Downloader::getFreeGames)" << std::endl << json << std::endl;
#endif
- std::cout << jsonparser->getFormatedErrorMessages();
+ std::cout << jsonparser->getFormattedErrorMessages();
delete jsonparser;
exit(1);
}
@@ -2190,7 +2270,7 @@ Json::Value Downloader::getGameDetailsJSON(const std::string& gameid)
#ifdef DEBUG
std::cerr << "DEBUG INFO (Downloader::getGameDetailsJSON)" << std::endl << json << std::endl;
#endif
- std::cout << jsonparser->getFormatedErrorMessages();
+ std::cout << jsonparser->getFormattedErrorMessages();
delete jsonparser;
exit(1);
}
@@ -2230,9 +2310,13 @@ std::vector<gameFile> Downloader::getExtrasFromJSON(const Json::Value& json, con
path = "/" + gamename + "/extras/" + path;
}
- // Get name from path if name was not specified
+ // Get filename
+ std::string filename;
+ filename.assign(path.begin()+path.find_last_of("/")+1,path.end());
+
+ // Use filename if name was not specified
if (name.empty())
- name.assign(path.begin()+path.find_last_of("/")+1,path.end());
+ name = filename;
if (name.empty())
{
@@ -2243,6 +2327,15 @@ std::vector<gameFile> Downloader::getExtrasFromJSON(const Json::Value& json, con
continue;
}
+ if (filename.empty())
+ {
+ #ifdef DEBUG
+ std::cerr << "DEBUG INFO (getExtrasFromJSON)" << std::endl;
+ std::cerr << "Skipped file without a filename (game: " << gamename << ", fileid: " << id << ", name: " << name << ")" << std::endl;
+ #endif
+ continue;
+ }
+
extras.push_back(
gameFile ( false,
id,
@@ -2256,7 +2349,6 @@ std::vector<gameFile> Downloader::getExtrasFromJSON(const Json::Value& json, con
return extras;
}
-
std::string Downloader::getSerialsFromJSON(const Json::Value& json)
{
std::ostringstream serials;
@@ -2509,7 +2601,7 @@ void Downloader::checkStatus()
std::string remoteHash;
std::string localHash;
bool bHashOK = true; // assume hash OK
- size_t filesize;
+ uintmax_t filesize;
localHash = this->getLocalFileHash(filepath.string(), games[i].gamename);
remoteHash = this->getRemoteFileHash(games[i].gamename, games[i].installers[j].id);
@@ -2520,6 +2612,38 @@ void Downloader::checkStatus()
if (remoteHash != localHash)
bHashOK = false;
+ else
+ {
+ // Check for incomplete file by comparing the filesizes
+ // Remote hash was saved but download was incomplete and therefore getLocalFileHash returned the same as getRemoteFileHash
+ uintmax_t filesize_xml = 0;
+ boost::filesystem::path path = filepath;
+ boost::filesystem::path local_xml_file;
+ if (!games[i].gamename.empty())
+ local_xml_file = config.sXMLDirectory + "/" + games[i].gamename + "/" + path.filename().string() + ".xml";
+ else
+ local_xml_file = config.sXMLDirectory + "/" + path.filename().string() + ".xml";
+
+ if (boost::filesystem::exists(local_xml_file))
+ {
+ TiXmlDocument local_xml;
+ local_xml.LoadFile(local_xml_file.string());
+ TiXmlNode *fileNodeLocal = local_xml.FirstChild("file");
+ if (fileNodeLocal)
+ {
+ TiXmlElement *fileElemLocal = fileNodeLocal->ToElement();
+ std::string filesize_xml_str = fileElemLocal->Attribute("total_size");
+ filesize_xml = std::stoull(filesize_xml_str);
+ }
+ }
+
+ if (filesize_xml > 0 && filesize_xml != filesize)
+ {
+ localHash = Util::getFileHash(path.string(), RHASH_MD5);
+ std::cout << "FS " << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
+ continue;
+ }
+ }
std::cout << (bHashOK ? "OK " : "MD5 ") << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
}
@@ -2537,7 +2661,7 @@ void Downloader::checkStatus()
boost::filesystem::path filepath = games[i].extras[j].getFilepath();
std::string localHash = this->getLocalFileHash(filepath.string(), games[i].gamename);
- size_t filesize;
+ uintmax_t filesize;
if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath))
{
@@ -2558,7 +2682,7 @@ void Downloader::checkStatus()
boost::filesystem::path filepath = games[i].patches[j].getFilepath();
std::string localHash = this->getLocalFileHash(filepath.string(), games[i].gamename);
- size_t filesize;
+ uintmax_t filesize;
if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath))
{
@@ -2579,7 +2703,7 @@ void Downloader::checkStatus()
boost::filesystem::path filepath = games[i].languagepacks[j].getFilepath();
std::string localHash = this->getLocalFileHash(filepath.string(), games[i].gamename);
- size_t filesize;
+ uintmax_t filesize;
if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath))
{
@@ -2606,7 +2730,7 @@ void Downloader::checkStatus()
std::string remoteHash;
std::string localHash;
bool bHashOK = true; // assume hash OK
- size_t filesize;
+ uintmax_t filesize;
localHash = this->getLocalFileHash(filepath.string(), games[i].dlcs[j].gamename);
remoteHash = this->getRemoteFileHash(games[i].dlcs[j].gamename, games[i].dlcs[j].installers[k].id);
@@ -2617,12 +2741,44 @@ void Downloader::checkStatus()
if (remoteHash != localHash)
bHashOK = false;
+ else
+ {
+ // Check for incomplete file by comparing the filesizes
+ // Remote hash was saved but download was incomplete and therefore getLocalFileHash returned the same as getRemoteFileHash
+ uintmax_t filesize_xml = 0;
+ boost::filesystem::path path = filepath;
+ boost::filesystem::path local_xml_file;
+ if (!games[i].dlcs[j].gamename.empty())
+ local_xml_file = config.sXMLDirectory + "/" + games[i].dlcs[j].gamename + "/" + path.filename().string() + ".xml";
+ else
+ local_xml_file = config.sXMLDirectory + "/" + path.filename().string() + ".xml";
+
+ if (boost::filesystem::exists(local_xml_file))
+ {
+ TiXmlDocument local_xml;
+ local_xml.LoadFile(local_xml_file.string());
+ TiXmlNode *fileNodeLocal = local_xml.FirstChild("file");
+ if (fileNodeLocal)
+ {
+ TiXmlElement *fileElemLocal = fileNodeLocal->ToElement();
+ std::string filesize_xml_str = fileElemLocal->Attribute("total_size");
+ filesize_xml = std::stoull(filesize_xml_str);
+ }
+ }
+
+ if (filesize_xml > 0 && filesize_xml != filesize)
+ {
+ localHash = Util::getFileHash(path.string(), RHASH_MD5);
+ std::cout << "FS " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
+ continue;
+ }
+ }
- std::cout << (bHashOK ? "OK " : "MD5 ") << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
+ std::cout << (bHashOK ? "OK " : "MD5 ") << games[i].dlcs[j].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
}
else
{
- std::cout << "ND " << games[i].gamename << " " << filepath.filename().string() << std::endl;
+ std::cout << "ND " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << std::endl;
}
}
}
@@ -2634,16 +2790,16 @@ void Downloader::checkStatus()
boost::filesystem::path filepath = games[i].dlcs[j].patches[k].getFilepath();
std::string localHash = this->getLocalFileHash(filepath.string(), games[i].dlcs[j].gamename);
- size_t filesize;
+ uintmax_t filesize;
if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath))
{
filesize = boost::filesystem::file_size(filepath);
- std::cout << "OK " << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
+ std::cout << "OK " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
}
else
{
- std::cout << "ND " << games[i].gamename << " " << filepath.filename().string() << std::endl;
+ std::cout << "ND " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << std::endl;
}
}
}
@@ -2655,16 +2811,16 @@ void Downloader::checkStatus()
boost::filesystem::path filepath = games[i].dlcs[j].extras[k].getFilepath();
std::string localHash = this->getLocalFileHash(filepath.string(), games[i].dlcs[j].gamename);
- size_t filesize;
+ uintmax_t filesize;
if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath))
{
filesize = boost::filesystem::file_size(filepath);
- std::cout << "OK " << games[i].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
+ std::cout << "OK " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << " " << filesize << " " << localHash << std::endl;
}
else
{
- std::cout << "ND " << games[i].gamename << " " << filepath.filename().string() << std::endl;
+ std::cout << "ND " << games[i].dlcs[j].gamename << " " << filepath.filename().string() << std::endl;
}
}
}
@@ -2781,7 +2937,7 @@ int Downloader::loadGameDetailsCache()
{
res = 2;
std::cout << "Failed to parse cache" << std::endl;
- std::cout << jsonparser->getFormatedErrorMessages() << std::endl;
+ std::cout << jsonparser->getFormattedErrorMessages() << std::endl;
}
delete jsonparser;
if (json)
@@ -3048,7 +3204,7 @@ void Downloader::showWishlist()
#ifdef DEBUG
std::cerr << "DEBUG INFO (Downloader::showWishlist)" << std::endl << response << std::endl;
#endif
- std::cout << jsonparser->getFormatedErrorMessages();
+ std::cout << jsonparser->getFormattedErrorMessages();
delete jsonparser;
exit(1);
}
diff --git a/src/util.cpp b/src/util.cpp
index 29aadf2..1ee814b 100644
--- a/src/util.cpp
+++ b/src/util.cpp
@@ -71,7 +71,7 @@ std::string Util::getFileHash(const std::string& filename, unsigned hash_id)
return result;
}
-std::string Util::getChunkHash(unsigned char *chunk, size_t chunk_size, unsigned hash_id)
+std::string Util::getChunkHash(unsigned char *chunk, uintmax_t chunk_size, unsigned hash_id)
{
unsigned char digest[rhash_get_digest_size(hash_id)];
char result[rhash_get_hash_length(hash_id)];
@@ -87,12 +87,12 @@ std::string Util::getChunkHash(unsigned char *chunk, size_t chunk_size, unsigned
}
// Create GOG XML
-int Util::createXML(std::string filepath, size_t chunk_size, std::string xml_dir)
+int Util::createXML(std::string filepath, uintmax_t chunk_size, std::string xml_dir)
{
int res = 0;
FILE *infile;
FILE *xmlfile;
- size_t filesize, size;
+ uintmax_t filesize, size;
int chunks, i;
if (xml_dir.empty())
@@ -158,11 +158,11 @@ int Util::createXML(std::string filepath, size_t chunk_size, std::string xml_dir
char rhash_result[rhash_get_hash_length(RHASH_MD5)];
for (i = 0; i < chunks; i++) {
- size_t range_begin = i*chunk_size;
+ uintmax_t range_begin = i*chunk_size;
fseek(infile, range_begin, SEEK_SET);
if ((i == chunks-1) && (remaining != 0))
chunk_size = remaining;
- size_t range_end = range_begin + chunk_size - 1;
+ uintmax_t range_end = range_begin + chunk_size - 1;
unsigned char *chunk = (unsigned char *) malloc(chunk_size * sizeof(unsigned char *));
if (chunk == NULL)
{
@@ -267,11 +267,16 @@ int Util::getGameSpecificConfig(std::string gamename, gameSpecificConfig* conf,
conf->bDLC = root["dlc"].asBool();
res++;
}
+ if (root.isMember("ignore-dlc-count"))
+ {
+ conf->bIgnoreDLCCount = root["ignore-dlc-count"].asBool();
+ res++;
+ }
}
else
{
std::cout << "Failed to parse game specific config" << std::endl;
- std::cout << jsonparser->getFormatedErrorMessages() << std::endl;
+ std::cout << jsonparser->getFormattedErrorMessages() << std::endl;
}
delete jsonparser;
if (json)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/lgogdownloader.git
More information about the Pkg-games-commits
mailing list