diff --git a/UpdatePTUDevCtrl/process/command.cpp b/UpdatePTUDevCtrl/process/command.cpp index b0b8747..0b06e7e 100644 --- a/UpdatePTUDevCtrl/process/command.cpp +++ b/UpdatePTUDevCtrl/process/command.cpp @@ -151,12 +151,34 @@ bool Command::execute(QString workingDirectory, QStringList args) { // qDebug() << "PROCESS" << m_command << "STARTED IN" << p->workingDirectory(); if (p->state() == QProcess::ProcessState::Running) { // qDebug() << "PROCESS" << m_command << "RUNNING IN" << p->workingDirectory(); - // wait forever for git/opkg-commands to finish + // wait forever for git/opkg-commands to finish ------------------- wait for 30 min max! int wait = m_waitForFinishTimeout; if (m_command.trimmed().startsWith("git", Qt::CaseInsensitive) || m_command.trimmed().startsWith("opkg", Qt::CaseInsensitive)) { - wait = -1; + wait = 30 * 60 * 1000; } + + if (!p->waitForFinished(wait)) { + qCritical() << "PROCESS TIMEOUT:" << m_command; + + p->terminate(); + + if (!p->waitForFinished(5000)) { + qCritical() << "Process did not terminate, killing:" + << m_command; + + p->kill(); + + if (!p->waitForFinished(5000)) { + qCritical() << "ERROR: Process still did not finish after kill:" + << m_command; + } + } + + m_exitCode = -1; + return false; + } + bool const no_timeout = p->waitForFinished(wait); if (no_timeout) { // qDebug() << "PROCESS" << m_command << "FINISHED IN" << p->workingDirectory(); diff --git a/UpdatePTUDevCtrl/process/command.h b/UpdatePTUDevCtrl/process/command.h index 71efb22..989a96e 100644 --- a/UpdatePTUDevCtrl/process/command.h +++ b/UpdatePTUDevCtrl/process/command.h @@ -22,9 +22,11 @@ class Command : public QObject { QProcess *m_p; Worker *m_worker; public: + static constexpr int COMMAND_START_TIMEOUT_MS = 30 * 1000; + static constexpr int COMMAND_FINISH_TIMEOUT_MS = 30 * 60 * 1000; explicit Command(QString const &command, - int start_timeout = 100000, - int finish_timeout = 100000); + int start_timeout = COMMAND_START_TIMEOUT_MS, + int finish_timeout = COMMAND_FINISH_TIMEOUT_MS); QString getCommandResult(bool reset = false) const; QString command() const { return m_command; } diff --git a/UpdatePTUDevCtrl/worker.cpp b/UpdatePTUDevCtrl/worker.cpp index a5e25c9..89160ec 100644 --- a/UpdatePTUDevCtrl/worker.cpp +++ b/UpdatePTUDevCtrl/worker.cpp @@ -284,7 +284,38 @@ bool Worker::repairCorruptedRepository() { return true; } +bool Worker::updateTimedOut() const +{ + if (!m_updateRuntime.isValid()) + return false; + + if (m_updateRuntime.elapsed() < UPDATE_MAX_RUNTIME_MS) + return false; + + qCritical() << "UPDATE TOOL GLOBAL TIMEOUT:" + << m_updateRuntime.elapsed() + << "ms"; + + return true; +} + + + void Worker::privateUpdate() { + m_updateRuntime.start(); + + auto abortIfTimedOut = [this]() -> bool { + if (!updateTimedOut()) + return false; + + qCritical() << "UPDATE TOOL GLOBAL TIMEOUT - aborting update"; + + // Ideally use a dedicated timeout step later. + m_lastFailedUpdateStep = UPDATE_STEP::UPDATE_FINALIZE; + + return true; + }; + if (!m_mainWindow) { Utils::printCriticalErrorMsg("m_mainWindow NOT SET"); return; @@ -351,6 +382,9 @@ void Worker::privateUpdate() { // update procedure, even if ISMAS-trigger is not correctly set ("WAIT") } + if (abortIfTimedOut()) + return; + emit this->disableExit(); QDir customerRepository(m_customerRepository); @@ -434,6 +468,9 @@ void Worker::privateUpdate() { CONSOLE() << UPDATE_STEP::CHECK_REPOSITORY_SUCCESS; setProgress(_CHECKOUT_REPOSITORY_SUCCESS); + if (abortIfTimedOut()) + return; + if (m_clone == false) { if (m_ismasTriggerActive == false) { return; @@ -462,6 +499,9 @@ void Worker::privateUpdate() { // CHECK-OUT BRANCH // //////////////////////////////////////////////////////////////////////////// + if (abortIfTimedOut()) + return; + if ((continueUpdate = customerEnvironment()) == false) { // even if something goes wrong creating the environment, try to execute // opkg_commands @@ -484,6 +524,8 @@ void Worker::privateUpdate() { // COMPUTE CHANGED FILES OF CUSTOMER REPOSITORY // //////////////////////////////////////////////////////////////////////////// + if (abortIfTimedOut()) + return; if ((continueUpdate = filesToUpdate()) == false) { // even if something goes wrong in filesToUpdate, try to execute // opkg_commands @@ -505,6 +547,9 @@ void Worker::privateUpdate() { // (R)SYNC THE REPOSITORY WITH THE LOCAL FILEYSTEM // //////////////////////////////////////////////////////////////////////////// + if (abortIfTimedOut()) + return; + if ((continueUpdate = syncCustomerRepositoryAndFS()) == false) { // even if something goes wrong with rsync, try to execute // opkg_commands @@ -526,6 +571,9 @@ void Worker::privateUpdate() { // EXECUTE OPKG COMMANDS // //////////////////////////////////////////////////////////////////////////// + if (abortIfTimedOut()) + return; + if ((continueUpdate = execOpkgCommands()) == false) { return; } @@ -533,6 +581,9 @@ void Worker::privateUpdate() { GUI(lst) << (CONSOLE(lst) << UPDATE_STEP::EXEC_OPKG_COMMANDS); setProgress(_EXEC_OPKG_COMMAND_SUCCESS); + if (abortIfTimedOut()) + return; + //////////////////////////////////////////////////////////////////////////// // @@ -1295,18 +1346,28 @@ qint64 Worker::getFileSize(QString const &fileName) const { return fInfo.exists() ? fInfo.size() : -1; } +static constexpr int OPKG_START_TIMEOUT_MS = 30 * 1000; +static constexpr int OPKG_FINISH_TIMEOUT_MS = 30 * 60 * 1000; + bool Worker::executeOpkgCommand(QString opkgCommand) { - Command c(opkgCommand); + Command c(opkgCommand, + OPKG_START_TIMEOUT_MS, + OPKG_FINISH_TIMEOUT_MS); + if (c.execute(m_workingDirectory)) { - QString const r = c.getCommandResult(); - Utils::printInfoMsg(QString("EXECUTE OPKG COMMAND %1 OK: %2") - .arg(opkgCommand) - .arg(c.getCommandResult())); + Utils::printInfoMsg( + QString("EXECUTE OPKG COMMAND %1 OK: %2") + .arg(opkgCommand) + .arg(c.getCommandResult())); + return true; - } else { - Utils::printCriticalErrorMsg(QString("EXECUTE OPKG COMMAND %1 FAILED") - .arg(opkgCommand)); } + + Utils::printCriticalErrorMsg( + QString("EXECUTE OPKG COMMAND %1 FAILED/TIMED OUT: %2") + .arg(opkgCommand) + .arg(c.getCommandResult())); + return false; } diff --git a/UpdatePTUDevCtrl/worker.h b/UpdatePTUDevCtrl/worker.h index 8c54d69..96bb512 100644 --- a/UpdatePTUDevCtrl/worker.h +++ b/UpdatePTUDevCtrl/worker.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -19,6 +20,7 @@ #include "ismas/ismas_client.h" #include "utils.h" + #ifdef PTU5 #define SERIAL_PORT "ttymxc2" #else @@ -141,6 +143,14 @@ class hwinf; class Worker : public QThread{ Q_OBJECT + private: + static constexpr qint64 UPDATE_MAX_RUNTIME_MS = + 60LL * 60LL * 1000LL; // 60 minutes + + QElapsedTimer m_updateRuntime; + +public: + bool updateTimedOut() const; int const m_customerNr; QString const m_customerNrStr; int const m_machineNr;