#include "git_client.h" #include "update.h" #include "worker.h" #include "utils.h" #include "process/command.h" #include #include #include #include GitClient::GitClient(QString const &customerRepositoryPath, QString const &customerNrStr, QString const &customerRepository, QString const &workingDirectory, QString const &branchName, QObject *parent) : QObject(parent) , m_worker(qobject_cast(parent)) , m_repositoryPath(customerRepositoryPath) , m_customerNr(customerNrStr) , m_workingDirectory(workingDirectory) , m_branchName(branchName) , m_customerRepository(customerRepository) { if (!m_worker) { qCritical() << "ERROR CASTING PARENT TO WORKER FAILED"; } } bool GitClient::gitCloneCustomerRepository() { /* Blobless clone ============== When using the --filter=blob:none option, the initial git clone will download all reachable commits and trees, and only download the blobs for commits when you do a git checkout. This includes the first checkout inside the git clone operation. The important thing to notice is that we have a copy of every blob at HEAD but the blobs in the history are not present. If your repository has a deep history full of large blobs, then this option can significantly reduce your git clone times. The commit and tree data is still present, so any subsequent git checkout only needs to download the missing blobs. The Git client knows how to batch these requests to ask the server only for the missing blobs. Further, when running git fetch in a blobless clone, the server only sends the new commits and trees. The new blobs are downloaded only after a git checkout. Note that git pull runs git fetch and then git merge, so it will download the necessary blobs during the git merge command. When using a blobless clone, you will trigger a blob download whenever you need the contents of a file, but you will not need one if you only need the OID (object-id) of a file. This means that git log can detect which commits changed a given path without needing to download extra data. This means that blobless clones can perform commands like git merge-base, git log, or even git log -- with the same performance as a full clone. Commands like git diff or git blame require the contents of the paths to compute diffs, so these will trigger blob downloads the first time they are run. However, the good news is that after that you will have those blobs in your repository and do not need to download them a second time. Most developers only need to run git blame on a small number of files, so this tradeoff of a slightly slower git blame command is worth the faster clone and fetch times. Note: git v2.18 does not support treeless clones: --filter=tree:0. */ // Note: for some reason it is necessary to pass "--progress ---v", // otherwise QProcess returns an error of 128 = 0x80 for the command. QString gitCommand("git clone --progress -vvv --filter=blob:none "); gitCommand += m_repositoryPath; Command c(gitCommand); qInfo() << "IN CURRENT WD" << m_workingDirectory << "CLONE VIA COMMAND" << gitCommand; if (c.execute(m_workingDirectory)) { // execute the command in wd QString const result = c.getCommandResult(); if (!result.isEmpty()) { // Cloning into 'customer_281'...\n int customer = -1; int cloning = result.indexOf("Cloning", 0, Qt::CaseInsensitive); if (cloning != -1) { customer = result.indexOf("customer_", cloning, Qt::CaseInsensitive); if (customer != -1) { QString customerNr = result.mid(customer); static constexpr char const ch = '\''; int i = customerNr.indexOf(QChar(ch)); if (i != -1) { if ((customerNr = customerNr.mid(0, i)) == m_customerNr) { qInfo() << "CLONING" << m_repositoryPath << "OK"; return true; } Utils::printCriticalErrorMsg( QString("ERROR CLONE RESULT HAS WRONG CUSTOMER-NR. (%1 != %2) CLONE_RESULT=%3") .arg(customerNr) .arg(m_customerNr) .arg(result)); return false; } } } Utils::printCriticalErrorMsg( QString("ERROR CLONE RESULT HAS WRONG FORMAT. CLONING=%1 CUSTOMER=%2 CLONE_RESULT=%3") .arg(cloning) .arg(customer) .arg(result)); return false; } Utils::printCriticalErrorMsg("ERROR CLONE RESULT IS EMPTY"); } return false; } bool GitClient::copyGitConfigFromMaster() { // only allowed when called in // master branch (???) if (QDir(m_customerRepository).exists()) { QString const cp = QString("cp .gitconfig .git/config"); Command c("bash"); if (c.execute(m_customerRepository, QStringList() << "-c" << cp)) { qInfo() << "cp .gitconfig .git/config OK"; return true; } qCritical() << "ERROR cp .gitconfig .git/config"; } return false; } QStringList GitClient::gitBranchNames() { // git config --global pager.branch false QStringList bNames; if (QDir(m_customerRepository).exists()) { QString gitCommand("git branch -a"); Command c(gitCommand); if (c.execute(m_customerRepository)) { QString const result = c.getCommandResult(); return result.split('\n'); } } return bNames; } bool GitClient::gitCheckoutBranch() { // TODO: nachsehen, ob der Branch ueberhaupt existiert if (QDir(m_customerRepository).exists()) { int zoneNr = Utils::read1stLineOfFile("/mnt/system_data/zone_nr"); m_branchName = (zoneNr != 0) ? QString("zg1/zone%1").arg(zoneNr) : "master"; QString gitCommand("git checkout "); gitCommand += m_branchName; Command c(gitCommand); return c.execute(m_customerRepository); // execute command in customerRepo } Utils::printCriticalErrorMsg(QString("ERROR ") + m_customerRepository + " DOES NOT EXIST"); return false; } bool GitClient::gitCloneAndCheckoutBranch() { qInfo() << "CLONE" << m_repositoryPath << "AND CHECKOUT" << m_branchName; if (gitCloneCustomerRepository()) { //if (copyGitConfigFromMaster()) { if (gitCheckoutBranch()) { return true; } else { // TODO } //} } Utils::printCriticalErrorMsg(QString("CLONE ") + m_repositoryPath + " AND CHECKOUT FAILED"); return false; } QStringList GitClient::gitShowReason(QString branchName) { QStringList lst; if (QDir(m_customerRepository).exists()) { // %h: commit (short form) // %s: commit message // %cI: commit date, strict ISO 8601 format // Note: branch with branchName has to exist: format zg1/zone1 Command c(QString("git show origin/%1 -s --format=\"c=%h m=%s d=%cI\"").arg(branchName)); if (c.execute(m_customerRepository)) { QString const s = c.getCommandResult().trimmed(); int const c = s.indexOf("c="); int const m = s.indexOf("m="); int const d = s.indexOf("d="); QString msg{""}; // IsmasClient::getReasonForLastSendVersion(); QString commit{""}, date{""}; if (c != -1) { int start = c + 2; if (m >= start) { int length = m - start; commit = s.mid(start, length).trimmed(); start = m + 2; if (d >= start) { length = d - start; // msg = " ("; msg = s.mid(start, length).trimmed(); // msg += ")"; start = d + 2; date = s.mid(start); } } if (!commit.isEmpty() && !msg.isEmpty() && !date.isEmpty()) { lst << commit << msg << date; } } } } else { qCritical() << "CUSTOMER_REPOSITORY" << m_customerRepository << "DOES NOT EXIST"; } return lst; } /* Zu beachten: wird eine datei neu hinzugefuegt (git add/commit) dann aber gleich wieder geloscht, so wird sie im diff nicht angezeigt. */ std::optional GitClient::gitDiff(QString const &commits) { if (QDir(m_customerRepository).exists()) { // 409f198..6c22726 QString gitCommand("git diff --compact-summary "); gitCommand += commits; Command c(gitCommand); if (c.execute(m_customerRepository)) { // execute command in local customerRepo QString s = c.getCommandResult().trimmed(); Utils::printInfoMsg("GIT DIFF RESULT " + s); QStringList lines = Update::split(s, '\n'); QStringList fileNames; // each line has the format "etc/psa_config/DC2C_print01.json | 1 + // or the format "etc/psa_config/DC2C_print01.json (new) | 1 + // the filenames are relativ to the repository for (int i = 0; i < lines.size(); ++i) { QString const &line = lines.at(i); int newIndex = line.indexOf("(new"); // for new files int goneIndex = line.indexOf("(gone"); // for removed files int modeIndex = line.indexOf("(mode"); int pipeIndex = line.indexOf('|'); if (newIndex != -1) { QString file = line.left(newIndex).trimmed(); qInfo() << "FILE (NEW)" << file; fileNames << file; } else if (modeIndex != -1) { QString const file = line.left(modeIndex).trimmed(); qInfo() << "FILE (MODE)" << file; fileNames << file; } else if (goneIndex != -1) { QString const file = line.left(goneIndex).trimmed(); qCritical() << "FILE (GONE)" << file; } else if (pipeIndex != -1) { QString const file = line.left(pipeIndex).trimmed(); qInfo() << "FILE (PIPE)" << file; fileNames << file; } } if (!fileNames.isEmpty()) { return fileNames; } } } return std::nullopt; } bool GitClient::gitFsck() { bool r = false; if (QDir(m_customerRepository).exists()) { Command c("git fsck"); if ((r = c.execute(m_customerRepository)) == false) { QString const &s = c.getCommandResult().trimmed(); Utils::printCriticalErrorMsg(QString("GIT FSCK FAILED: %1").arg(s)); } } return r; } bool GitClient::branchExistsRemotely() { bool remoteBranchExists = false; if (QDir(m_customerRepository).exists()) { qInfo() << "BRANCH NAME" << m_branchName; QString const cmd = QString("git ls-remote --exit-code --heads origin %1").arg(m_branchName); Command c(cmd); if (c.execute(m_customerRepository)) { // expected result: c16c833c8778c1b3691a74afee5a469177e4e69b refs/heads/zg1/zone1000 QString const s = c.getCommandResult().trimmed(); if (!s.isEmpty()) { // the result is only one line if ((remoteBranchExists = s.contains(m_branchName)) == true) { qCritical() << "(" << __func__ << ":" << __LINE__ << ") branch" << m_branchName << "EXISTS REMOTELY. (" << s << ")"; } } else { Utils::printCriticalErrorMsg(QString("EMPTY RESULT FOR CMD %1").arg(cmd)); } } else { Utils::printCriticalErrorMsg(QString("FAILED TO EXEC '%1'").arg(cmd)); } } return remoteBranchExists; } bool GitClient::branchExistsLocally() { Command c("git branch -l"); if (c.execute(m_customerRepository)) { QString const s = c.getCommandResult().trimmed(); if (!s.isEmpty()) { QStringList lines = Update::split(s, '\n'); if (!lines.empty()) { for (int i=0; i < lines.size(); ++i) { QString line = lines.at(i); // expected: * [new branch] zg1/zone12 -> origin/zg1/zone12" if (line.contains(m_branchName)) { if (m_worker) { QStringList lst(QString("BRANCH-NAME %1 CONTAINED IN RESULT %2").arg(m_branchName).arg(s)); m_worker->CONSOLE(lst) << Worker::UPDATE_STEP::PULL_NEW_BRANCH; } return true; } } if (m_worker) { QStringList lst(QString("BRANCH-NAME %1 NOT CONTAINED IN RESULT %2").arg(m_branchName).arg(s)); m_worker->CONSOLE(lst) << Worker::UPDATE_STEP::PULL_NEW_BRANCH_FAILURE; } } else { if (m_worker) { QStringList lst(QString("'git branch -l' RETURNED NO LINES")); m_worker->CONSOLE(lst) << Worker::UPDATE_STEP::PULL_NEW_BRANCH_FAILURE; } } } else { if (m_worker) { QStringList lst(QString("'git branch -l' RETURNED EMPTY RESULT")); m_worker->CONSOLE(lst) << Worker::UPDATE_STEP::PULL_NEW_BRANCH_FAILURE; } } } else { if (m_worker) { QStringList lst(QString("FAILED TO EXEC 'git branch -l'")); m_worker->CONSOLE(lst) << Worker::UPDATE_STEP::PULL_NEW_BRANCH_FAILURE; } } return false; } bool GitClient::gitPullNewBranches() { if (QDir(m_customerRepository).exists()) { Command c("git pull"); if (c.execute(m_customerRepository)) { QString const s = c.getCommandResult().trimmed(); // expected: Already up-to-date. if (!s.isEmpty()) { QStringList lst; QString msg(QString("GIT-PULL-NEW-BRANCH. RESULT=%1").arg(s)); if (m_worker) { m_worker->CONSOLE(lst) << Worker::UPDATE_STEP::PULL_NEW_BRANCH; } return true; } } } return false; } /* Hat sich nichts geaendert, so werden auch keine Commits <>..<> angezeigt */ std::optional GitClient::gitPull() { if (QDir(m_customerRepository).exists()) { qInfo() << "BRANCH NAME" << m_branchName; #if 0 IMPORTANT COMMENT: If remote host keys are changed, then export GIT_SSH_COMMAND="ssh -i /opt/app/tools/atbupdate/.keys/id_ed25519_ptuConfig" git pull leads to the following warning/error message: @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! Someone could be eavesdropping on you right now (man-in-the-middle attack)! It is also possible that a host key has just been changed. The fingerprint for the ECDSA key sent by the remote host is SHA256:vOD5jF2hglGktqLhK9ABxfEjwEgIK68/v9erdT05NDQ. Please contact your system administrator. Add correct host key in /home/root/.ssh/known_hosts to get rid of this message. Offending ECDSA key in /home/root/.ssh/known_hosts:1 Password authentication is disabled to avoid man-in-the-middle attacks. Keyboard-interactive authentication is disabled to avoid man-in-the-middle attacks. Agent forwarding is disabled to avoid man-in-the-middle attacks. X11 forwarding is disabled to avoid man-in-the-middle attacks. Already up to date. This first part is from ssh itself. Only the last line is the git message. Here an output of running ATBUpdateTool with a corrupted known-hosts-file: ... Oct 23 14:18:18 ATB_PTU5 ATBUpdateTool[2696]: ( branchExistsRemotely : 310 ) branch "zg1/zone1" EXISTS REMOTELY. ( "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\r\n@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @\r\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\r\nIT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY!\r\nSomeone could be eavesdropping on you right now (man-in-the-middle attack)!\r\nIt is also possible that a host key has just been changed.\r\nThe fingerprint for the ECDSA key sent by the remote host is\nSHA256:vOD5jF2hglGktqLhK9ABxfEjwEgIK68/v9erdT05NDQ.\r\nPlease contact your system administrator.\r\nAdd correct host key in /home/root/.ssh/known_hosts to get rid of this message.\r\nOffending ECDSA key in /home/root/.ssh/known_hosts:1\r\nPassword authentication is disabled to avoid man-in-the-middle attacks.\r\nKeyboard-interactive authentication is disabled to avoid man-in-the-middle attacks.\r\nAgent forwarding is disabled to avoid man-in-the-middle attacks.\r\nX11 forwarding is disabled to avoid man-in-the-middle attacks.\r\nbd4e8da4780b1a7d6be3d3ce8419f43ccf7e706f\trefs/heads/zg1/zone1" ) Oct 23 14:18:18 ATB_PTU5 ATBUpdateTool[2696]: EXECUTED "git branch -l" "(runtime 16ms)" with code 0 IN "/opt/app/tools/atbupdate/customer_336" Oct 23 14:18:18 ATB_PTU5 ATBUpdateTool[2696]: "UPDATE_STEP::PULL_NEW_BRANCH" Oct 23 14:18:18 ATB_PTU5 ATBUpdateTool[2696]: "BRANCH-NAME zg1/zone1 CONTAINED IN RESULT master\n* zg1/zone1" ..... The download continues. #endif Command c("git pull"); if (c.execute(m_customerRepository)) { QString const s = c.getCommandResult().trimmed(); if (!s.isEmpty()) { QStringList lines = Update::split(s, '\n'); worker()->CONSOLE(lines) << Worker::UPDATE_STEP::UPDATE_REPOSITORY; if (!lines.empty()) { static const QRegularExpression alreadyUpToDate("^\\s*Already\\s+up\\s+to\\s+date.*$"); if (std::none_of(lines.cbegin(), lines.cend(), [](QString const &s) { return s.contains(alreadyUpToDate); })) { int zoneNr = Utils::read1stLineOfFile("/mnt/system_data/zone_nr"); m_branchName = (zoneNr != 0) ? QString("zg1/zone%1").arg(zoneNr) : "master"; // lines can look like this: // From https://git.mimbach49.de/GerhardHoffmann/customer_281 // 41ec581..5d25ac3 master -> origin/master // ff10f57..43530a1 zg1/zone1 -> origin/zg1/zone1 // 6ed893f..5d9882c zg1/zone2 -> origin/zg1/zone2 // 4384d17..77045d8 zg1/zone3 -> origin/zg1/zone3 // 89d2812..36a0d74 zg1/zone5 -> origin/zg1/zone5 // // More exactly: // remote: Counting objects: 382, done. // remote: Compressing objects: 100% (203/203), done. // remote: Total 278 (delta 177), reused 103 (delta 59) // Receiving objects: 100% (278/278), 4.89 MiB | 539 KiB/s, done. // Resolving deltas: 100% (177/177), completed with 40 local objects. // From ssh://longair@pacific.mpi-cbg.de/srv/git/fiji // 3036acc..9eb5e40 debian-release-20081030 -> origin/debian-release-20081030 // * [new branch] debian-release-20081112 -> origin/debian-release-20081112 // * [new branch] debian-release-20081112.1 -> origin/debian-release-20081112.1 // 3d619e7..6260626 master -> origin/master // // The most important bits here are the lines like these: // // 3036acc..9eb5e40 debian-release-20081030 -> origin/debian-release-20081030 // * [new branch] debian-release-20081112 -> origin/debian-release-20081112 // // The first line of these two shows that your remote-tracking branch // origin/debian-release-20081030 has been advanced from the commit 3036acc to 9eb5e40. // The bit before the arrow is the name of the branch in the remote repository. // The second line similarly show that since we last did this, a new remote tracking // branch has been created. (git fetch may also fetch new tags if they have appeared // in the remote repository.) bool found = false; for (int i=0; i < lines.size(); ++i) { if (lines.at(i).contains(m_branchName)) { found = true; // 409f198..6c22726 zg1/zone1 -> origin/zg1/zone1 static QRegularExpression re("(^\\s*)([0-9A-Fa-f]+..[0-9A-Fa-f]+)(.*$)"); QRegularExpressionMatch match = re.match(lines.at(i)); if (match.hasMatch()) { if (re.captureCount() == 3) { // start with full match (0), then the other 3 matches QString const matchCaptured = match.captured(2); worker()->CONSOLE(QStringList(matchCaptured)) << Worker::UPDATE_STEP::UPDATE_REPOSITORY; return matchCaptured; } else { QStringList lst(QString("(wrong capture count (%1)").arg(re.captureCount())); worker()->GUI(lst) << (worker()->CONSOLE(lst) << Worker::UPDATE_STEP::UPDATE_REPOSITORY_FAILURE); } } else { QStringList lst("no regex-match for commits"); worker()->GUI(lst) << (worker()->CONSOLE(lst) << Worker::UPDATE_STEP::UPDATE_REPOSITORY_FAILURE); } } } if (!found) { QStringList lst(QString("unknown branch name ") + m_branchName); worker()->GUI(lst) << (worker()->CONSOLE(lst) << Worker::UPDATE_STEP::UPDATE_REPOSITORY_FAILURE); } } else { return "Already up to date"; } } else { QStringList lst(QString("WRONG FORMAT FOR RESULT OF 'GIT PULL' ") + s); worker()->GUI(lst) << (worker()->CONSOLE(lst) << Worker::UPDATE_STEP::UPDATE_REPOSITORY_FAILURE); } } else { QStringList lst("EMPTY RESULT FOR 'GIT PULL'"); worker()->GUI(lst) << (worker()->CONSOLE(lst) << Worker::UPDATE_STEP::UPDATE_REPOSITORY_FAILURE); } } } else { QStringList lst(QString("REPOSITORY ") + m_customerRepository + " DOES NOT EXIST"); worker()->GUI(lst) << (worker()->CONSOLE(lst) << Worker::UPDATE_STEP::UPDATE_REPOSITORY_FAILURE); } return std::nullopt; } std::optional GitClient::gitMerge() { Command c("git merge"); if (c.execute(m_workingDirectory)) { QString s = c.getCommandResult(); QStringList lst = Update::split(s, '\n'); return lst; } return std::nullopt; } QString GitClient::gitLastCommit(QString fileName) { if (QDir(m_customerRepository).exists()) { QString const filePath = QDir::cleanPath(m_customerRepository + QDir::separator() + fileName); if (QFile(filePath).exists()) { QString const gitCommand = QString("git log %1 | head -n 1").arg(fileName); Command c("bash"); if (c.execute(m_customerRepository, QStringList() << "-c" << gitCommand)) { QString const r = c.getCommandResult(); int const idx = r.indexOf("commit "); if (idx != -1) { return r.mid(idx + 8).trimmed(); } } } } return ""; } // fileName has to an absolute path QString GitClient::gitBlob(QString fileName) { QFileInfo fi(fileName); if (fi.exists()) { QString const gitCommand = QString("git hash-object %1").arg(fileName); Command c(gitCommand); if (c.execute("/tmp")) { return c.getCommandResult().trimmed(); } } return "N/A"; } QString GitClient::gitCommitForBlob(QString blob) { if (QDir(m_customerRepository).exists()) { QString const gitCommand = QString("git whatchanged --all --find-object=%1 | head -n 1").arg(blob); Command c(gitCommand); if (c.execute(m_customerRepository)) { return c.getCommandResult(); } } return ""; } bool GitClient::gitIsFileTracked(QString fName) { if (QDir(m_customerRepository).exists()) { QString const gitCommand = QString("git ls-files --error-unmatch %1").arg(fName); Command c(gitCommand); return c.execute(m_customerRepository); } return false; } //get_commit_for_blob () { // # search for the blob in all commits for the file(name) $1 // echo $(git log --all --pretty=format:%H -- $2 | // xargs -I{} bash -c "git ls-tree {} -- $2 | // grep -q $1 && echo -n {} && head -n 1") //}