feat(vfs/folders): enable on demand fetch of folder content

.github/workflows/linux-clang-compile-tests.yml

@@ -8,7 +8,7 @@ jobs:
   build:
     name: Linux Clang compilation and tests
     runs-on: ubuntu-latest
-    container: ghcr.io/nextcloud/continuous-integration-client-qt6:client-6.8.1-2
+    container: ghcr.io/nextcloud/continuous-integration-client-qt6:client-trixie-6.8.2-1
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
@@ -17,7 +17,7 @@ jobs:
         run: |
           mkdir build
           cd build
-          cmake .. -G Ninja -DCMAKE_PREFIX_PATH=/opt/qt -DCMAKE_C_COMPILER=clang-14 -DCMAKE_CXX_COMPILER=clang++-14 -DCMAKE_BUILD_TYPE=Debug -DQT_MAJOR_VERSION=6 -DQUICK_COMPILER=ON -DBUILD_UPDATER=ON -DBUILD_TESTING=1 -DCMAKE_CXX_FLAGS=-Werror -DOPENSSL_ROOT_DIR=/usr/local/lib64
+          cmake .. -G Ninja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Debug -DQT_MAJOR_VERSION=6 -DQUICK_COMPILER=ON -DBUILD_UPDATER=ON -DBUILD_TESTING=1 -DCMAKE_CXX_FLAGS=-Werror
           ninja
       - name: Run tests
         run: |

.github/workflows/linux-gcc-compile-tests.yml

@@ -8,7 +8,7 @@ jobs:
   build:
     name: Linux GCC compilation and tests
     runs-on: ubuntu-latest
-    container: ghcr.io/nextcloud/continuous-integration-client-qt6:client-6.8.1-2
+    container: ghcr.io/nextcloud/continuous-integration-client-qt6:client-trixie-6.8.2-1
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
@@ -17,7 +17,7 @@ jobs:
         run: |
           mkdir build
           cd build
-          cmake .. -G Ninja -DCMAKE_PREFIX_PATH=/opt/qt -DCMAKE_C_COMPILER=gcc-11 -DCMAKE_CXX_COMPILER=g++-11 -DCMAKE_BUILD_TYPE=Debug -DQT_MAJOR_VERSION=6 -DQUICK_COMPILER=ON -DBUILD_UPDATER=ON -DBUILD_TESTING=1 -DCMAKE_CXX_FLAGS=-Werror -DOPENSSL_ROOT_DIR=/usr/local/lib64
+          cmake .. -G Ninja -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ -DCMAKE_BUILD_TYPE=Debug -DQT_MAJOR_VERSION=6 -DQUICK_COMPILER=ON -DBUILD_UPDATER=ON -DBUILD_TESTING=1 -DCMAKE_CXX_FLAGS=-Werror
           ninja
       - name: Run tests
         run: |

src/common/common.cmake

@@ -19,6 +19,9 @@ set(common_SOURCES
     ${CMAKE_CURRENT_LIST_DIR}/pinstate.cpp
     ${CMAKE_CURRENT_LIST_DIR}/plugin.cpp
     ${CMAKE_CURRENT_LIST_DIR}/syncfilestatus.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/syncitemenums.cpp
+    ${CMAKE_CURRENT_LIST_DIR}/remoteinfo.h
+    ${CMAKE_CURRENT_LIST_DIR}/folderquota.h
 )
 
 if(WIN32)

src/common/folderquota.h

@@ -0,0 +1,36 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2014 ownCloud GmbH
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#ifndef FOLDERQUOTA_H
+#define FOLDERQUOTA_H
+
+#include <cstdint>
+
+namespace OCC {
+
+/**
+ * Represent the quota for each folder retrieved from the server
+ * bytesUsed: space used in bytes
+ * bytesAvailale: free space available in bytes or
+ *                -1: Uncomputed free space - new folder (externally created) not yet scanned by the server
+ *                -2: Unknown free space
+ *                -3: Unlimited free space.
+ */
+struct FolderQuota
+{
+    int64_t bytesUsed = -1;
+    int64_t bytesAvailable = -1;
+    enum ServerEntry {
+        Invalid = 0,
+        Valid
+    };
+    static constexpr char availableBytesC[] = "quota-available-bytes";
+    static constexpr char usedBytesC[] = "quota-used-bytes";
+};
+
+}
+
+#endif // FOLDERQUOTA_H

src/common/remoteinfo.h

@@ -0,0 +1,62 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2014 ownCloud GmbH
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#ifndef REMOTEINFO_H
+#define REMOTEINFO_H
+
+#include "folderquota.h"
+#include "common/remotepermissions.h"
+#include "common/syncitemenums.h"
+
+#include <QString>
+#include <QByteArray>
+
+namespace OCC {
+
+/**
+ * Represent all the meta-data about a file in the server
+ */
+struct RemoteInfo
+{
+    /** FileName of the entry (this does not contains any directory or path, just the plain name */
+    QString name;
+    QByteArray etag;
+    QByteArray fileId;
+    QByteArray checksumHeader;
+    OCC::RemotePermissions remotePerm;
+    time_t modtime = 0;
+    int64_t size = 0;
+    int64_t sizeOfFolder = 0;
+    bool isDirectory = false;
+    bool _isE2eEncrypted = false;
+    bool isFileDropDetected = false;
+    QString e2eMangledName;
+    bool sharedByMe = false;
+
+    [[nodiscard]] bool isValid() const { return !name.isNull(); }
+    [[nodiscard]] bool isE2eEncrypted() const { return _isE2eEncrypted; }
+
+    QString directDownloadUrl;
+    QString directDownloadCookies;
+
+    SyncFileItemEnums::LockStatus locked = SyncFileItemEnums::LockStatus::UnlockedItem;
+    QString lockOwnerDisplayName;
+    QString lockOwnerId;
+    SyncFileItemEnums::LockOwnerType lockOwnerType = SyncFileItemEnums::LockOwnerType::UserLock;
+    QString lockEditorApp;
+    qint64 lockTime = 0;
+    qint64 lockTimeout = 0;
+    QString lockToken;
+
+    bool isLivePhoto = false;
+    QString livePhotoFile;
+
+    FolderQuota folderQuota;
+};
+
+}
+
+#endif // REMOTEINFO_H

src/common/syncitemenums.cpp

@@ -0,0 +1,7 @@
+/*
+ * SPDX-FileCopyrightText: 2020 ownCloud GmbH
+ * SPDX-License-Identifier: LGPL-2.1-or-later
+ */
+
+#include "syncitemenums.h"
+#include "moc_syncitemenums.cpp"

src/common/syncitemenums.h

@@ -0,0 +1,39 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-FileCopyrightText: 2014 ownCloud GmbH
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#ifndef SYNCITEMENUMS_H
+#define SYNCITEMENUMS_H
+
+#include "ocsynclib.h"
+
+#include <QObject>
+
+namespace OCC {
+
+namespace SyncFileItemEnums {
+
+OCSYNC_EXPORT Q_NAMESPACE
+
+enum class LockStatus {
+    UnlockedItem = 0,
+    LockedItem = 1,
+};
+
+Q_ENUM_NS(LockStatus)
+
+enum class LockOwnerType : int{
+    UserLock = 0,
+    AppLock = 1,
+    TokenLock = 2,
+};
+
+Q_ENUM_NS(LockOwnerType)
+
+}
+
+}
+
+#endif // SYNCITEMENUMS_H

src/common/syncjournaldb.cpp

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: LGPL-2.1-or-later
  */
 
 #include <QCryptographicHash>
 #include <QFile>
 #include <QLoggingCategory>
 #include <QStringList>
@@ -1015,6 +1015,11 @@
     SyncJournalFileRecord record = _record;
     QMutexLocker locker(&_mutex);
 
+    Q_ASSERT(record._modtime > 0);
+    if (record._modtime <= 0) {
+        qCCritical(lcDb) << "invalid modification time";
+    }
+
     if (!_etagStorageFilter.isEmpty()) {
         // If we are a directory that should not be read from db next time, don't write the etag
         QByteArray prefix = record._path + "/";
@@ -1044,6 +1049,8 @@
                  << "livePhotoFile" << record._livePhotoFile
                  << "folderQuota - bytesUsed:" << record._folderQuota.bytesUsed << "bytesAvailable:" << record._folderQuota.bytesAvailable;
 
+    Q_ASSERT(!record.path().isEmpty());
+
     const qint64 phash = getPHash(record._path);
     if (!checkConnect()) {
         qCWarning(lcDb) << "Failed to connect database.";
@@ -2377,6 +2384,42 @@
     commitInternal(QStringLiteral("setSelectiveSyncList"));
 }
 
+QStringList SyncJournalDb::addSelectiveSyncLists(SelectiveSyncListType type, const QString &path)
+{
+    bool ok = false;
+
+    const auto pathWithTrailingSlash = Utility::trailingSlashPath(path);
+
+    const auto blackListList = getSelectiveSyncList(type, &ok);
+    auto blackListSet = QSet<QString>{blackListList.begin(), blackListList.end()};
+    blackListSet.insert(pathWithTrailingSlash);
+    auto blackList = blackListSet.values();
+    blackList.sort();
+    setSelectiveSyncList(type, blackList);
+
+    qCInfo(lcSql()) << "add" << path << "into" << type << blackList;
+
+    return blackList;
+}
+
+QStringList SyncJournalDb::removeSelectiveSyncLists(SelectiveSyncListType type, const QString &path)
+{
+    bool ok = false;
+
+    const auto pathWithTrailingSlash = Utility::trailingSlashPath(path);
+
+    const auto blackListList = getSelectiveSyncList(type, &ok);
+    auto blackListSet = QSet<QString>{blackListList.begin(), blackListList.end()};
+    blackListSet.remove(pathWithTrailingSlash);
+    auto blackList = blackListSet.values();
+    blackList.sort();
+    setSelectiveSyncList(type, blackList);
+
+    qCInfo(lcSql()) << "remove" << path << "into" << type << blackList;
+
+    return blackList;
+}
+
 void SyncJournalDb::avoidRenamesOnNextSync(const QByteArray &path)
 {
     QMutexLocker locker(&_mutex);

src/common/syncjournaldb.h

@@ -7,7 +7,7 @@
 #ifndef SYNCJOURNALDB_H
 #define SYNCJOURNALDB_H
 
 #include <QObject>
 #include <QDateTime>
 #include <QHash>
 #include <QMutex>
@@ -173,6 +173,10 @@
     /* Write the selective sync list (remove all other entries of that list */
     void setSelectiveSyncList(SelectiveSyncListType type, const QStringList &list);
 
+    QStringList addSelectiveSyncLists(SelectiveSyncListType type, const QString &path);
+
+    QStringList removeSelectiveSyncLists(SelectiveSyncListType type, const QString &path);
+
     /**
      * Make sure that on the next sync fileName and its parents are discovered from the server.
      *

src/common/syncjournalfilerecord.h

@@ -1,3 +1,3 @@
 /*
  * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors
  * SPDX-FileCopyrightText: 2014 ownCloud GmbH
@@ -7,7 +7,7 @@
 #ifndef SYNCJOURNALFILERECORD_H
 #define SYNCJOURNALFILERECORD_H
 
 #include <QString>
 #include <QDateTime>
 
 #include "csync.h"
@@ -53,7 +53,7 @@
     [[nodiscard]] QByteArray numericFileId() const;
     [[nodiscard]] QDateTime modDateTime() const { return Utility::qDateTimeFromTime_t(_modtime); }
 
-    [[nodiscard]] bool isDirectory() const { return _type == ItemTypeDirectory; }
+    [[nodiscard]] bool isDirectory() const { return _type == ItemTypeVirtualDirectory || _type == ItemTypeDirectory; }
     [[nodiscard]] bool isFile() const { return _type == ItemTypeFile || _type == ItemTypeVirtualFileDehydration; }
     [[nodiscard]] bool isVirtualFile() const { return _type == ItemTypeVirtualFile || _type == ItemTypeVirtualFileDownload; }
     [[nodiscard]] QString path() const { return QString::fromUtf8(_path); }

src/common/vfs.h

@@ -1,3 +1,3 @@
 /*
  * SPDX-FileCopyrightText: 2020 Nextcloud GmbH and Nextcloud contributors
  * SPDX-FileCopyrightText: 2018 ownCloud GmbH
@@ -5,14 +5,18 @@
  */
 #pragma once
 
 #include "ocsynclib.h"
 #include "result.h"
 #include "syncfilestatus.h"
 #include "pinstate.h"
 
+#include "common/remoteinfo.h"
+
 #include <QObject>
 #include <QScopedPointer>
 #include <QSharedPointer>
+#include <QString>
+#include <QMap>
 
 #include <deque>
 #include <memory>
@@ -28,6 +32,13 @@
 class SyncFileItem;
 using SyncFileItemPtr = QSharedPointer<SyncFileItem>;
 
+struct OCSYNC_EXPORT PlaceholderCreateInfo {
+    QString name;
+    std::wstring stdWStringName;
+    QString fullPath;
+    RemoteInfo parsedProperties;
+};
+
 /** Collection of parameters for initializing a Vfs instance. */
 struct OCSYNC_EXPORT VfsSetupParams
 {
@@ -183,7 +194,7 @@
      */
     [[nodiscard]] virtual OCC::Result<OCC::Vfs::ConvertToPlaceholderResult, QString> updateMetadata(const SyncFileItem &syncItem, const QString &filePath, const QString &replacesFile) = 0;
 
-    [[nodiscard]] virtual Result<Vfs::ConvertToPlaceholderResult, QString> updatePlaceholderMarkInSync(const QString &filePath, const QByteArray &fileId) = 0;
+    [[nodiscard]] virtual Result<Vfs::ConvertToPlaceholderResult, QString> updatePlaceholderMarkInSync(const QString &filePath, const SyncFileItem &item) = 0;
 
     [[nodiscard]] virtual bool isPlaceHolderInSync(const QString &filePath) const = 0;
 
@@ -326,7 +337,7 @@
     [[nodiscard]] bool isHydrating() const override { return false; }
 
     OCC::Result<OCC::Vfs::ConvertToPlaceholderResult, QString> updateMetadata(const SyncFileItem &, const QString &, const QString &) override { return {OCC::Vfs::ConvertToPlaceholderResult::Ok}; }
-    Result<Vfs::ConvertToPlaceholderResult, QString> updatePlaceholderMarkInSync(const QString &filePath, const QByteArray &fileId) override {Q_UNUSED(filePath) Q_UNUSED(fileId) return {QString{}};}
+    Result<Vfs::ConvertToPlaceholderResult, QString> updatePlaceholderMarkInSync(const QString &filePath, const SyncFileItem &item) override {Q_UNUSED(filePath) Q_UNUSED(item) return {QString{}};}
     [[nodiscard]] bool isPlaceHolderInSync(const QString &filePath) const override { Q_UNUSED(filePath) return true; }
 
     Result<void, QString> createPlaceholder(const SyncFileItem &) override { return {}; }

src/csync/csync.h

@@ -20,7 +20,7 @@
 #ifndef _CSYNC_H
 #define _CSYNC_H
 
 #include <QByteArray>
 #include <QVariant>
 #include <QLoggingCategory>
 
@@ -186,6 +186,8 @@
      * file dehydration without changing the pin state.
      */
     ItemTypeVirtualFileDehydration = 6,
+
+    ItemTypeVirtualDirectory = 7,
 };
 Q_ENUM_NS(ItemType)
 }

src/gui/folder.cpp

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: GPL-2.0-or-later
  */
 
 #include "common/syncjournaldb.h"
 #include "config.h"
 
 #include "account.h"
@@ -696,7 +696,8 @@
         SyncJournalFileRecord rec;
         const auto isFileRecordValid = journalDb()->getFileRecord(fileRecordPath, &rec) && rec.isValid();
         if (isFileRecordValid) {
-            [[maybe_unused]] const auto result = _vfs->updatePlaceholderMarkInSync(path() + rec.path(), rec._fileId);
+            const auto itemPointer = SyncFileItem::fromSyncJournalFileRecord(rec);
+            [[maybe_unused]] const auto result = _vfs->updatePlaceholderMarkInSync(path() + rec.path(), *itemPointer);
         }
         const auto canUnlockFile = isFileRecordValid
             && rec._lockstate._locked
@@ -740,7 +741,8 @@
         const auto fileRecordPath = fileFromLocalPath(file);
         SyncJournalFileRecord rec;
         if (journalDb()->getFileRecord(fileRecordPath, &rec) && rec.isValid()) {
-            [[maybe_unused]] const auto result = _vfs->updatePlaceholderMarkInSync(path() + rec.path(), rec._fileId);
+            const auto itemPointer = SyncFileItem::fromSyncJournalFileRecord(rec);
+            [[maybe_unused]] const auto result = _vfs->updatePlaceholderMarkInSync(path() + rec.path(), *itemPointer);
         }
     }
 }