/**************************************************************************** ** Artriculate: Art comes tumbling down ** Copyright (C) 2016 Chaos Reins ** ** This program is free software: you can redistribute it and/or modify ** it under the terms of the GNU General Public License as published by ** the Free Software Foundation, either version 3 of the License, or ** (at your option) any later version. ** ** This program is distributed in the hope that it will be useful, ** but WITHOUT ANY WARRANTY; without even the implied warranty of ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ** GNU General Public License for more details. ** ** You should have received a copy of the GNU General Public License ** along with this program. If not, see . ****************************************************************************/ #include "picturemodel.h" #include #include #include #include #include #include #include #include #include #include #include #include #include namespace { QSqlDatabase openDBConnection(const QString &connectionName) { QSqlDatabase db = QSqlDatabase::addDatabase("QSQLITE", connectionName); QFileInfo dbFile(QStandardPaths::standardLocations(QStandardPaths::DataLocation).first() + "/" + qApp->applicationName() + ".db"); QDir().mkpath(dbFile.absolutePath()); db.setDatabaseName(dbFile.absoluteFilePath()); if (!db.open()) { qDebug() << "Failed to open the database:" << dbFile.absoluteFilePath(); qDebug() << "Error:" << db.lastError().text(); qApp->exit(-1); } return db; } QString stripDbHostileCharacters(QString path) { return path.replace(QString("/"), QString("")); } inline int offsetHash(int hash) { return hash + 1; } } struct ArtPiece { ArtPiece() : refCount(0) { /**/ } QString path; QSize size; int refCount; }; struct FSNode { FSNode(const QString& rname, const FSNode *pparent = nullptr); QString qualify() const; const QString name; const FSNode *parent; }; struct FSLeafNode : public FSNode { using FSNode::FSNode; QSize size; }; FSNode::FSNode(const QString& rname, const FSNode *pparent) : name(rname), parent(pparent) { } QString FSNode::qualify() const { QString qualifiedPath; const FSNode *node = this; while(node->parent != nullptr) { qualifiedPath = "/" + node->name + qualifiedPath; node = node->parent; } qualifiedPath = node->name + qualifiedPath; return qualifiedPath; } class FSNodeTree : public QObject { Q_OBJECT public: FSNodeTree(const QString& path); virtual ~FSNodeTree(); void addModelNode(const FSNode* parentNode); int fileCount() const { return files.length(); } QVector files; public slots: void populate(bool useDatabaseBackend); signals: void countChanged(); private: void dumpTreeToDb(); QStringList extensions; QString rootDir; }; FSNodeTree::FSNodeTree(const QString& path) : QObject(nullptr), rootDir(path) { QMimeDatabase mimeDatabase; foreach(const QByteArray &m, QImageReader::supportedMimeTypes()) { foreach(const QString &suffix, mimeDatabase.mimeTypeForName(m).suffixes()) extensions.append(suffix); } if (extensions.isEmpty()) { qFatal("Your Qt install has no image format support"); } } FSNodeTree::~FSNodeTree() { QSet nodes; foreach(const FSNode *node, files) { while(node) { nodes << node; node = node->parent; } } qDeleteAll(nodes.values()); } void FSNodeTree::addModelNode(const FSNode* parentNode) { // TODO: Check for symlink recursion QDir parentDir(parentNode->qualify()); foreach(const QString ¤tDir, parentDir.entryList(QDir::Dirs | QDir::NoDotAndDotDot)) { const FSNode *dir = new FSNode(currentDir, parentNode); addModelNode(dir); } foreach(const QString ¤tFile, parentDir.entryList(QDir::Files)) { QString extension = currentFile.mid(currentFile.length() - 3); if (!extensions.contains(extension)) continue; FSLeafNode *file = new FSLeafNode(currentFile, parentNode); const QString fullPath = file->qualify(); QSize size = QImageReader(fullPath).size(); bool rational = false; if (size.isValid()) { file->size = size; qreal ratio = qreal(size.width())/size.height(); if ((ratio < 0.01) || (ratio > 100)) { qDebug() << "Image" << fullPath << "has excessive ratio" << ratio << "excluded"; } else { rational = true; } } else { qDebug() << "Discarding" << fullPath << "due to invalid size"; } if (rational) { files << file; emit countChanged(); } else { delete file; } } } void FSNodeTree::populate(bool useDatabaseBackend) { QElapsedTimer timer; timer.start(); QDir currentDir(rootDir); if (!currentDir.exists()) { qDebug() << "Being told to watch a non existent directory:" << rootDir; } addModelNode(new FSNode(rootDir)); qDebug() << "Completed building file tree containing:" << files.length() << "images after:" << timer.elapsed() << "ms"; if (useDatabaseBackend) { qDebug() << "No database found; dumping tree to db" << rootDir; timer.restart(); dumpTreeToDb(); qDebug() << "Completed database dump after:" << timer.elapsed() << "ms"; } } void FSNodeTree::dumpTreeToDb() { QSqlDatabase db = openDBConnection("write"); QSqlQuery q("", db); if (!q.exec(QString("create table %1 (path varchar, width integer, height integer)").arg(::stripDbHostileCharacters(rootDir)))) { qDebug() << "Failed to init DB with:" << q.lastError().text(); return; } qDebug() << "Database supports transactions" << QSqlDatabase::database().driver()->hasFeature(QSqlDriver::Transactions); // Turns out SQLITE has a 999 variable limit by default // Arch shieleded me from this int varLimitPerWave = 999; int varCountPerItem = 3; int itemCountPerWave = varLimitPerWave/varCountPerItem; int waveCount = files.length()/itemCountPerWave; const int waveTail = files.length()%itemCountPerWave; if (waveTail > 0) { waveCount += 1; } qDebug() << "About to drop" << files.length() << "files to DB"; qDebug() << "This will require" << waveCount << "separate DB transactions"; for (int wave = 0; wave < waveCount; wave++) { int itemCount = itemCountPerWave; if ((waveTail > 0) && (wave == waveCount - 1)) { itemCount = waveTail; } QString insertQuery = QString("INSERT INTO %1 (path, width, height) VALUES ").arg(::stripDbHostileCharacters(rootDir)); QString insertQueryValues("(?, ?, ?),"); insertQuery.reserve(insertQuery.size() + insertQueryValues.size()*itemCount); for(int i = 0; i < itemCount; i++) { insertQuery.append(insertQueryValues); } insertQuery = insertQuery.replace(insertQuery.length()-1, 1, ";"); db.transaction(); QSqlQuery query("", db); if (!query.prepare(insertQuery)) { qDebug() << "Query preperation failed with" << query.lastError().text(); return; } for(int i = wave*itemCountPerWave; i < (wave*itemCountPerWave + itemCount); i++) { const FSLeafNode *node = files.at(i); query.addBindValue(node->qualify()); query.addBindValue(node->size.width()); query.addBindValue(node->size.height()); } query.exec(); if (db.commit()) { qDebug() << "SQL transaction succeeded"; } else { qDebug() << "SQL transaction failed"; } QSqlError err = query.lastError(); if (err.type() != QSqlError::NoError) { qDebug() << "Database dump of content tree failed with" << err.text(); } else { qDebug() << "Successfully finished adding wave" << wave << "to DB" << rootDir; } } } class PictureModel::PictureModelPrivate { public: PictureModelPrivate(PictureModel* p); ~PictureModelPrivate(); FSNodeTree *fsTree; bool useDatabaseBackend; bool assumeLinearAccess = false; void cacheIndex(int index); void retireCachedIndex(int index); int itemCount(); QHash artwork; private: PictureModel *parent; int collectionSize; QString artPath; void createFSTree(const QString &path); QThread scanningThread; }; PictureModel::PictureModelPrivate::PictureModelPrivate(PictureModel* p) : fsTree(nullptr), parent(p) { QSettings settings; useDatabaseBackend = settings.value("useDatabaseBackend", true).toBool(); settings.setValue("useDatabaseBackend", useDatabaseBackend); artPath = settings.value("artPath", QStandardPaths::standardLocations(QStandardPaths::PicturesLocation).first()).toString(); settings.setValue("artPath", artPath); if (useDatabaseBackend) { QSqlDatabase db = openDBConnection("read"); QStringList tables = db.tables(); if (tables.contains(::stripDbHostileCharacters(artPath), Qt::CaseInsensitive)) { QString queryString = "SELECT COUNT(*) FROM " % ::stripDbHostileCharacters(artPath) % ";"; QSqlQuery query(queryString, db); query.next(); collectionSize = query.value(0).toInt(); QMetaObject::invokeMethod(parent, "countChanged"); qDebug() << "Using existing database entry for" << artPath; } else { qDebug() << "No database found; creating file tree" << artPath; createFSTree(artPath); } } else { createFSTree(artPath); } }; void PictureModel::PictureModelPrivate::createFSTree(const QString &path) { fsTree = new FSNodeTree(path); connect(fsTree, &FSNodeTree::countChanged, parent, &PictureModel::countChanged); fsTree->moveToThread(&scanningThread); scanningThread.start(); QMetaObject::invokeMethod(fsTree, "populate", Qt::QueuedConnection, Q_ARG(bool, useDatabaseBackend)); } PictureModel::PictureModelPrivate::~PictureModelPrivate() { if (fsTree) { scanningThread.quit(); scanningThread.wait(5000); delete fsTree; fsTree = nullptr; } } int PictureModel::PictureModelPrivate::itemCount() { return fsTree ? fsTree->fileCount() : collectionSize; }; void PictureModel::PictureModelPrivate::cacheIndex(int index) { int hashIndex = ::offsetHash(index); if (artwork.contains(hashIndex)) { artwork[hashIndex]->refCount++; return; } QString queryString = "SELECT path, width, height FROM " % ::stripDbHostileCharacters(artPath) % " LIMIT 1 OFFSET " % QString::number(index) % ";"; QSqlDatabase db = QSqlDatabase::database("read", true); QSqlQuery query(queryString, db); query.next(); ArtPiece *art = new ArtPiece; art->path = query.value(0).toString(); art->size = QSize(query.value(1).toInt(), query.value(2).toInt()); art->refCount++; artwork[hashIndex] = art; } void PictureModel::PictureModelPrivate::retireCachedIndex(int index) { int hashIndex = ::offsetHash(index); artwork[hashIndex]->refCount--; if (assumeLinearAccess || artwork[hashIndex]->refCount < 1) { delete artwork[hashIndex]; artwork.remove(hashIndex); } } PictureModel::PictureModel(QObject *parent) : QAbstractListModel(parent), d(new PictureModelPrivate(this)) { /**/ } PictureModel::~PictureModel() { delete d; d = nullptr; } int PictureModel::rowCount(const QModelIndex &parent) const { Q_UNUSED(parent) return d->itemCount(); } QVariant PictureModel::data(const QModelIndex &index, int role) const { if (d->assumeLinearAccess) { requestIndex(index.row()); } // What the fuck; Qt queries item 0 before we substantiate it // I get to offset my hash by 1 or loss a piece of art if (index.row() <= 0 || index.row() >= d->itemCount()) { switch (role) { case SizeRole: return QSize(1222,900); case NameRole: return "Qt logo"; case PathRole: default: return QString("qrc:///qt_logo_green_rgb.png"); } } if (d->fsTree) { switch (role) { case SizeRole: return d->fsTree->files.at(index.row())->size; case NameRole: return d->fsTree->files.at(index.row())->name; case PathRole: default: return QUrl::fromLocalFile(d->fsTree->files.at(index.row())->qualify()); } } else { int hashIndex = ::offsetHash(index.row()); switch (role) { case SizeRole: { return d->artwork[hashIndex]->size; } case NameRole: return d->artwork[hashIndex]->path; case PathRole: default: return QUrl::fromLocalFile(d->artwork[hashIndex]->path); } } return QVariant(); } int PictureModel::requestIndex(int index) const { if (index == -1) { index = d->itemCount() == 0 ? 0 : qrand() % d->itemCount(); } if (!d->fsTree) { d->cacheIndex(index); } return index; } void PictureModel::retireIndex(int index) const { if (!d->fsTree) { d->retireCachedIndex(index); } } void PictureModel::assumeLinearAccess() { d->assumeLinearAccess = true; } QHash PictureModel::roleNames() const { QHash roles; roles[NameRole] = "name"; roles[PathRole] = "path"; roles[SizeRole] = "size"; return roles; } #include "moc/picturemodel.moc"