From 71e6d12dd4d55c61585460b5895412e5a07cd38d Mon Sep 17 00:00:00 2001 From: jensenzhang Date: Wed, 16 Nov 2022 12:15:32 +0800 Subject: [PATCH] Merge pull request #4 from MattSlm:project-schema - Revised database schema - Fixed compilation issue TODO: DON'T MERGE TO zero-order-grad Signed-off-by: jensenzhang Signed-off-by: Kai Gao --- .gitignore | 1 + src/db/generic/Pair.h | 12 +- src/db/generic/ThrInfo.h | 7 +- src/db/mysql/OptimizerDataSource.cpp | 299 ++----- src/db/schema/mysql/fts-diff-8.1.0.sql | 38 + src/db/schema/mysql/fts-diff-9.0.1.sql | 51 -- src/db/schema/mysql/fts-schema-9.0.1.sql | 820 ------------------ src/server/services/optimizer/Optimizer.h | 14 +- .../server/services/optimizer/Optimizer.cpp | 21 + 9 files changed, 158 insertions(+), 1105 deletions(-) create mode 100644 src/db/schema/mysql/fts-diff-8.1.0.sql delete mode 100644 src/db/schema/mysql/fts-diff-9.0.1.sql delete mode 100644 src/db/schema/mysql/fts-schema-9.0.1.sql diff --git a/.gitignore b/.gitignore index 280bb0cb3..c5522de19 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +build *.info *.pyc *.rpm diff --git a/src/db/generic/Pair.h b/src/db/generic/Pair.h index b305d4494..38c51fd7c 100644 --- a/src/db/generic/Pair.h +++ b/src/db/generic/Pair.h @@ -27,9 +27,12 @@ #include "common/Uri.h" struct Pair { - std::string source, destination; + std::string source, destination, vo; - Pair(const std::string &s, const std::string &d): source(s), destination(d) { + Pair(const std::string &s, const std::string &d): source(s), destination(d), vo("") { + } + + Pair(const std::string &s, const std::string &d, const std::string &v): source(s), destination(d), vo(v) { } bool isLanTransfer() const { @@ -37,13 +40,16 @@ struct Pair { } }; +// FIXME: So far, assume each (source, destination) pair only has one vo first. +// Support multiple vo later. + // Required so it can be used as a key on a std::map inline bool operator < (const Pair &a, const Pair &b) { return a.source < b.source || (a.source == b.source && a.destination < b.destination); } inline bool operator == (const Pair &a, const Pair &b) { - return a.source == b.source && a.destination == b.destination; + return a.source == b.source && a.destination == b.destination; } inline std::ostream& operator << (std::ostream &os, const Pair &pair) { diff --git a/src/db/generic/ThrInfo.h b/src/db/generic/ThrInfo.h index a09cf529b..ff4df0c1c 100644 --- a/src/db/generic/ThrInfo.h +++ b/src/db/generic/ThrInfo.h @@ -27,11 +27,10 @@ #include "common/Uri.h" struct TransferredStat { - int64_t transferred, limit; + int64_t transferred; + double limit; - TransferredStat(const int64_t &t, const int64_t &l): - transferred(t), limit(l) - { + TransferredStat(const int64_t &t, const double &l): transferred(t), limit(l) { } }; diff --git a/src/db/mysql/OptimizerDataSource.cpp b/src/db/mysql/OptimizerDataSource.cpp index e773d4328..f56e8de08 100644 --- a/src/db/mysql/OptimizerDataSource.cpp +++ b/src/db/mysql/OptimizerDataSource.cpp @@ -19,13 +19,13 @@ */ #include -#include -#include -#include -#include +/* #include */ +/* #include */ +/* #include */ +/* #include */ #include "MySqlAPI.h" #include "db/generic/DbUtils.h" -#include "db/generic/ThrInfo.h" +/* #include "db/generic/ThrInfo.h" */ #include "common/Exceptions.h" #include "common/Logger.h" #include "sociConversions.h" @@ -117,15 +117,16 @@ class MySqlOptimizerDataSource: public OptimizerDataSource { std::list result; soci::rowset rs = (sql.prepare << - "SELECT DISTINCT source_se, dest_se " + "SELECT DISTINCT source_se, dest_se, vo_name " "FROM t_file " "WHERE file_state IN ('ACTIVE', 'SUBMITTED') " - "GROUP BY source_se, dest_se, file_state " + "GROUP BY source_se, dest_se, vo_name, file_state " "ORDER BY NULL" ); for (auto i = rs.begin(); i != rs.end(); ++i) { - result.push_back(Pair(i->get("source_se"), i->get("dest_se"))); + result.push_back(Pair(i->get("source_se"), i->get("dest_se"), + i->get("vo_name"))); } return result; @@ -168,7 +169,7 @@ class MySqlOptimizerDataSource: public OptimizerDataSource { " SELECT 1 AS configured, min_active, max_active FROM t_link_config WHERE source_se = :source AND dest_se = '*' UNION " " SELECT 1 AS configured, min_active, max_active FROM t_link_config WHERE source_se = '*' AND dest_se = :dest UNION " " SELECT 0 AS configured, min_active, max_active FROM t_link_config WHERE source_se = '*' AND dest_se = '*' " - ") AS lc LIMIT 1", + ") AS lc LIMIT 1", soci::use(pair.source, "source"), soci::use(pair.destination, "dest"), soci::into(range->specific), soci::into(range->min, isNullMin), soci::into(range->max, isNullMax); @@ -192,236 +193,97 @@ class MySqlOptimizerDataSource: public OptimizerDataSource { return currentActive; } - std::string getPairProject(cost Pair &pair) - { - std::string project_id; + std::string getTcnProject(const Pair &pair) { + std::string pid; + + const static std::string tname("t_tcn_projects"); soci::indicator isNullProject; sql << - "SELECT proj_id FROM (SELECT proj_id" - " FROM t_projects" - " WHERE vo_name = :qvo AND source_se = :qsrcSe AND dest_se = :qdstSe" - " UNION" - " SELECT proj_id" - " FROM t_projects" - " WHERE (vo_name = :qvo AND source_se = :qsrcSe) OR" - " (vo_name = :qvo AND dest_se = :qdstSe)" - " UNION" - " SELECT proj_id" - " FROM t_projects" - " WHERE (vo_name = :qvo ) ) as tpr" - " LIMIT 1", - soci::use(pair.source, "qsrcSe"), - soci::use(pair.destination, "qdstSe"), - spici::use(pair.vo, "qvo"), - soci::into(project_id, isNullProject); + "SELECT proj_id FROM (" + " SELECT proj_id FROM :tbl WHERE vo_name = :vo_name AND source_se = :source_se AND dest_se = :dest_se UNION" + " SELECT proj_id FROM :tbl WHERE vo_name = :vo_name AND source_se = :source_se AND dest_se = '*' UNION" + " SELECT proj_id FROM :tbl WHERE vo_name = :vo_name AND source_se = '*' AND dest_se = :dest_se UNION" + " SELECT proj_id FROM :tbl WHERE vo_name = :vo_name AND source_se = '*' AND dest_se = '*'" + " ) AS tpr LIMIT 1", + soci::use(tname, "tbl"), + soci::use(pair.vo, "vo_name"), + soci::use(pair.source, "source_se"), + soci::use(pair.destination, "dest_se"), + soci::into(pid, isNullProject); if (isNullProject == soci::i_null) { - project_id = "vo1"; - } - - return project_id; - } - - /*void getProjectTransfers(const std::string &proj_id, std::vector *pairs) - { - pairs.clear(); - std::string psrcSe, pdstSe, pvo, - psrcSe = pdstSe = pvo = ""; - sql << - "SELECT vo_name, source_se, dest_se" - " FROM t_projects" - " WHERE proj_id = :projix", - soci::use(proj_id, "projix"), - soci::into(pvo), soci::into(psrcSe), soci::into(pdstSe); - - //find - if (psrcSe == "*" && pdstSe == "*") - { - - } - else if (psrcSe == "*") - { - - } - else if (dstSe == "*") - { - - } - else - { - + pid = "project0"; } - }*/ - - void getPairLinks(const Pair &pair, std::vector &link_ids) - { - link_ids.clear(); - - soci::rowset links = (sql.prepare << - "SELECT plink_id" - " FROM t_routing " - " WHERE " - " source_se = :sourceSe AND dest_se = :destSe", - soci::use(pair.source, "sourceSe"), - soci::use(pair.destination, "destSe"); - - for (auto j = links.begin(); j != links.end(); ++j) { - auto plink_id = j->get("plink_id"); - - link_ids.push_back(plink_id); - } - - return; + return pid; } - void getPairBWLimits(const Pair &pair, std::map &link_limits) - { - link_limits.clear(); - - std::vector plinks; - - getPairLinks(pair, plinks); + void getTcnPipeResource(const Pair &pair, std::vector &usedResources) { + usedResources.clear(); - auto proj_id = getPairProject(pair); + const static std::string tname("t_tcn_resource_use"); - soci::rowset limits = (sql.prepare << - " SELECT plink_id, max_throughput" - " FROM t_bounds" - " WHERE proj_id = :projid", - soci::use(proj_id, "projid")); + soci::rowset resources = (sql.prepare << + "SELECT resc_id FROM :tbl" + "WHERE source_se = :source_se AND dest_se = :dest_se", + soci::use(tname, "tbl"), + soci::use(pair.source, "source_se"), + soci::use(pair.destination, "dest_se")); - for (auto j = limits.begin(); j != limits.end(); ++j) { - auto plink_id = j->get("plink_id"); - auto limit = j->get(limit, 0.0); + for (auto j = resources.begin(); j != resources.end(); ++j) { + auto rescId = j->get("resc_id"); - if ( std::find(plinks.begin(), plinks.end(), plink_id) != plinks.end()) - { - link_limits.insert({plink_id, limit}); - } - } - - return; - } - - /*void getSimilarityVector(std::vector &routing1, std::vector &routing2, - std::vector *similarity) - { - *similarity.clear(); - for (const auto &plink_id : routing1) - { - if (std::find(routing2.begin(), routing2.end(), plink_id) != routing2.end()) - { - *similarity.push_back(plink_id); - } + usedResources.push_back(rescId); } + } - return; - }*/ - - - void getPairLimitOnPLinks(const Pair &pair, - time_t windowStart, - std::map &thr_map) - { - thr_map.clear(); - - static struct tm nulltm = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - - time_t now = time(NULL); - time_t total_seconds = now - windowStart; - //select all physical links that this queue (project+pipe) is traversing. - //std::vector plinks; - //getPairLinks(pair, plinks); - //std::string proj_id=getPairProject(pair); - - //get and set the limits - initalize the transferred bytes - std::map link_limits; - getPairBWLimits(pair, link_limits); - - std::map::iterator it; + void getTcnResourceSpec(const std::string &project, std::map &resourceConstraints) { + resourceConstraints.clear(); - for (it = link_limits.begin(); it != link_limits.end(); it++) - { - thr_map.insert(std::make_pair(it->first, TransferredStat(0, it->second))); - } + const static std::string tname("t_tcn_resource_ctrlspec"); - soci::rowset transfers = (sql.prepare << - " SELECT t2.plink_id, t3.start_time, t3.finish_time, t3.transferred, t3.filesize" - " FROM t_routing t1" - " INNER JOIN t_routing t2 ON t2.plink_id = t1.plink_id" - " INNER JOIN t_file t3 ON t3.source_se=t2.source_se AND t3.dest_se = t2.dest_se" - " WHERE t1.source_se = :sourceSe" - " AND t1.dest_se = :destSe" - " AND t3.vo_name = :vo_name" - " AND t3.file_state = 'ACTIVE'" - " UNION ALL" - " SELECT t2.plink_id, t3.start_time, t3.finish_time, t3.transferred, t3.filesize" - " FROM t_routing t1" - " INNER JOIN t_routing t2 ON t2.plink_id = t1.plink_id" - " INNER JOIN t_file t3 ON t3.source_se=t2.source_se AND t3.dest_se = t2.dest_se" - " WHERE t1.source_se = :sourceSe" - " AND t1.dest_se = :destSe" - " AND t3.vo_name = :vo_name" - " AND t3.file_state IN ('FINISHED', 'ARCHIVING')" - " AND t3.finish_time >= (UTC_TIMESTAMP() - INTERVAL :interval SECOND)", - soci::use(pair.source, "sourceSe"), soci::use(pair.destination, "destSe"), soci::use(pair.vo_name, "vo_name"), - soci::use(totalSeconds, "interval")); + soci::rowset specs = (sql.prepare << + " SELECT resc_id, max_usage from :tbl WHERE proj_id = :proj_id" + " WHERE proj_id = :proj_id", + soci::use(tname, "tbl"), + soci::use(project, "proj_id")); - for (auto j = transfers.begin(); j != transfers.end(); ++j) { - auto plink_id = j->get("plink_id"); - auto transferred = j->get("transferred", 0.0); - auto filesize = j->get("filesize", 0.0); - auto starttm = j->get("start_time"); - auto endtm = j->get("finish_time", nulltm); - - time_t start = timegm(&starttm); - time_t end = timegm(&endtm); - time_t periodInWindow = 0; - double bytesInWindow = 0; - - // Not finish information - if (endtm.tm_year <= 0) { - bytesInWindow = transferred - } - // Finished - else { - bytesInWindow = filesize; - } + for (auto i = specs.begin(); i != specs.end(); ++i) { + auto rescId = i->get("resc_id"); + auto capacity = i->get("max_usage"); - std::map::iterator itf = thr_map.find(plink_id); - if (itf != m.end()) - { - itf->second.transferred += bytesInWindow}; - } - } - - return; + resourceConstraints[rescId] = capacity; + } } - int64_t getTransferredInfo(const Pair &pair, time_t windowStart) - { + int64_t getTransferredInfo(const Pair &pair, time_t windowStart) { static struct tm nulltm = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - *throughput = *filesizeAvg = *filesizeStdDev = 0; - time_t now = time(NULL); - time_t total_seconds = now-windowStart; - - soci::rowset transfers = (sql.prepare << - "SELECT start_time, finish_time, transferred, filesize " - " FROM t_file " - " WHERE " - " source_se = :sourceSe AND dest_se = :destSe AND vo_name = :vo_name AND file_state = 'ACTIVE' " - "UNION ALL " - "SELECT start_time, finish_time, transferred, filesize " - " FROM t_file USE INDEX(idx_finish_time)" - " WHERE " - " source_se = :sourceSe AND dest_se = :destSe AND vo_name = :vo_name " - " AND file_state IN ('FINISHED', 'ARCHIVING') AND finish_time >= (UTC_TIMESTAMP() - INTERVAL :interval SECOND)", - soci::use(pair.source, "sourceSe"), soci::use(pair.destination, "destSe"), soci::use(pair.vo_name, "vo_name"), - soci::use(totalSeconds, "interval")); + time_t total_seconds = now - windowStart; + + soci::rowset transfers = + (sql.prepare + << "SELECT start_time, finish_time, transferred, filesize " + " FROM t_file " + " WHERE " + " source_se = :sourceSe AND dest_se = :destSe AND " + " vo_name = :voName AND " + "file_state = 'ACTIVE' " + "UNION ALL " + "SELECT start_time, finish_time, transferred, filesize " + " FROM t_file USE INDEX(idx_finish_time)" + " WHERE " + " source_se = :sourceSe AND dest_se = :destSe " + " AND vo_name = :voName " + " AND file_state IN ('FINISHED', 'ARCHIVING') AND " + "finish_time >= (UTC_TIMESTAMP() - INTERVAL :interval " + "SECOND)", + soci::use(pair.source, "sourceSe"), + soci::use(pair.destination, "destSe"), + soci::use(pair.vo, "voName"), + soci::use(total_seconds, "interval")); int64_t totalBytes = 0; std::vector filesizes; @@ -439,18 +301,17 @@ class MySqlOptimizerDataSource: public OptimizerDataSource { // Not finish information if (endtm.tm_year <= 0) { - bytesInWindow = transferred + bytesInWindow = transferred; } // Finished else { - bytesInWindow = filesize; + bytesInWindow = filesize; } totalBytes += bytesInWindow; } - return totalBytes; + return totalBytes; } - void getThroughputInfo(const Pair &pair, const boost::posix_time::time_duration &interval, double *throughput, double *filesizeAvg, double *filesizeStdDev) diff --git a/src/db/schema/mysql/fts-diff-8.1.0.sql b/src/db/schema/mysql/fts-diff-8.1.0.sql new file mode 100644 index 000000000..ccda9af89 --- /dev/null +++ b/src/db/schema/mysql/fts-diff-8.1.0.sql @@ -0,0 +1,38 @@ +-- +-- FTS3 Schema 8.1.0 +-- Add "t_projects" table for project management +-- Add "t_routing" table for network resource management +-- Add "t_bound" table for resource control specification +-- + +DROP TABLE IF EXISTS `t_tcn_projects`; + +CREATE TABLE `t_tcn_projects` ( + `source_se` varchar(150) NOT NULL, + `dest_se` varchar(150) NOT NULL, + `vo_name` varchar(50) NOT NULL, + `proj_id` varchar(50) NOT NULL, + PRIMARY KEY (`source_se`, `dest_se`, `vo_name`) +); + +DROP TABLE IF EXISTS `t_tcn_resource_use`; + +CREATE TABLE `t_tcn_resource_use` ( + `source_se` varchar(50) NOT NULL, + `dest_se` varchar(150) NOT NULL, + `resc_id` varchar(64) NOT NULL, + PRIMARY KEY (`source_se`, `dest_se`, `resc_id`) +); + +DROP TABLE IF EXISTS `t_tcn_resource_ctrlspec`; + +CREATE TABLE `t_tcn_resource_ctrlspec` ( + `proj_id` varchar(50) NOT NULL, + `resc_id` varchar(64) NOT NULL, + `max_usage` double DEFAULT NULL, + `max_share` double DEFAULT NULL, + PRIMARY KEY (`proj_id`, `resc_id`) +); + +INSERT INTO t_schema_vers (major, minor, patch, message) +VALUES (8, 1, 0, 'Add new tables for TCN resource control'); diff --git a/src/db/schema/mysql/fts-diff-9.0.1.sql b/src/db/schema/mysql/fts-diff-9.0.1.sql deleted file mode 100644 index 5c93229e6..000000000 --- a/src/db/schema/mysql/fts-diff-9.0.1.sql +++ /dev/null @@ -1,51 +0,0 @@ --- --- FTS3 Schema 9.0.1 --- Per physical link control variables --- - -DROP TABLE IF EXISTS `t_plinks`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_plinks` ( - `plink_id` varchar(36) NOT NULL, - `ingress` varchar(150) NOT NULL, - `egress` varchar(150) NOT NULL, - `cap` int(11) DEFAULT '10240', - `ldelay` int(11) DEFAULT NULL, - PRIMARY KEY (`plink_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -`/*!40101 SET character_set_client = @saved_cs_client */;` -INSERT INTO t_link_config (plink_id, ingress, egress, cap, ldelay) -VALUES (UUID(), '*', '*', '10', '100'); - -DROP TABLE IF EXISTS `t_routing`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_routing` ( - `source_se` varchar(150) NOT NULL, - `dest_se ` varchar(150) NOT NULL, - `plink_id` varchar(36) DEFAULT NULL, - PRIMARY KEY (`source_se`, `dest_se`, `plink_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -`/*!40101 SET character_set_client = @saved_cs_client */;` -INSERT INTO t_link_config (source_se, dest_se, plink_id) -SELECT '*', '*', plink_id FROM `t_plinks` -LIMIT 1; -/*VALUES ('*', '*', '*', '*', NULL);*/ - - -DROP TABLE IF EXISTS `t_bounds`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_bounds` ( - `vo_name` varchar(150) NOT NULL, - `source_se` varchar(150) NOT NULL, - `dest_se` varchar(150) NOT NULL, - `plink_id` varchar(36) DEFAULT NULL, - `max_throughput` int(11) DEFAULT NULL, - `relative_share` int(11) DEFAULT '10', - PRIMARY KEY ('vo_name', 'source_se', 'dest_se', `plink_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -`/*!40101 SET character_set_client = @saved_cs_client */;` -INSERT INTO t_link_config (vo_name, source_se, dest_se, plink_id, max_throughput, relative_share) -VALUES ('*', '*', '*', '*', '5', '0'); diff --git a/src/db/schema/mysql/fts-schema-9.0.1.sql b/src/db/schema/mysql/fts-schema-9.0.1.sql deleted file mode 100644 index 2313ef83d..000000000 --- a/src/db/schema/mysql/fts-schema-9.0.1.sql +++ /dev/null @@ -1,820 +0,0 @@ --- MySQL dump 10.14 Distrib 5.5.68-MariaDB, for Linux (x86_64) --- --- Host: dbod-fts-dev.cern.ch Database: fts3 --- ------------------------------------------------------ --- Server version 8.0.18 - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - --- --- Table structure for table `t_activity_share_config` --- - -DROP TABLE IF EXISTS `t_activity_share_config`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_activity_share_config` ( - `vo` varchar(100) NOT NULL, - `activity_share` varchar(1024) NOT NULL, - `active` varchar(3) DEFAULT NULL, - PRIMARY KEY (`vo`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_authz_dn` --- - -DROP TABLE IF EXISTS `t_authz_dn`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_authz_dn` ( - `dn` varchar(255) NOT NULL, - `operation` varchar(64) NOT NULL, - PRIMARY KEY (`dn`,`operation`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_bad_dns` --- - -DROP TABLE IF EXISTS `t_bad_dns`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_bad_dns` ( - `dn` varchar(255) NOT NULL DEFAULT '', - `message` varchar(2048) DEFAULT NULL, - `addition_time` timestamp NULL DEFAULT NULL, - `admin_dn` varchar(255) DEFAULT NULL, - `status` varchar(10) DEFAULT NULL, - `wait_timeout` int(11) DEFAULT '0', - PRIMARY KEY (`dn`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_bad_ses` --- - -DROP TABLE IF EXISTS `t_bad_ses`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_bad_ses` ( - `se` varchar(256) NOT NULL DEFAULT '', - `message` varchar(2048) DEFAULT NULL, - `addition_time` timestamp NULL DEFAULT NULL, - `admin_dn` varchar(255) DEFAULT NULL, - `vo` varchar(100) DEFAULT NULL, - `status` varchar(10) DEFAULT NULL, - `wait_timeout` int(11) DEFAULT '0', - PRIMARY KEY (`se`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_cloudStorage` --- - -DROP TABLE IF EXISTS `t_cloudStorage`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_cloudStorage` ( - `cloudStorage_name` varchar(150) NOT NULL, - `app_key` varchar(255) DEFAULT NULL, - `app_secret` varchar(255) DEFAULT NULL, - `service_api_url` varchar(1024) DEFAULT NULL, - PRIMARY KEY (`cloudStorage_name`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_cloudStorageUser` --- - -DROP TABLE IF EXISTS `t_cloudStorageUser`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_cloudStorageUser` ( - `user_dn` varchar(700) NOT NULL DEFAULT '', - `vo_name` varchar(100) NOT NULL DEFAULT '', - `cloudStorage_name` varchar(150) NOT NULL, - `access_token` varchar(255) DEFAULT NULL, - `access_token_secret` varchar(255) DEFAULT NULL, - `request_token` varchar(255) DEFAULT NULL, - `request_token_secret` varchar(255) DEFAULT NULL, - PRIMARY KEY (`user_dn`,`vo_name`,`cloudStorage_name`), - KEY `cloudStorage_name` (`cloudStorage_name`), - CONSTRAINT `t_cloudStorageUser_ibfk_1` FOREIGN KEY (`cloudStorage_name`) REFERENCES `t_cloudStorage` (`cloudStorage_name`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_config_audit` --- - -DROP TABLE IF EXISTS `t_config_audit`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_config_audit` ( - `datetime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `dn` varchar(255) DEFAULT NULL, - `config` varchar(4000) DEFAULT NULL, - `action` varchar(100) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_credential` --- - -DROP TABLE IF EXISTS `t_credential`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_credential` ( - `dlg_id` char(16) NOT NULL, - `dn` varchar(255) NOT NULL, - `proxy` longtext, - `voms_attrs` longtext, - `termination_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (`dlg_id`,`dn`), - KEY `termination_time` (`termination_time`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_credential_cache` --- - -DROP TABLE IF EXISTS `t_credential_cache`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_credential_cache` ( - `dlg_id` char(16) NOT NULL, - `dn` varchar(255) NOT NULL, - `cert_request` longtext, - `priv_key` longtext, - `voms_attrs` longtext, - PRIMARY KEY (`dlg_id`,`dn`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_dm` --- - -DROP TABLE IF EXISTS `t_dm`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_dm` ( - `file_id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, - `job_id` char(36) NOT NULL, - `file_state` varchar(32) NOT NULL, - `dmHost` varchar(150) DEFAULT NULL, - `source_surl` varchar(900) DEFAULT NULL, - `dest_surl` varchar(900) DEFAULT NULL, - `source_se` varchar(150) DEFAULT NULL, - `dest_se` varchar(150) DEFAULT NULL, - `error_scope` varchar(32) DEFAULT NULL, - `error_phase` varchar(32) DEFAULT NULL, - `reason` varchar(2048) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, - `checksum` varchar(100) DEFAULT NULL, - `finish_time` timestamp NULL DEFAULT NULL, - `start_time` timestamp NULL DEFAULT NULL, - `internal_file_params` varchar(255) DEFAULT NULL, - `job_finished` timestamp NULL DEFAULT NULL, - `pid` int(11) DEFAULT NULL, - `tx_duration` double DEFAULT NULL, - `retry` int(11) DEFAULT '0', - `user_filesize` double DEFAULT NULL, - `file_metadata` varchar(255) DEFAULT NULL, - `activity` varchar(255) DEFAULT 'default', - `selection_strategy` varchar(255) DEFAULT NULL, - `dm_start` timestamp NULL DEFAULT NULL, - `dm_finished` timestamp NULL DEFAULT NULL, - `dm_token` varchar(255) DEFAULT NULL, - `retry_timestamp` timestamp NULL DEFAULT NULL, - `wait_timestamp` timestamp NULL DEFAULT NULL, - `wait_timeout` int(11) DEFAULT NULL, - `hashed_id` int(10) unsigned DEFAULT '0', - `vo_name` varchar(100) DEFAULT NULL, - PRIMARY KEY (`file_id`), - KEY `dm_job_id` (`job_id`), - CONSTRAINT `fk_dmjob_id` FOREIGN KEY (`job_id`) REFERENCES `t_job` (`job_id`) -) ENGINE=InnoDB AUTO_INCREMENT=534976 DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_dm_backup` --- - -DROP TABLE IF EXISTS `t_dm_backup`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_dm_backup` ( - `file_id` bigint(20) unsigned NOT NULL DEFAULT '0', - `job_id` char(36) NOT NULL, - `file_state` varchar(32) NOT NULL, - `dmHost` varchar(150) DEFAULT NULL, - `source_surl` varchar(900) DEFAULT NULL, - `dest_surl` varchar(900) DEFAULT NULL, - `source_se` varchar(150) DEFAULT NULL, - `dest_se` varchar(150) DEFAULT NULL, - `error_scope` varchar(32) DEFAULT NULL, - `error_phase` varchar(32) DEFAULT NULL, - `reason` varchar(2048) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, - `checksum` varchar(100) DEFAULT NULL, - `finish_time` timestamp NULL DEFAULT NULL, - `start_time` timestamp NULL DEFAULT NULL, - `internal_file_params` varchar(255) DEFAULT NULL, - `job_finished` timestamp NULL DEFAULT NULL, - `pid` int(11) DEFAULT NULL, - `tx_duration` double DEFAULT NULL, - `retry` int(11) DEFAULT '0', - `user_filesize` double DEFAULT NULL, - `file_metadata` varchar(255) DEFAULT NULL, - `activity` varchar(255) DEFAULT 'default', - `selection_strategy` varchar(255) DEFAULT NULL, - `dm_start` timestamp NULL DEFAULT NULL, - `dm_finished` timestamp NULL DEFAULT NULL, - `dm_token` varchar(255) DEFAULT NULL, - `retry_timestamp` timestamp NULL DEFAULT NULL, - `wait_timestamp` timestamp NULL DEFAULT NULL, - `wait_timeout` int(11) DEFAULT NULL, - `hashed_id` int(10) unsigned DEFAULT '0', - `vo_name` varchar(100) DEFAULT NULL -) ENGINE=ARCHIVE DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_file` --- - -DROP TABLE IF EXISTS `t_file`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_file` ( - `log_file_debug` tinyint(1) DEFAULT NULL, - `file_id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, - `file_index` int(11) DEFAULT NULL, - `job_id` char(36) NOT NULL, - `file_state` enum('STAGING','ARCHIVING','QOS_TRANSITION','QOS_REQUEST_SUBMITTED','STARTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','CANCELED','NOT_USED','ON_HOLD','ON_HOLD_STAGING','FORCE_START') NOT NULL, - `transfer_host` varchar(255) DEFAULT NULL, - `source_surl` varchar(1100) DEFAULT NULL, - `dest_surl` varchar(1100) DEFAULT NULL, - `source_se` varchar(255) DEFAULT NULL, - `dest_se` varchar(255) DEFAULT NULL, - `staging_host` varchar(1024) DEFAULT NULL, - `reason` varchar(2048) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, - `current_failures` int(11) DEFAULT NULL, - `filesize` bigint(20) DEFAULT NULL, - `checksum` varchar(100) DEFAULT NULL, - `finish_time` timestamp NULL DEFAULT NULL, - `start_time` timestamp NULL DEFAULT NULL, - `internal_file_params` varchar(255) DEFAULT NULL, - `pid` int(11) DEFAULT NULL, - `tx_duration` double DEFAULT NULL, - `throughput` float DEFAULT NULL, - `retry` int(11) DEFAULT '0', - `user_filesize` bigint(20) DEFAULT NULL, - `file_metadata` text, - `selection_strategy` char(32) DEFAULT NULL, - `staging_start` timestamp NULL DEFAULT NULL, - `staging_finished` timestamp NULL DEFAULT NULL, - `bringonline_token` varchar(255) DEFAULT NULL, - `retry_timestamp` timestamp NULL DEFAULT NULL, - `log_file` varchar(2048) DEFAULT NULL, - `t_log_file_debug` int(11) DEFAULT NULL, - `hashed_id` int(10) unsigned DEFAULT '0', - `vo_name` varchar(50) DEFAULT NULL, - `activity` varchar(255) DEFAULT 'default', - `transferred` bigint(20) DEFAULT '0', - `priority` int(11) DEFAULT '3', - `dest_surl_uuid` char(36) DEFAULT NULL, - `archive_start_time` timestamp NULL DEFAULT NULL, - `archive_finish_time` timestamp NULL DEFAULT NULL, - `staging_metadata` text, - `archive_metadata` text, - PRIMARY KEY (`file_id`), - UNIQUE KEY `dest_surl_uuid` (`dest_surl_uuid`), - KEY `idx_job_id` (`job_id`), - KEY `idx_activity` (`vo_name`,`activity`), - KEY `idx_link_state_vo` (`source_se`,`dest_se`,`file_state`,`vo_name`), - KEY `idx_finish_time` (`finish_time`), - KEY `idx_staging` (`file_state`,`vo_name`,`source_se`), - KEY `idx_state_host` (`file_state`,`transfer_host`), - KEY `idx_state` (`file_state`), - KEY `idx_host` (`transfer_host`), - CONSTRAINT `job_id` FOREIGN KEY (`job_id`) REFERENCES `t_job` (`job_id`) -) ENGINE=InnoDB AUTO_INCREMENT=6658108 DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_file_backup` --- - -DROP TABLE IF EXISTS `t_file_backup`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_file_backup` ( - `log_file_debug` tinyint(1) DEFAULT NULL, - `file_id` bigint(20) unsigned NOT NULL DEFAULT '0', - `file_index` int(11) DEFAULT NULL, - `job_id` char(36) NOT NULL, - `file_state` enum('STAGING','ARCHIVING','QOS_TRANSITION','QOS_REQUEST_SUBMITTED','STARTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','CANCELED','NOT_USED','ON_HOLD','ON_HOLD_STAGING','FORCE_START') NOT NULL, - `transfer_host` varchar(255) DEFAULT NULL, - `source_surl` varchar(1100) DEFAULT NULL, - `dest_surl` varchar(1100) DEFAULT NULL, - `source_se` varchar(255) DEFAULT NULL, - `dest_se` varchar(255) DEFAULT NULL, - `staging_host` varchar(1024) DEFAULT NULL, - `reason` varchar(2048) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, - `current_failures` int(11) DEFAULT NULL, - `filesize` bigint(20) DEFAULT NULL, - `checksum` varchar(100) DEFAULT NULL, - `finish_time` timestamp NULL DEFAULT NULL, - `start_time` timestamp NULL DEFAULT NULL, - `internal_file_params` varchar(255) DEFAULT NULL, - `pid` int(11) DEFAULT NULL, - `tx_duration` double DEFAULT NULL, - `throughput` float DEFAULT NULL, - `retry` int(11) DEFAULT '0', - `user_filesize` bigint(20) DEFAULT NULL, - `file_metadata` text, - `selection_strategy` char(32) DEFAULT NULL, - `staging_start` timestamp NULL DEFAULT NULL, - `staging_finished` timestamp NULL DEFAULT NULL, - `bringonline_token` varchar(255) DEFAULT NULL, - `retry_timestamp` timestamp NULL DEFAULT NULL, - `log_file` varchar(2048) DEFAULT NULL, - `t_log_file_debug` int(11) DEFAULT NULL, - `hashed_id` int(10) unsigned DEFAULT '0', - `vo_name` varchar(50) DEFAULT NULL, - `activity` varchar(255) DEFAULT 'default', - `transferred` bigint(20) DEFAULT '0', - `priority` int(11) DEFAULT '3', - `dest_surl_uuid` char(36) DEFAULT NULL, - `archive_start_time` timestamp NULL DEFAULT NULL, - `archive_finish_time` timestamp NULL DEFAULT NULL, - `staging_metadata` text, - `archive_metadata` text -) ENGINE=ARCHIVE DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_file_retry_errors` --- - -DROP TABLE IF EXISTS `t_file_retry_errors`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_file_retry_errors` ( - `file_id` bigint(20) unsigned NOT NULL, - `attempt` int(11) NOT NULL, - `datetime` timestamp NULL DEFAULT NULL, - `reason` varchar(2048) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL, - PRIMARY KEY (`file_id`,`attempt`), - KEY `idx_datetime` (`datetime`), - CONSTRAINT `t_file_retry_errors_ibfk_1` FOREIGN KEY (`file_id`) REFERENCES `t_file` (`file_id`) ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_gridmap` --- - -DROP TABLE IF EXISTS `t_gridmap`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_gridmap` ( - `dn` varchar(255) NOT NULL, - `vo` varchar(100) NOT NULL, - PRIMARY KEY (`dn`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_hosts` --- - -DROP TABLE IF EXISTS `t_hosts`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_hosts` ( - `hostname` varchar(64) NOT NULL, - `beat` timestamp NULL DEFAULT NULL, - `drain` int(11) DEFAULT '0', - `service_name` varchar(64) NOT NULL, - PRIMARY KEY (`hostname`,`service_name`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_job` --- - -DROP TABLE IF EXISTS `t_job`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_job` ( - `job_id` char(36) NOT NULL, - `job_state` enum('STAGING','ARCHIVING','QOS_TRANSITION','QOS_REQUEST_SUBMITTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','FINISHEDDIRTY','CANCELED','DELETE') NOT NULL, - `job_type` char(1) DEFAULT NULL, - `cancel_job` char(1) DEFAULT NULL, - `source_se` varchar(255) DEFAULT NULL, - `dest_se` varchar(255) DEFAULT NULL, - `user_dn` varchar(1024) DEFAULT NULL, - `cred_id` char(16) DEFAULT NULL, - `vo_name` varchar(50) DEFAULT NULL, - `reason` varchar(2048) DEFAULT NULL, - `submit_time` timestamp NULL DEFAULT NULL, - `priority` int(11) DEFAULT '3', - `submit_host` varchar(255) DEFAULT NULL, - `max_time_in_queue` int(11) DEFAULT NULL, - `space_token` varchar(255) DEFAULT NULL, - `internal_job_params` varchar(255) DEFAULT NULL, - `overwrite_flag` char(1) DEFAULT NULL, - `job_finished` timestamp NULL DEFAULT NULL, - `source_space_token` varchar(255) DEFAULT NULL, - `copy_pin_lifetime` int(11) DEFAULT NULL, - `checksum_method` char(1) DEFAULT NULL, - `bring_online` int(11) DEFAULT NULL, - `retry` int(11) DEFAULT '0', - `retry_delay` int(11) DEFAULT '0', - `target_qos` varchar(255) DEFAULT NULL, - `job_metadata` text, - `archive_timeout` int(11) DEFAULT NULL, - `dst_file_report` char(1) DEFAULT NULL, - `os_project_id` varchar(512) DEFAULT NULL, - PRIMARY KEY (`job_id`), - KEY `idx_vo_name` (`vo_name`), - KEY `idx_jobfinished` (`job_finished`), - KEY `idx_link` (`source_se`,`dest_se`), - KEY `idx_submission` (`submit_time`,`submit_host`), - KEY `idx_jobtype` (`job_type`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_job_backup` --- - -DROP TABLE IF EXISTS `t_job_backup`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_job_backup` ( - `job_id` char(36) NOT NULL, - `job_state` enum('STAGING','ARCHIVING','QOS_TRANSITION','QOS_REQUEST_SUBMITTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','FINISHEDDIRTY','CANCELED','DELETE') NOT NULL, - `job_type` char(1) DEFAULT NULL, - `cancel_job` char(1) DEFAULT NULL, - `source_se` varchar(255) DEFAULT NULL, - `dest_se` varchar(255) DEFAULT NULL, - `user_dn` varchar(1024) DEFAULT NULL, - `cred_id` char(16) DEFAULT NULL, - `vo_name` varchar(50) DEFAULT NULL, - `reason` varchar(2048) DEFAULT NULL, - `submit_time` timestamp NULL DEFAULT NULL, - `priority` int(11) DEFAULT '3', - `submit_host` varchar(255) DEFAULT NULL, - `max_time_in_queue` int(11) DEFAULT NULL, - `space_token` varchar(255) DEFAULT NULL, - `internal_job_params` varchar(255) DEFAULT NULL, - `overwrite_flag` char(1) DEFAULT NULL, - `job_finished` timestamp NULL DEFAULT NULL, - `source_space_token` varchar(255) DEFAULT NULL, - `copy_pin_lifetime` int(11) DEFAULT NULL, - `checksum_method` char(1) DEFAULT NULL, - `bring_online` int(11) DEFAULT NULL, - `retry` int(11) DEFAULT '0', - `retry_delay` int(11) DEFAULT '0', - `target_qos` varchar(255) DEFAULT NULL, - `job_metadata` text, - `archive_timeout` int(11) DEFAULT NULL, - `dst_file_report` char(1) DEFAULT NULL, - `os_project_id` varchar(512) DEFAULT NULL -) ENGINE=ARCHIVE DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_link_config` --- - -DROP TABLE IF EXISTS `t_link_config`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_link_config` ( - `source_se` varchar(150) NOT NULL, - `dest_se` varchar(150) NOT NULL, - `symbolic_name` varchar(150) NOT NULL, - `min_active` int(11) DEFAULT NULL, - `max_active` int(11) DEFAULT NULL, - `optimizer_mode` int(11) DEFAULT NULL, - `tcp_buffer_size` int(11) DEFAULT NULL, - `nostreams` int(11) DEFAULT NULL, - `no_delegation` varchar(3) DEFAULT NULL, - `3rd_party_turl` varchar(150) DEFAULT NULL, - PRIMARY KEY (`source_se`,`dest_se`), - UNIQUE KEY `symbolic_name` (`symbolic_name`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -`/*!40101 SET character_set_client = @saved_cs_client */;` -INSERT INTO t_link_config (source_se, dest_se, symbolic_name, min_active, max_active, optimizer_mode, nostreams, no_delegation) -VALUES ('*', '*', '*', 2, 130, 0, 10240, 2, 0, 'off'); - --- --- FTS3 Schema 9.0.1 --- Per physical link control variables --- - -DROP TABLE IF EXISTS `t_plinks`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_plinks` ( - `plink_id` varchar(36) NOT NULL, - `ingress` varchar(150) NOT NULL, - `egress` varchar(150) NOT NULL, - `cap` int(11) DEFAULT '10240', - `ldelay` int(11) DEFAULT NULL, - PRIMARY KEY (`plink_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -`/*!40101 SET character_set_client = @saved_cs_client */;` -INSERT INTO t_plinks (plink_id, ingress, egress, cap, ldelay) -VALUES (UUID(), '*', '*', '10', '100'); - -DROP TABLE IF EXISTS `t_projects`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_projects` ( - `vo_name` varchar(150) NOT NULL, - `source_se` varchar(150) NOT NULL, - `dest_se` varchar(150) NOT NULL, - `proj_id` varchar(36) NOT NULL, - PRIMARY KEY (`proj_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -`/*!40101 SET character_set_client = @saved_cs_client */;` -/*INSERT INTO t_projects (vo_name, source_se, dest_se, proj_id) -VALUES ('vo1', 'root://xrd1', 'root://xrd2', UUID()), - ('vo1', 'root://xrd1', 'root://xrd3', UUID());*/ /*demo 1*/ - -INSERT INTO t_projects (vo_name, source_se, dest_se, proj_id) /*demo 2*/ -VALUES ('vo1', '*', '*', UUID()), /* NOTE: both should use similar ends*/ - ('vo2', '*', '*', UUID()), - ('vo1', '*', '*', UUID()); - -DROP TABLE IF EXISTS `t_routing`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_routing` ( - `source_se` varchar(150) NOT NULL, - `dest_se` varchar(150) NOT NULL, - `plink_id` varchar(36) NOT NULL, - PRIMARY KEY (`source_se`, `dest_se`, `plink_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -`/*!40101 SET character_set_client = @saved_cs_client */;` - -INSERT INTO t_routing (source_se, dest_se, plink_id) -SELECT 'root://xrd1', 'root://xrd2', plink_id FROM t_plinks; - -INSERT INTO t_routing (source_se, dest_se, plink_id) -SELECT 'root://xrd1', 'root://xrd3', plink_id FROM t_plinks; - - -/*VALUES ('*', '*', '*', '*', NULL);*/ -DROP TABLE IF EXISTS `t_bounds`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_bounds` ( - `proj_id` varchar(150) NOT NULL, - /*`source_se` varchar(150) NOT NULL, - `dest_se` varchar(150) NOT NULL,*/ - `plink_id` varchar(36) NOT NULL, - `max_throughput` int(11) DEFAULT NULL, - `relative_share` int(11) DEFAULT '10', - PRIMARY KEY (`proj_id`, `plink_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -`/*!40101 SET character_set_client = @saved_cs_client */;` -/*INSERT INTO t_bounds (proj_id, plink_id, max_throughput, relative_share) -SELECT t1.proj_id, t2.plink_id, '5', '0' -FROM t_projects t1 -INNER JOIN t_routing t2 ON (t1.source_se = t2.source_se) - AND (t1.dest_se = t2.dest_se)*/ /*demo 1: exact matching */ -INSERT INTO t_bounds (proj_id, plink_id, max_throughput, relative_share) -SELECT t1.proj_id, t2.plink_id, '5', '0' -FROM t_projects t1 -CROSS JOIN t_plinks t2 /*demo 2: wildpcard matching */ - --- --- Table structure for table `t_oauth2_apps` --- - -DROP TABLE IF EXISTS `t_oauth2_apps`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_oauth2_apps` ( - `client_id` varchar(64) NOT NULL, - `client_secret` varchar(128) NOT NULL, - `owner` varchar(1024) NOT NULL, - `name` varchar(128) NOT NULL, - `description` varchar(512) DEFAULT NULL, - `website` varchar(1024) DEFAULT NULL, - `redirect_to` varchar(4096) DEFAULT NULL, - PRIMARY KEY (`client_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_oauth2_codes` --- - -DROP TABLE IF EXISTS `t_oauth2_codes`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_oauth2_codes` ( - `client_id` varchar(64) DEFAULT NULL, - `code` varchar(128) NOT NULL, - `scope` varchar(512) DEFAULT NULL, - `dlg_id` varchar(100) NOT NULL, - PRIMARY KEY (`code`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_oauth2_providers` --- - -DROP TABLE IF EXISTS `t_oauth2_providers`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_oauth2_providers` ( - `provider_url` varchar(250) NOT NULL, - `provider_jwk` varchar(1000) NOT NULL, - PRIMARY KEY (`provider_url`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_oauth2_tokens` --- - -DROP TABLE IF EXISTS `t_oauth2_tokens`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_oauth2_tokens` ( - `client_id` varchar(64) NOT NULL, - `scope` varchar(512) DEFAULT NULL, - `access_token` varchar(128) DEFAULT NULL, - `token_type` varchar(64) DEFAULT NULL, - `expires` datetime DEFAULT NULL, - `refresh_token` varchar(128) DEFAULT NULL, - `dlg_id` varchar(100) DEFAULT NULL, - PRIMARY KEY (`client_id`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_optimizer` --- - -DROP TABLE IF EXISTS `t_optimizer`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_optimizer` ( - `source_se` varchar(150) NOT NULL, - `dest_se` varchar(150) NOT NULL, - `datetime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `ema` double DEFAULT '0', - `active` int(11) DEFAULT '2', - `nostreams` int(11) DEFAULT '1', - PRIMARY KEY (`source_se`,`dest_se`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_optimizer_evolution` --- - -DROP TABLE IF EXISTS `t_optimizer_evolution`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_optimizer_evolution` ( - `datetime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `source_se` varchar(150) DEFAULT NULL, - `dest_se` varchar(150) DEFAULT NULL, - `active` int(11) DEFAULT NULL, - `throughput` float DEFAULT NULL, - `success` float DEFAULT NULL, - `rationale` text, - `diff` int(11) DEFAULT '0', - `actual_active` int(11) DEFAULT NULL, - `queue_size` int(11) DEFAULT NULL, - `ema` double DEFAULT NULL, - `filesize_avg` double DEFAULT NULL, - `filesize_stddev` double DEFAULT NULL, - KEY `idx_optimizer_evolution` (`source_se`,`dest_se`,`datetime`), - KEY `idx_datetime` (`datetime`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_schema_vers` --- - -DROP TABLE IF EXISTS `t_schema_vers`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_schema_vers` ( - `major` int(11) NOT NULL, - `minor` int(11) NOT NULL, - `patch` int(11) NOT NULL, - `message` text, - PRIMARY KEY (`major`,`minor`,`patch`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO t_schema_vers (major, minor, patch, message) -VALUES (8, 0, 1, 'Schema 8.0.1'); - --- --- Table structure for table `t_se` --- - -DROP TABLE IF EXISTS `t_se`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_se` ( - `storage` varchar(150) NOT NULL, - `site` varchar(45) DEFAULT NULL, - `metadata` text, - `ipv6` tinyint(1) DEFAULT NULL, - `udt` tinyint(1) DEFAULT NULL, - `debug_level` int(11) DEFAULT NULL, - `inbound_max_active` int(11) DEFAULT NULL, - `inbound_max_throughput` float DEFAULT NULL, - `outbound_max_active` int(11) DEFAULT NULL, - `outbound_max_throughput` float DEFAULT NULL, - `eviction` char(1) DEFAULT NULL, - PRIMARY KEY (`storage`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO t_se (storage, inbound_max_active, outbound_max_active) -VALUES ('*', 200, 200); - --- --- Table structure for table `t_server_config` --- - -DROP TABLE IF EXISTS `t_server_config`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_server_config` ( - `retry` int(11) DEFAULT '0', - `max_time_queue` int(11) DEFAULT '0', - `sec_per_mb` int(11) DEFAULT '0', - `global_timeout` int(11) DEFAULT '0', - `vo_name` varchar(100) DEFAULT NULL, - `no_streaming` varchar(3) DEFAULT NULL, - `show_user_dn` varchar(3) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO t_server_config (vo_name) -VALUES ('*'); - --- --- Table structure for table `t_share_config` --- - -DROP TABLE IF EXISTS `t_share_config`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_share_config` ( - `source` varchar(150) NOT NULL, - `destination` varchar(150) NOT NULL, - `vo` varchar(100) NOT NULL, - `active` int(11) NOT NULL, - PRIMARY KEY (`source`,`destination`,`vo`), - CONSTRAINT `t_share_config_fk` FOREIGN KEY (`source`, `destination`) REFERENCES `t_link_config` (`source_se`, `dest_se`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Table structure for table `t_stage_req` --- - -DROP TABLE IF EXISTS `t_stage_req`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `t_stage_req` ( - `vo_name` varchar(100) NOT NULL, - `host` varchar(150) NOT NULL, - `operation` varchar(150) NOT NULL, - `concurrent_ops` int(11) DEFAULT '0', - PRIMARY KEY (`vo_name`,`host`,`operation`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; diff --git a/src/server/services/optimizer/Optimizer.h b/src/server/services/optimizer/Optimizer.h index ded24d452..8398ff0f7 100644 --- a/src/server/services/optimizer/Optimizer.h +++ b/src/server/services/optimizer/Optimizer.h @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include "common/Uri.h" @@ -141,14 +141,12 @@ class OptimizerDataSource { // Get the stored optimizer value (current value) virtual int getOptimizerValue(const Pair&) = 0; - virtual std::string getPairProject(cost Pair &pair) = 0; + virtual std::string getTcnProject(const Pair &pair) = 0; - virtual void getPairLinks(const Pair &pair, std::vector &link_ids) = 0; + virtual void getTcnPipeResource(const Pair &pair, std::vector &usedResources) = 0; - virtual void getPairBWLimits(const Pair &pair, std::map &link_limits) = 0; - - virtual void getPairLimitOnPLinks(const Pair &pair, time_t windowStart, - std::map &thr_map) = 0; + virtual void getTcnResourceSpec(const std::string &project, + std::map &resourceConstraints) = 0; // Get the weighted throughput for the pair virtual void getThroughputInfo(const Pair &, const boost::posix_time::time_duration &, @@ -264,4 +262,4 @@ inline std::ostream& operator << (std::ostream &os, const Range &range) { } } -#endif // FTS3_OPTIMIZER_H \ No newline at end of file +#endif // FTS3_OPTIMIZER_H diff --git a/test/unit/server/services/optimizer/Optimizer.cpp b/test/unit/server/services/optimizer/Optimizer.cpp index 47c626ca0..313a5e6f0 100644 --- a/test/unit/server/services/optimizer/Optimizer.cpp +++ b/test/unit/server/services/optimizer/Optimizer.cpp @@ -196,6 +196,27 @@ class BaseOptimizerFixture: public OptimizerDataSource, public Optimizer { return totalSize; } + std::string getTcnProject(const Pair &pair) { + // TODO: mock function for unit test + return "project0"; + } + + void getTcnPipeResource(const Pair &pair, std::vector &usedResources) { + // TODO: mock function for unit test + return; + } + + void getTcnResourceSpec(const std::string &project, std::map &resourceConstraints) { + // TODO: mock function for unit test + return; + } + + void getPairLimitOnPLinks(const Pair &pair, time_t windowStart, + std::map &thr_map) { + // TODO: mock function for unit test + return; + } + time_t getAverageDuration(const Pair &pair, const boost::posix_time::time_duration &interval) { auto tsi = transferStore.find(pair); if (tsi == transferStore.end()) {